From f7431b432875e334170ccdb19934d05545d2cebd Mon Sep 17 00:00:00 2001 From: Ariel Weisberg Date: Thu, 5 Jul 2018 18:10:40 -0400 Subject: [PATCH] Transient Replication and Cheap Quorums Patch by Blake Eggleston, Benedict Elliott Smith, Marcus Eriksson, Alex Petrov, Ariel Weisberg; Reviewed by Blake Eggleston, Marcus Eriksson, Benedict Elliott Smith, Alex Petrov, Ariel Weisberg for CASSANDRA-14404 Co-authored-by: Blake Eggleston Co-authored-by: Benedict Elliott Smith Co-authored-by: Marcus Eriksson Co-authored-by: Alex Petrov --- CHANGES.txt | 1 + NEWS.txt | 4 + conf/cassandra.yaml | 4 + doc/source/architecture/dynamo.rst | 29 + doc/source/cql/ddl.rst | 14 +- ...er-internal-only-3.12.0.post0-5838e2fd.zip | Bin 0 -> 269418 bytes pylib/cqlshlib/cql3handling.py | 1 + pylib/cqlshlib/cqlshhandling.py | 1 + pylib/cqlshlib/test/test_cqlsh_completion.py | 6 +- pylib/cqlshlib/test/test_cqlsh_output.py | 3 +- .../cassandra/batchlog/BatchlogManager.java | 45 +- .../org/apache/cassandra/config/Config.java | 2 + .../cassandra/config/DatabaseDescriptor.java | 35 +- .../apache/cassandra/cql3/QueryProcessor.java | 13 +- .../cql3/statements/BatchStatement.java | 4 +- .../statements/BatchUpdatesCollector.java | 2 +- .../statements/ModificationStatement.java | 4 +- .../SingleTableUpdatesCollector.java | 2 +- .../cql3/statements/UpdatesCollector.java | 5 +- .../schema/AlterKeyspaceStatement.java | 86 +- .../schema/AlterTableStatement.java | 7 + .../schema/CreateIndexStatement.java | 5 + .../schema/CreateTableStatement.java | 9 + .../schema/CreateViewStatement.java | 5 + .../statements/schema/TableAttributes.java | 3 + .../cassandra/db/ColumnFamilyStore.java | 24 +- .../apache/cassandra/db/ConsistencyLevel.java | 211 +++-- .../cassandra/db/DiskBoundaryManager.java | 39 +- .../org/apache/cassandra/db/Memtable.java | 1 + .../cassandra/db/MutationVerbHandler.java | 5 +- .../db/PartitionRangeReadCommand.java | 28 +- .../org/apache/cassandra/db/ReadCommand.java | 33 +- .../apache/cassandra/db/SSTableImporter.java | 6 +- .../db/SinglePartitionReadCommand.java | 26 +- .../apache/cassandra/db/SystemKeyspace.java | 98 +- .../db/SystemKeyspaceMigrator40.java | 45 + .../apache/cassandra/db/TableCQLHelper.java | 1 + .../AbstractCompactionStrategy.java | 3 +- .../db/compaction/AbstractStrategyHolder.java | 7 +- .../db/compaction/CompactionManager.java | 293 ++++-- .../compaction/CompactionStrategyHolder.java | 34 +- .../compaction/CompactionStrategyManager.java | 108 +-- .../db/compaction/CompactionTask.java | 26 +- .../db/compaction/PendingRepairHolder.java | 42 +- .../db/compaction/PendingRepairManager.java | 45 +- .../cassandra/db/compaction/Scrubber.java | 4 +- .../cassandra/db/compaction/Upgrader.java | 10 +- .../cassandra/db/compaction/Verifier.java | 3 +- .../writers/CompactionAwareWriter.java | 2 + .../writers/DefaultCompactionWriter.java | 1 + .../writers/MajorLeveledCompactionWriter.java | 1 + .../writers/MaxSSTableSizeWriter.java | 1 + .../SplittingSizeTieredCompactionWriter.java | 1 + .../db/partitions/PartitionIterators.java | 12 - .../CassandraKeyspaceRepairManager.java | 10 +- .../db/repair/PendingAntiCompaction.java | 22 +- .../db/streaming/CassandraOutgoingFile.java | 11 +- .../db/streaming/CassandraStreamManager.java | 36 +- .../db/streaming/CassandraStreamReader.java | 2 +- .../apache/cassandra/db/view/TableViews.java | 5 + .../apache/cassandra/db/view/ViewBuilder.java | 19 +- .../apache/cassandra/db/view/ViewManager.java | 2 +- .../apache/cassandra/db/view/ViewUtils.java | 64 +- src/java/org/apache/cassandra/dht/Range.java | 27 +- .../dht/RangeFetchMapCalculator.java | 58 +- .../apache/cassandra/dht/RangeStreamer.java | 577 ++++++++---- .../org/apache/cassandra/dht/Splitter.java | 95 +- .../cassandra/dht/StreamStateStore.java | 25 +- .../ReplicationAwareTokenAllocator.java | 2 +- .../dht/tokenallocator/TokenAllocation.java | 6 +- .../exceptions/UnavailableException.java | 20 +- .../apache/cassandra/gms/EndpointState.java | 5 + .../apache/cassandra/hints/HintsService.java | 21 +- .../sstable/AbstractSSTableSimpleWriter.java | 3 +- .../apache/cassandra/io/sstable/SSTable.java | 13 + .../cassandra/io/sstable/SSTableLoader.java | 2 +- .../io/sstable/SSTableTxnWriter.java | 14 +- .../io/sstable/SimpleSSTableMultiWriter.java | 3 +- .../format/RangeAwareSSTableWriter.java | 8 +- .../io/sstable/format/SSTableReader.java | 5 + .../io/sstable/format/SSTableWriter.java | 17 +- .../cassandra/io/sstable/format/Version.java | 2 + .../io/sstable/format/big/BigFormat.java | 17 +- .../io/sstable/format/big/BigTableWriter.java | 3 +- .../sstable/metadata/IMetadataSerializer.java | 4 +- .../sstable/metadata/MetadataCollector.java | 8 +- .../sstable/metadata/MetadataSerializer.java | 4 +- .../io/sstable/metadata/StatsMetadata.java | 52 +- .../locator/AbstractEndpointSnitch.java | 38 +- .../AbstractNetworkTopologySnitch.java | 5 +- .../locator/AbstractReplicaCollection.java | 264 ++++++ .../locator/AbstractReplicationStrategy.java | 142 +-- .../locator/DynamicEndpointSnitch.java | 67 +- .../apache/cassandra/locator/Ec2Snitch.java | 2 +- .../apache/cassandra/locator/Endpoints.java | 157 ++++ .../cassandra/locator/EndpointsByRange.java | 63 ++ .../cassandra/locator/EndpointsByReplica.java | 61 ++ .../cassandra/locator/EndpointsForRange.java | 188 ++++ .../cassandra/locator/EndpointsForToken.java | 172 ++++ .../cassandra/locator/IEndpointSnitch.java | 18 +- .../cassandra/locator/InetAddressAndPort.java | 5 +- .../cassandra/locator/LocalStrategy.java | 29 +- .../locator/NetworkTopologyStrategy.java | 87 +- .../locator/OldNetworkTopologyStrategy.java | 40 +- .../cassandra/locator/PendingRangeMaps.java | 161 ++-- .../cassandra/locator/RangesAtEndpoint.java | 313 +++++++ .../cassandra/locator/RangesByEndpoint.java | 54 ++ .../org/apache/cassandra/locator/Replica.java | 196 ++++ .../cassandra/locator/ReplicaCollection.java | 160 ++++ .../cassandra/locator/ReplicaLayout.java | 381 ++++++++ .../cassandra/locator/ReplicaMultimap.java | 127 +++ .../apache/cassandra/locator/Replicas.java | 83 ++ .../cassandra/locator/ReplicationFactor.java | 130 +++ .../cassandra/locator/SimpleSnitch.java | 8 +- .../cassandra/locator/SimpleStrategy.java | 37 +- .../cassandra/locator/SystemReplicas.java | 62 ++ .../cassandra/locator/TokenMetadata.java | 102 +- .../cassandra/metrics/KeyspaceMetrics.java | 43 +- .../cassandra/metrics/ReadRepairMetrics.java | 1 + .../cassandra/metrics/TableMetrics.java | 17 +- .../apache/cassandra/net/IAsyncCallback.java | 11 +- .../cassandra/net/MessagingService.java | 36 +- .../cassandra/net/WriteCallbackInfo.java | 15 +- .../cassandra/repair/AbstractSyncTask.java | 31 + .../repair/AsymmetricLocalSyncTask.java | 7 +- .../repair/AsymmetricRemoteSyncTask.java | 6 + .../cassandra/repair/AsymmetricSyncTask.java | 10 +- .../apache/cassandra/repair/CommonRange.java | 82 ++ .../repair/KeyspaceRepairManager.java | 8 +- .../apache/cassandra/repair/RepairJob.java | 42 +- .../cassandra/repair/RepairRunnable.java | 87 +- .../cassandra/repair/RepairSession.java | 57 +- .../cassandra/repair/StreamingRepairTask.java | 8 +- ...cTask.java => SymmetricLocalSyncTask.java} | 25 +- ...Task.java => SymmetricRemoteSyncTask.java} | 22 +- .../{SyncTask.java => SymmetricSyncTask.java} | 11 +- .../repair/SystemDistributedKeyspace.java | 6 +- .../repair/consistent/LocalSessions.java | 36 +- .../cassandra/schema/KeyspaceParams.java | 5 + .../cassandra/schema/ReplicationParams.java | 9 +- .../cassandra/schema/SchemaKeyspace.java | 4 + .../cassandra/schema/TableMetadata.java | 6 + .../apache/cassandra/schema/TableParams.java | 11 + .../service/AbstractWriteResponseHandler.java | 95 +- .../service/ActiveRepairService.java | 52 +- .../service/BatchlogResponseHandler.java | 2 +- .../DatacenterSyncWriteResponseHandler.java | 21 +- .../DatacenterWriteResponseHandler.java | 28 +- .../PendingRangeCalculatorService.java | 2 +- .../cassandra/service/StorageProxy.java | 680 ++++++-------- .../cassandra/service/StorageService.java | 876 +++++++++++------- .../service/StorageServiceMBean.java | 2 + .../service/WriteResponseHandler.java | 25 +- .../service/reads/AbstractReadExecutor.java | 249 +++-- .../cassandra/service/reads/DataResolver.java | 83 +- .../service/reads/DigestResolver.java | 81 +- .../cassandra/service/reads/ReadCallback.java | 57 +- .../service/reads/ResponseResolver.java | 32 +- .../reads/ShortReadPartitionsProtection.java | 36 +- .../service/reads/ShortReadProtection.java | 3 +- .../reads/ShortReadRowsProtection.java | 6 +- .../reads/repair/AbstractReadRepair.java | 88 +- .../reads/repair/BlockingPartitionRepair.java | 73 +- .../reads/repair/BlockingReadRepair.java | 29 +- .../reads/repair/BlockingReadRepairs.java | 19 - .../service/reads/repair/NoopReadRepair.java | 15 +- .../PartitionIteratorMergeListener.java | 14 +- .../reads/repair/ReadOnlyReadRepair.java | 15 +- .../service/reads/repair/ReadRepair.java | 39 +- .../reads/repair/ReadRepairDiagnostics.java | 5 +- .../service/reads/repair/ReadRepairEvent.java | 11 +- .../reads/repair/ReadRepairStrategy.java | 13 +- .../repair/RowIteratorMergeListener.java | 58 +- .../streaming/DefaultConnectionFactory.java | 7 +- .../cassandra/streaming/StreamPlan.java | 38 +- .../cassandra/streaming/StreamRequest.java | 98 +- .../cassandra/streaming/StreamSession.java | 54 +- .../streaming/TableStreamManager.java | 7 +- .../org/apache/cassandra/tools/NodeProbe.java | 5 + .../org/apache/cassandra/tools/NodeTool.java | 2 + .../tools/SSTableRepairedAtSetter.java | 4 +- .../cassandra/tools/nodetool/GetReplicas.java | 47 + .../apache/cassandra/tracing/TraceState.java | 2 +- .../transport/messages/ErrorMessage.java | 2 +- src/java/org/apache/cassandra/utils/Pair.java | 12 + .../utils/concurrent/Accumulator.java | 27 +- .../na-1-big-CompressionInfo.db | Bin 87 -> 87 bytes .../legacy_na_clust/na-1-big-Data.db | Bin 5259 -> 5214 bytes .../legacy_na_clust/na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../legacy_na_clust/na-1-big-Statistics.db | Bin 7095 -> 7096 bytes .../legacy_na_clust/na-1-big-TOC.txt | 8 +- .../na-1-big-CompressionInfo.db | Bin 79 -> 79 bytes .../legacy_na_clust_counter/na-1-big-Data.db | Bin 5888 -> 5759 bytes .../na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust_counter/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../na-1-big-Statistics.db | Bin 7104 -> 7105 bytes .../legacy_na_clust_counter/na-1-big-TOC.txt | 8 +- .../legacy_na_simple/na-1-big-Data.db | Bin 89 -> 88 bytes .../legacy_na_simple/na-1-big-Digest.crc32 | 2 +- .../legacy_na_simple/na-1-big-Statistics.db | Bin 4648 -> 4649 bytes .../legacy_na_simple/na-1-big-TOC.txt | 8 +- .../legacy_na_simple_counter/na-1-big-Data.db | Bin 140 -> 138 bytes .../na-1-big-Digest.crc32 | 2 +- .../na-1-big-Statistics.db | Bin 4657 -> 4658 bytes .../legacy_na_simple_counter/na-1-big-TOC.txt | 8 +- .../DynamicEndpointSnitchLongTest.java | 20 +- .../streaming/LongStreamingTest.java | 11 +- .../test/microbench/PendingRangesBench.java | 27 +- test/unit/org/apache/cassandra/Util.java | 20 + .../config/DatabaseDescriptorRefTest.java | 5 +- .../org/apache/cassandra/cql3/CQLTester.java | 3 +- .../validation/operations/CreateTest.java | 3 +- .../org/apache/cassandra/db/CleanupTest.java | 9 +- .../cassandra/db/CleanupTransientTest.java | 195 ++++ .../org/apache/cassandra/db/ImportTest.java | 2 +- .../db/RepairedDataTombstonesTest.java | 2 +- .../apache/cassandra/db/RowUpdateBuilder.java | 6 + .../org/apache/cassandra/db/ScrubTest.java | 6 +- .../db/SystemKeyspaceMigrator40Test.java | 26 + .../cassandra/db/TableCQLHelperTest.java | 3 + .../org/apache/cassandra/db/VerifyTest.java | 4 +- .../compaction/AbstractPendingRepairTest.java | 13 +- .../db/compaction/AntiCompactionTest.java | 246 +++-- ...ctionStrategyManagerPendingRepairTest.java | 163 +++- .../CompactionStrategyManagerTest.java | 50 +- .../db/compaction/CompactionTaskTest.java | 10 +- .../db/compaction/CompactionsCQLTest.java | 4 +- .../LeveledCompactionStrategyTest.java | 2 +- .../compaction/PendingRepairManagerTest.java | 28 +- .../compaction/SingleSSTableLCSTaskTest.java | 6 +- .../db/lifecycle/LogTransactionTest.java | 2 +- .../db/lifecycle/RealTransactionsTest.java | 1 + ...onManagerGetSSTablesForValidationTest.java | 4 +- .../db/repair/PendingAntiCompactionTest.java | 39 +- .../streaming/CassandraOutgoingFileTest.java | 1 + .../streaming/CassandraStreamManagerTest.java | 16 +- .../db/streaming/StreamRequestTest.java | 98 ++ .../cassandra/db/view/ViewUtilsTest.java | 23 +- .../cassandra/dht/BootStrapperTest.java | 34 +- .../dht/RangeFetchMapCalculatorTest.java | 138 ++- .../org/apache/cassandra/dht/RangeTest.java | 12 +- .../apache/cassandra/dht/SplitterTest.java | 63 +- .../cassandra/dht/StreamStateStoreTest.java | 5 +- .../PendingRangeCalculatorServiceTest.java | 2 +- .../io/sstable/BigTableWriterTest.java | 2 +- .../io/sstable/CQLSSTableWriterTest.java | 2 +- .../io/sstable/LegacySSTableTest.java | 9 +- .../io/sstable/SSTableLoaderTest.java | 7 +- .../io/sstable/SSTableRewriterTest.java | 2 +- .../cassandra/io/sstable/SSTableUtils.java | 2 +- .../io/sstable/SSTableWriterTest.java | 60 ++ .../io/sstable/SSTableWriterTestBase.java | 11 +- .../format/SSTableFlushObserverTest.java | 2 +- .../metadata/MetadataSerializerTest.java | 2 +- .../locator/DynamicEndpointSnitchTest.java | 42 +- .../locator/NetworkTopologyStrategyTest.java | 92 +- .../OldNetworkTopologyStrategyTest.java | 28 +- .../locator/PendingRangeMapsTest.java | 49 +- .../locator/ReplicaCollectionTest.java | 468 ++++++++++ .../cassandra/locator/ReplicaUtils.java | 44 + .../locator/ReplicationFactorTest.java | 73 ++ .../ReplicationStrategyEndpointCacheTest.java | 56 +- .../cassandra/locator/SimpleStrategyTest.java | 81 +- .../cassandra/locator/TokenMetadataTest.java | 8 +- .../cassandra/net/WriteCallbackInfoTest.java | 7 +- .../OutboundMessagingConnectionTest.java | 3 +- .../cassandra/repair/RepairRunnableTest.java | 12 +- .../cassandra/repair/RepairSessionTest.java | 6 +- ...t.java => SymmetricLocalSyncTaskTest.java} | 57 +- .../repair/SymmetricRemoteSyncTaskTest.java | 71 ++ .../consistent/LocalSessionAccessor.java | 3 +- .../repair/consistent/LocalSessionTest.java | 9 +- .../apache/cassandra/schema/MockSchema.java | 2 +- .../service/ActiveRepairServiceTest.java | 50 +- .../service/BootstrapTransientTest.java | 179 ++++ .../service/LeaveAndBootstrapTest.java | 48 +- .../apache/cassandra/service/MoveTest.java | 324 +++---- .../cassandra/service/MoveTransientTest.java | 638 +++++++++++++ .../cassandra/service/StorageServiceTest.java | 148 +++ .../service/WriteResponseHandlerTest.java | 58 +- .../WriteResponseHandlerTransientTest.java | 224 +++++ .../reads/AbstractReadResponseTest.java | 300 ++++++ .../service/reads/DataResolverTest.java | 484 ++++------ .../reads/DataResolverTransientTest.java | 226 +++++ .../service/reads/DigestResolverTest.java | 144 +++ .../service/reads/ReadExecutorTest.java | 46 +- .../reads/repair/AbstractReadRepairTest.java | 55 +- .../reads/repair/BlockingReadRepairTest.java | 113 +-- .../DiagEventsBlockingReadRepairTest.java | 45 +- .../reads/repair/InstrumentedReadRepair.java | 4 +- .../reads/repair/ReadOnlyReadRepairTest.java | 30 +- .../service/reads/repair/ReadRepairTest.java | 353 +++++++ .../reads/repair/TestableReadRepair.java | 50 +- .../streaming/StreamingTransferTest.java | 7 +- .../utils/concurrent/AccumulatorTest.java | 2 +- 296 files changed, 11501 insertions(+), 3903 deletions(-) create mode 100644 lib/cassandra-driver-internal-only-3.12.0.post0-5838e2fd.zip create mode 100644 src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java create mode 100644 src/java/org/apache/cassandra/locator/Endpoints.java create mode 100644 src/java/org/apache/cassandra/locator/EndpointsByRange.java create mode 100644 src/java/org/apache/cassandra/locator/EndpointsByReplica.java create mode 100644 src/java/org/apache/cassandra/locator/EndpointsForRange.java create mode 100644 src/java/org/apache/cassandra/locator/EndpointsForToken.java create mode 100644 src/java/org/apache/cassandra/locator/RangesAtEndpoint.java create mode 100644 src/java/org/apache/cassandra/locator/RangesByEndpoint.java create mode 100644 src/java/org/apache/cassandra/locator/Replica.java create mode 100644 src/java/org/apache/cassandra/locator/ReplicaCollection.java create mode 100644 src/java/org/apache/cassandra/locator/ReplicaLayout.java create mode 100644 src/java/org/apache/cassandra/locator/ReplicaMultimap.java create mode 100644 src/java/org/apache/cassandra/locator/Replicas.java create mode 100644 src/java/org/apache/cassandra/locator/ReplicationFactor.java create mode 100644 src/java/org/apache/cassandra/locator/SystemReplicas.java create mode 100644 src/java/org/apache/cassandra/repair/AbstractSyncTask.java create mode 100644 src/java/org/apache/cassandra/repair/CommonRange.java rename src/java/org/apache/cassandra/repair/{LocalSyncTask.java => SymmetricLocalSyncTask.java} (81%) rename src/java/org/apache/cassandra/repair/{RemoteSyncTask.java => SymmetricRemoteSyncTask.java} (77%) rename src/java/org/apache/cassandra/repair/{SyncTask.java => SymmetricSyncTask.java} (87%) create mode 100644 src/java/org/apache/cassandra/tools/nodetool/GetReplicas.java create mode 100644 test/unit/org/apache/cassandra/db/CleanupTransientTest.java create mode 100644 test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java create mode 100644 test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java create mode 100644 test/unit/org/apache/cassandra/locator/ReplicaUtils.java create mode 100644 test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java rename test/unit/org/apache/cassandra/repair/{LocalSyncTaskTest.java => SymmetricLocalSyncTaskTest.java} (73%) create mode 100644 test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java create mode 100644 test/unit/org/apache/cassandra/service/BootstrapTransientTest.java create mode 100644 test/unit/org/apache/cassandra/service/MoveTransientTest.java create mode 100644 test/unit/org/apache/cassandra/service/StorageServiceTest.java create mode 100644 test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java create mode 100644 test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java create mode 100644 test/unit/org/apache/cassandra/service/reads/DataResolverTransientTest.java create mode 100644 test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java create mode 100644 test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 9e765861db..b53b9863bb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Transient Replication and Cheap Quorums (CASSANDRA-14404) * Log server-generated timestamp and nowInSeconds used by queries in FQL (CASSANDRA-14675) * Add diagnostic events for read repairs (CASSANDRA-14668) * Use consistent nowInSeconds and timestamps values within a request (CASSANDRA-14671) diff --git a/NEWS.txt b/NEWS.txt index aa8281cf55..506637821e 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -38,6 +38,10 @@ using the provided 'sstableupgrade' tool. New features ------------ + - *Experimental* support for Transient Replication and Cheap Quorums introduced by CASSANDRA-14404 + The intended audience for this functionality is expert users of Cassandra who are prepared + to validate every aspect of the database for their application and deployment practices. Future + releases of Cassandra will make this feature suitable for a wider audience. - *Experimental* support for Java 11 has been added. JVM options that differ between or are specific for Java 8 and 11 have been moved from jvm.options into jvm8.options and jvm11.options. IMPORTANT: Running C* on Java 11 is *experimental* and do it at your own risk. diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 064ee4fdd6..503a0fa0df 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1052,6 +1052,10 @@ enable_scripted_user_defined_functions: false # Materialized views are considered experimental and are not recommended for production use. enable_materialized_views: true +# Enables creation of transiently replicated keyspaces on this node. +# Transient replication is experimental and is not recommended for production use. +#enable_transient_replication: true + # The default Windows kernel timer and scheduling resolution is 15.6ms for power conservation. # Lowering this value on Windows can provide much tighter latency and better throughput, however # some virtualized environments may see a negative performance impact from changing this setting diff --git a/doc/source/architecture/dynamo.rst b/doc/source/architecture/dynamo.rst index 365a695cbb..12c586e2cb 100644 --- a/doc/source/architecture/dynamo.rst +++ b/doc/source/architecture/dynamo.rst @@ -74,6 +74,35 @@ nodes in each rack, the data load on the smallest rack may be much higher. Simi into a new rack, it will be considered a replica for the entire ring. For this reason, many operators choose to configure all nodes on a single "rack". +.. _transient-replication: + +Transient Replication +~~~~~~~~~~~~~~~~~~~~~ + +Transient replication allows you to configure a subset of replicas to only replicate data that hasn't been incrementally +repaired. This allows you to decouple data redundancy from availability. For instance, if you have a keyspace replicated +at rf 3, and alter it to rf 5 with 2 transient replicas, you go from being able to tolerate one failed replica to being +able to tolerate two, without corresponding increase in storage usage. This is because 3 nodes will replicate all the data +for a given token range, and the other 2 will only replicate data that hasn't been incrementally repaired. + +To use transient replication, you first need to enable it in ``cassandra.yaml``. Once enabled, both SimpleStrategy and +NetworkTopologyStrategy can be configured to transiently replicate data. You configure it by specifying replication factor +as ``/` has been enabled, transient replicas can be configured for both +SimpleStrategy and NetworkTopologyStrategy by defining replication factors in the format ``'/'`` + +For instance, this keyspace will have 3 replicas in DC1, 1 of which is transient, and 5 replicas in DC2, 2 of which are transient:: + + CREATE KEYSPACE some_keysopace + WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1' : '3/1'', 'DC2' : '5/2'}; + .. _use-statement: USE @@ -455,6 +463,9 @@ A table supports the following options: | ``speculative_retry`` | *simple* | 99PERCENTILE| :ref:`Speculative retry options | | | | | `. | +--------------------------------+----------+-------------+-----------------------------------------------------------+ +| ``speculative_write_threshold``| *simple* | 99PERCENTILE| :ref:`Speculative retry options | +| | | | `. | ++--------------------------------+----------+-------------+-----------------------------------------------------------+ | ``gc_grace_seconds`` | *simple* | 864000 | Time to wait before garbage collecting tombstones | | | | | (deletion markers). | +--------------------------------+----------+-------------+-----------------------------------------------------------+ @@ -485,7 +496,8 @@ Speculative retry options By default, Cassandra read coordinators only query as many replicas as necessary to satisfy consistency levels: one for consistency level ``ONE``, a quorum for ``QUORUM``, and so on. ``speculative_retry`` determines when coordinators may query additional replicas, which is useful -when replicas are slow or unresponsive. The following are legal values (case-insensitive): +when replicas are slow or unresponsive. ``speculative_write_threshold`` specifies the threshold at which +a cheap quorum write will be upgraded to include transient replicas. The following are legal values (case-insensitive): ============================ ======================== ============================================================================= Format Example Description diff --git a/lib/cassandra-driver-internal-only-3.12.0.post0-5838e2fd.zip b/lib/cassandra-driver-internal-only-3.12.0.post0-5838e2fd.zip new file mode 100644 index 0000000000000000000000000000000000000000..8d627a9f818cd39d61a009d470e72aac8460e075 GIT binary patch literal 269418 zcmbTeV~}oLvn}|PZQHhMmv-5-2YeMdXb2V@2c| zf94!Ha?F&M0s(~v{OchsyqErum;Y^H0^kA+^&K7cZHyfB>5LrAU5p**Sm>FU=^5$m zY#p5#>DV|~IE|T2j2Qlp&lpscp#UJ8A#z&up>kTzZZH5K&{Gfq;6E1m|JPIk|36ac z{?9+6H#Rk;Gq*9Z{r^mj`VTHhfRvf2^lRlWx0ioFClUbw%>RCBV-H*dXHtlUlP}3ng@Oe>*=x=;@yE>F{#(emZ8`)XgOOkLjVvAnp$BWbsl7a~u1Xj0qJ5;#i7|OgP-l4Q;JuH8!<@IcbB8?DX%cb2Sc5 zG;eg$;xpKXDRi(QjI~-;_wX<~M<+%kz0a09-vF*5Lc0C6`g_fONqFz57nJ#_4&Jb3 z6?wdT)CG(&o@)&<;Rbpn&lO+6k6ue1B`O6Oje^Lkm9!o$O3gBNEE8Y$WNe?brVu@~ z>-(6bIZjqxxqYS~8*pi<0+FX_(>!*=e*+qEX9Cv88UJ*!c5;+QoIG)>$GB?QX2@M; zGlKrGNJ5phz&L^6uKk3t=1gF?SF6=#62+Qe2IEZ>cqKT3OaB)eV#MKg)Tzf(1BSA; zc1xE3X@d$XMWn>*G&085h;$?}0#TTb5l#aR3HGf2RdPG*jTm-%{t&r*x;l^=lJ_fjI!IF4IuX&kv29SpnJhM9iroK)>EdBz6n5-8hy4e{T!>E? zn{%r-LJYI(luEQMpfP}?vIjzHD@Byz@9qn<90e>+yR(1Ku2dA}mze+Jx#_|7NH%hN z#UyJZKcduSxbF)DX(D?%J_0kP4lrn&hY*D!>Njymd9I3)SVmO_oJq+<9I&zjCYggB z#kGP|^S-8AlUNIk!d=$}8XPYIvKRsU$h2Gpm}gsj8gNB2o^s@qGa-@CBd zQlYgZ964S(K0WT*^S-@Z^d!R0G2bs^Tpyc;MMUS_3elm6OC)!Bz%a)74XJS7u6h>X zmSTi9M{}JauvkE4iO3`Gp20*}!)LHGQ?#+sfCU9hbvQ+7Us*gR!S_R3@209>MV;3G ziyKfP6TvPTx(awi0t*6IX|Xym(pL`b4a*T{lPlud_TZ~4@&Hoe-eRFjfZ3cT`X4e@ zgB`i9&w|rT6ns?^(+?<5obQ)i1xZoQ;xF-NJV|CGB6}IBM$AO_Y3nRY4c_V$B$F7}~km7$W(3Nyn*##j`u>;ZRKm9IjBC3#{;;Yg8X-=bS_sK7< z$$DQQI=w0eHR#a7Q%S_{fvcs8^5m{qYdO`ifd{(ee!JZ37fPj0n*C<+){F`Y!X{zmCg3MNR|BNa6 z;ky5p(e$9+Nf0%YB0sItc8(a|oqU{hnZ6&QcqK4*Y_N9z)|+=%K$gfSc7EFEC7AL( zFNZ3sA!yv4qs70f#!PW3yMSt%F~2GxFkq>VG#Qof1&#;}6oe-G``S%Kw)Laew%vma zl@z6f_B>&Zy>p$2wK zoTK#sv{cXnhKwiwtOL3yf)crcM59*s+9{VfVWK#9PIBT7RWppugj-g@Vd2gK^ETE8 zT)#)66X%`wOKMDHD$a4DB3hV$-2_Z^lvffPX zTe$P6`_A<2A}CiL$zJK5HZAU+E3mW&0`X%Z{ADTbnxBr1NdQO80Oy2@yC3_{n`|=X zQq`{8xu^Z3@BOgJpj=GR$7P;k1qM?&YehL}VuPa7?JJTeg`2(P2-W~L!YGiiDUEpB z1}20DH>@B}s6!C2AnA!BRV|(IP9&C@l)}|R(O_ITlJJdCYmLOF9Z!S`{9Ug%$l7l* zmiXCabtbJ#)d&z`S7efgy)^-b10EpHKe);?sJ$-nYdMK%yJ}*!l`B;Hk?cy#FLk@tpn{l^N$)c z+o0nnTz-GUF%@x!nsn$em*NS#0h_u}^Yv{4Q+>Stx=p*<0_MuS30A*DAnfd0sqPC7 z0Cf?Z4AVFk7ftoLBA_T5?)GHA_BaH~7(I~1m|6kuqBi-(E>GSWWJFEeGQzstrA?(l z(^mrlg=DLr=KcOiK&cKyBp>i2F9HRe{q=Qeg^A${hJ4~Vvnbk=!J`M)Ur(F6SsLaN z#_`xGWAbA#=RTPoEgdKkd=V7xJCK0+mN_GRs>oKOl`m22qSbFidEnSVv@q(xC!1UO zSdA-9bOBVLRVU{SN^S>D6RMl|OBkr%coMhevFlveHrG%mJHB~lKSPBu~T{-tRa+ULJ zBQKX7Hy?Wk|4uL4$~7&TH}`v3F-){vXg2QEtIZpa8lWN_1G!hD$9p}}RtAf_E#l0~ zQA5EW4;bA84a1qrPxPGC2IXb8Vigr&`227aIA$UW(ZIi39esyz3P?Cdl73{8ZVE=p zBaDo8?olxfA^BeAn+2RPdFlk@QE zX&bR$5^(C_X)tr%rt<)YY{jLT!x zS|Po9UnQSiHBuI>zkEC0g7(gSPh_tn$t7TE11SgQOgKoZT+%AWpg)nZE@sg$#|+KqXcQL z@;Vzk50(v{6a5ElLUN(G_#yr?Vm~*8cVZ9jWWTGfQ0({mHvb@Lg=h!_!Bl4=7T!=8 zjrWw)l)xJBrfHv4Y2fwiiy*9MB64Q*jXTvRMZGUt)#agV-h-VkN3v&)IuX35$Oy#E zLU+8;N%AkVNb*>RA^{YKn2ig3vZZ!#%jfZA%J=h+mcMPqSFouea0H-H0XV(qryzC- zD)i!@aav1$NRn{Xnc_2#()2$Aku3ILmUkT79n)R#KeOa9?br&*XzpHaK<7}6OYmqw zU0PSTU_3)u6hLi9w2SPuT2NGwXG__lnzkxcl|e_3E9tb~SS@BUMSEuP_0*i|uEs3! zRD@6>rr>j@st4D*Vy5SNzI#VzLUxj1%Lr{^{+*aUd5~|q7h7g$H@nHA#<%t@UmSt` zZRhpe_bhfJtMWdJ%Z;}q4h-D0T%?e~A}6;pSis@gtlx%og?bOMtp*_Fxe5a`jjGrs|Nh@cPp*cT*xeG^=k=@nuXQ7e+xi#!n>bcM4Ffp&41CfNT3T z@b5<*vg#;R=@IF>P zxj63R2Cetfq~gq0IGSL8|H{4* zON9d|?DQ_n_*!$=0CDrUfo==TiKHiZ@S&NJ^xv^)B6Uh1T@P8rp`ZyHuj3zhSM;W` zrS7}m-#ahqOzx+d=%BBBKu*+(v&H%@128d8gQ~13qv6q4oQ3&@r?qtT&<{n`#^lXP z`ihK*K05Y?6jAGFc0j(#vH(cV0TDR4iG*}C8gbBV1vttV4v-<~+|;y0WJ}^d;2i`s z1Pw`}O%4SZSdr5CgbnQLaGW;3xXSMv4W-=9%nvC9JC)NHl^FXUD~a&;D63bF%tRr^ zHY7i=@@B|gP$^R!JXVmNg(&l5Mr$!hkoqi|fjy<-PR_ktvim7Mx$wD-b}^{rZ1@L9 zo{g01AdH)#_n{~Yh)E%e9=sKixt>lS?-q8Oi-0^G+eG^_iU>gFS{(0S=BYQI|M2+e zsT#02*gw^I)TNEh(~22`sVEwOBSRA9vea?zgYm=fCHCHWu=yGbz5+e6EeVjUoJzl> z^W5wXzTUCb;&%)TkKr1#;OefZ_EEqMUe6^$9-1y~zWi;~KW)|nJ5-Oc_X>7`n*Okd zvFr;+EcGTKnms_7O_QoBpvYD~^wyGnAVEjb;M=$p{6%4<5Ant3I4c&G`|)=-i$=Zp z9S8+hGnz=ARDRZ8$cfH_&+?cz_kz%hfdH+pGZ*n`uDCkxL;y)1u9x8l195284-dt@ z7sYr68p@=9Y!>vgptj=j1AIef!Ea$~JoJxFQ+*uv<}ER>2luGXN#Rmr zRukAG6kgPP-t$q|-=pe0IQFc2G^AYwObMf>#!DvwPDIz%?sfln>HFgGH#+#!aQTa4 z1C4?oLxw{@8DnvQ5_p#(+M(=z4%DI7flm^&>f!P-=SwC%URaRsWW9X^UnZ#MM zLfi*3hFa=s_|*KEQI~K*-vBPV z3@H4r-uqE9aUA^yB>w1qNG!O|oxyM(>x{O*-vLG<7D1zgRE?TNs}oJ;xfJ{F__Y@! z@}@P1Y&lMW;v0G0rT8y6yQ*+v!n5OLwwLgCCzrYL=L6;xcPL47wI}Qw2@o7%l7<99 zU)%`Jw84mt2<-!$d)ITu8zh56v7%|K?Kh*C;YZT0^bJ{6*o1xIy%`PY+3g^+r;m1P z!}bHHh+rn_<H#J+#3`#)v&1YBH`Af|UnG>;x0RmDHbG20aO4M&x9r*L06;Y4WvCc$~z) zQJ$1Iw@ay)R19&O`jdmFa0W8|mP!nCfy9;JF{Ygj>W7px*-U1oKMKUr|`*&%o zq&d?=%Ea!i&g@fC(3xARGUawgTbxql)KT`VSQCS?M};ViB3})#Kye3z5QE@g7!C!Xeie5;8=CN&qwu6`c5QTa} ztH@WmB?Y~|-(mBjrFqMk1KfbO%@8YG=I^`a&2RQ@fWN`zF(E^R)F<7Xj3IwP6mPCY zA!2XqmNI!mtzgy#(_fn|O zj}^(k%0#Rzl)|ZFxrsqRJXVLwK*(Cm95)bU`0seJwaNppD*nxhz_}`Q?w8uo^aB2d za``n(8d`1>v)QY75=E!#4|Gq=#$XPuN*eOE=_;{AvxpV~UP5Q_%^uA_G2J!XhR*+O zRfjY&CH>jCO3TsLnv?v}v(391As6xFQP-}!Z8NyV)sx)g$$(z*d+>Kk1x_w^x~*@H z9oPB11nZt;Qj~-e_*|wkcqW%%f>d#S@MyLWE#`xd`_Eo8prS^HLG5N`Vu_)pXQ%oa z=I-O_eXAz-X#eCWTktLgks~AJzH#o?uCUuCU}YVto?GdB zm7JyY!&5bV_!y(=WQulV`}gtfuT)eF?^%ZmoOT{vgCjKlzwLgHdRVumdHNKmkXbJ5 z7El}-i2Oo)%%_XhbG_Y?&VL3st__&SqY`o$hCk3yz%tJ`Jfs4n6F2sFh_|%@=^#GA z{R=kkCf@o^e^C)|CPL9$vRYqK03T|ujc+*x$5afc0nev^XxH4@@OF3y2Lc;Rs6mPA zxfMrXIOv%1n`O57ZF5#PdznV7@GM(&vhi6vjinn7rpD!*PII#iEO{B&J?&??%|gd) zVZyIwF)ayjyj|Y1pc*f{AeRJ=LWwGOCbwhJsgbNL8m5fGq4mM+fc9wtV7E8%JL?<} zIWa@TXXPO5bwqKCC(acX$jq;Lx`D8EqY%3{cZVFoKN&G1kS&2Vi3l04JH>;zq{8Jq z3$GWPc)Mu5?%L^UXB})|7UP2JY0wDrPnb%@dPs({EVw@{ zB(Ov4G%|8K7&nIH@-t`oaBETaRhh9PFVmo8PYP@s zon%~akbkxGpyv@)|Bd8N*GqgPD^9RcK+Z3bP(TOwA&=xusK&}++WWOL#_rrGQ)FSL z<_5-iaNd8ay`CIP8GB(ilsH|`bk;m67Y&qArQ6jD0$w&~{iXhec{8j3E?9%b|$ zD`F^$Q$UZ~+fsp3S&`BW{&L>)e=aCj;cs58|LNci{1g9;o{fxvtc0kDk}|!y&3{!? zPE1NoQ^`oEP|DI#(aqA7DnmLrJKZ@i(NNPX+&Vu#I61<>&rM7!-QGF5Ji)lAh*yP& zP#uj2p3BJ6(#uXtQAsY!P1Df<6B~9J(Z;8}z5hy+C^J412?GNw6H#9>1DdMprPqn` zSK$eO0sJSh^MZS6{#ETQO-KL$7ZU(r`1dPic_}eE30YCO|3KyTPs8p1p%c}muI0JW zisb!L(@XxjZszgHuOQ#OypeGC6w%!5RNZfEF*ZNUfI)QRCy6Zh*L9=^+)L23B< zX6l}FuN|{)qQ6>sTvESTIX!C;`*~W0I2bD?*TvYH{r*dQU8VaLIO)~Mx9+oVCNtQh z!#-NqW3V~`^L={BK)?Jan0SvFpGivYTY-V%kLQgsp<;kFpX`nd1E0)`Y!FoAWF^Zd zYCcQoa&vZbb2JNnrSOQ6YM0OZSB;6x!_Y&j^yt;REyzgK@yk3c$Qc!$cb_YRR6!2| zt6ad698@+4+;X>Q@`d>7ma@${yGa;ykSv5%H|plGCTtU|>g9;$m0HZHdDHZ#waLw_C%`0MeoM-u3y&QR zu6s>0Wjd`!oeBr6k80oAoA}La5CvM*Zfm_Alb4qYoV~cUK zy+h`D%m6X>b$(TH@Oh+gXYo72hWM^V3GIh#31K4QAS31P^G%WZg5dsZLz7+$af384 z9VzjL7Y(zqyw&bpe(!Mua9x#3IY7{HzNt!OS7Hf)Bgb^!w!U7D3HaKLCSy=8qo?cr zCXF$%iuaDte>0p*_{8wxXZt;PA=#++)xp4na+EXz2|Y#ANIRl7vXx_#7_JXj9wF=? zqpGtY@OE|henz7TSJfHzT%V;4lTm%>Kz*>%2}~AFdotjbHX$w4I6QF4`AoQ|3{pJa zSn{6@?3;ryM>(eyl*4Zx*!F4q7(8vyB~*)SQ~Xm1JDd95~23qWY^U61{5J z_K64t*blYXDSs(dRH!02Z+9kVWy@u;c5_|}T|8CeVBLd@rSmFvm{*MoeidbJDr^lXRM)rs zc(Nor&dL;O6gsJb%2sp`+SB?x;63kwIT3skQmr9l=poxGQK|2JYt(sVx!1aksH8MH zF}oY~Pej&*dHX$p&=|%KIU`_^S_D}Tvnp_VY+9b@GuY3NKLv!VIN9uSUh80;>yS9y zI@xrR+JyNLQcD)L^CIiT)o*3#-LZ+ckQF=r&rEUscd=%-H3TYSzsHE#rQe#0D-~-K z`lw)T>ZFT&MiWgB?q3zBzjpfXXy{buj<19sF-GEd7C2w+MyT0wgPu1E6$gQM0 zP7cYmp52Gc5F)WF7i)(5s3nKJdJOSAm3bI^=p!DHgwIStZWYTze5h@MgIIfztWZ80 z;>17H-N~=UYKTLhcLYRcz%4)v_E1HnLk4`Qg zBB++M%%$PyEpcglfA)QwRNZlj2YLyO5`(o*u^Q&Ti}+oe7yEf3lmw)bk4k})rnHn- z%>6~2>fA)}V4L`E@ijTw4Fyu?}urm>_@UH^=7rD>M%5pO5EyFzYQ@qaHtd zKSzRQE5~H>{8$!*P$u5?N3|SXMk&_|6NtLP+Qdpt;PbcRZsM90jbAYLQ7OWVtL&-+ z{*E6minG{o@RlLZd2~IuKJ;$Q4IV)s^yFk_%9xr8xz)8j8-`0;o=btJ1jLopF0&0Y z>9W+ui4p!4zYO_fNf(xGrnxv@-8UqPk^V*P)SkPX43bpsD|LN2<~B^;Kyf+c#-0X; zEt5kYx)-x@x$t^!!PAwUIkh^3CREOqr7cHa{Q8s}Rg$yTG4|-x3raX-vQM;d2m@Q3 zCYRs&%xtoP8P4<`^-UBjB~5JaU4k;#Du{w1)GS+=!Te|3!fA*(&X|ZX!^XwzLl)Jt zNNAz$7&jLbUS8?tydwA+`EIbNz6jN0P- zR#%m`Ey4j6D3ir?M(G+tN+>R_s_ZIk;P>JYX&%_0rK7GT`25GKSF(9Qdw39HkTPlld;=>CM|bCPR>XU4>o*;O}>NbE

pZGt^C!6XzRJNKaqu=jWRV7EP}zlW10J zv<}~{jyC4@JQ_Hhg`1$2w109iZZT|Lo1JV%5T(JQr9VN79>-TC6SQnx2^dg z@p{}d``m?;EUIf94Z4=~*J!D3Y$fk#X^xG~5l#qMQ@Vw3C1!%(OxO0*Qg z`Q|NTox>E^8V}=B3)Pq7h}kg$?D7H!x|#Jn)|6$Fr5}bj?_EY} zcRj0}t2y21<3R{N-7}2FB2F?cmgH`1$8kPNFy5^1o)0dLm*$=~PMo_(-*QSczj#<& zFCJDR)c#7_klB6;%#2as6kAr~N#rW@o|p7JBsb1YOKDEGj8YZQz)^&ZJ~lgr5k3)p zp(sp{UcIHbO%pHM?@2znx#7LkoLE&Ys!Y0OvPXX;YRF#xXgvwKC2jkT4K0G-7;DhX zzVzGC!J1l=cOtGOC@x%fr?L0x~oW|d8$#G#r; z(ukMHusGfD@N~5dc18B7m}29RCEHj&%^IX8;O3oHH!`$Mw!Q*ZwM^q6M?EW=CoV{m zKc&|7Th9a2Id+TvWw%xP@YJ#KzT<$-+*M#ngl1}}%O(;kdQoXAx{i6EHQP(vk5cqW z6DbZ+))y(e@^+5VD_c~0@rC@8BYa_JHHVLLD6#-uk_f$xt9%lGwy0H*(Rm9cFb~d` zG+GwfaBwh+0QuPAQpGDAfHFXNi%!?J_$No{vUO*J`u4B%f})pjvQSJ!GKA;-Fo(bH z)!p~nw*KO=k`tCLJfExk=JWlRGpFKQ_*^=6tdyF#$84M410gqa_Xdj|FeElVzeki` zQG2t#wm|iVc78E=$=E;8`O)wALI>1--4uPzmi3hj2>DPJXC*~kzjhm;dc)U-GlIDXSd_<|x+1KLL7L=O@Idm2_PS`{`%2 zBaeD`w=ZPJVqQHc8sxJB{i(q`&Z5ER=nCr$QNbADV8NOw>>xFC zRq30-go%9g-!iVhPqmt_iU-Wz4jvuqIlx_#qnn!Q?4X~Z1}wDeq*O(1X(=#rs<*Rd zya?i!*$Hz5FafKKrM6B5%to}RBtjv%hQ(!C+$)h(2G*RF0Rp)&OK|1-rZ~b+XD4FY zLX*GW-dv7C;QzP^uU9|^5yXM4ZCC%^5Oxk+0;#t6$SND=Jphra2h5jX(%jwT#a6&rO^#L=U@pacqhQZ(-#LepvZ|Z7tPkVa?6dDg}S6I(qDL z@SG>(^L{xV3TcH~w@x>@(iHAudvEU`4e04M@k#}!vx4TKAbpI20eKub;xNhS+h$A- z&?D1_w+C)nvDZPO2X{1BxUL?4Jsw)I|Hc^@r+k;s+h6^5z_dWR$M78BFB%N)g^vrc zN;AcIw26FF7I;{+YysjRh7G_IdFeZ&^H2G}5wtH5OZX*o5_39b#FrJ5AER_Q7((;T z+18mkSzT=E$kjty_;~hc;lVf1)54nOou5xbN35!HeR{+_0rFB2GEU$g1B z@gtTab%c#letxsqMxgsOC{wCh1ltlGdmk5HWCfG^wVB2Fj`5#I>c3Qh%Sypcl7FNQ z@_&Ny-&Y0xr$5!u#Pq+C1tMf+g2COyz;U=ou$TnwpEAi?iT zc8|IM{R~vveJyl~Uo^}>g#M9L{_>1>yJ4)R0+XGjk^{aY*?Jj-J!Gudkguc+6mPxg@~!nfAhB@hc|CJpwSNav&uyl_joRvuixxqfM7O8cr~e-purBr{Qc zQvyRxXxm`P`Ju-!`1QxysuN@BN&(&6gPL!R5+dwBr5cY>@7PfR!U>GpV3S3*^%C=7 zm$e80mHSLM`1Gq@Oh&))=#L|qdS5wvJY2u`!o7JqJ9&6I(sO86-^jSResgE*PR-2h zUej^&d<^=e<>1An0oZ}C?_wnj2|{6GgZjmM=mO0UYb8+yG^jiE(A;XP-TvG*>0KP^ zjAAAaC=wMA4n$4*@(|<(3ZNeHlu%@e{0iukBu8@GQi;P*Y^6tz9nCCa4s~?H_T+TV z%^^=~8*e@$vqP{q4G^dW7H5hlvabV(<{T5I zujj$RhsFGaY~byrHPDR19$E}ViXx0(a-@*l=dPnGJ8pBmxMx(3oy_8WS6qj5qvVsy z!dfY+@hXr|3Tzifi7Nk12)x93R|10>;iN~8E6?GaL?My7Cod?^%S^e*;}I6Gc_@(j zD>o6Ta#jOf;Iv~ukF?ZlHj5|-%jrHgATt-V5gdvNMZ=&uCYt}H4|^2Fi_c`lWLIR= z5ph$Twsf3MuTQ$)b9&A6U zPDl4B4_0BR3ZpwGZfu#fB2s{I7A}6zZ+oL;1hL2fZxu?<<0@RL)|h(vs%T1ncQ#KQ z8#$^$ZndRl5q|sRLH})M=e|y~S_ifr82uwWN*8xFF3OhfA%Mmpn)kS^S<*_m3^Z^% zJMJQ*v-!KEc2T-Z-JY!H-#}s=h&oD+_lZk!MAKiGgw{OcRpn5IVR>G2!&3X=ICX{( ziB%xx_}bdpJy|)^Gd|Ua4!M{vnQof8R$myatfzZ(nuCJ(+PN%(d`!RZ!XZR8Q67Mj_j&KZKry>|L_?Pw!z`Q%V(I#Zq3~DksGK-V zfM|Rnt}{dWa{6H#SdgkM@_k$2=}L{F(mJ8{#Y(8R)8}fVp8>I9$e#j+Z-lZfqblke z?9O9{vTP{9QGN>hFub*I_49Qjde_E6%`3I+-o3UDl#ASEK5%iYJ*1ArJ`|t)ChP_{ zY@(D*m(%5|h3@mLNmEcWt3k>Z(GCLTn1czdhxnq2>t{w*R6Nmp#Vu2|32p{~D4?cq z9k=|kpMipkF;ZQo#lI(`UJY!ehMWeoZMFm?i#4;_QvookF61 z(Zz@*b&2SHI9mLN7rn|(~9V>oJ3OgEo#-zc+`ZV zT`B?!aY}1X;NQ8!$fPKLZdT$SQw~A8Bb*yw)wjLUn=|@-!j1uT%~wNl=LTZX4~Vk@ zPJZUNaUDIoAa}0Ro-WuQ7(ys7!UNF?5E#FX1K+xl(&EmLSLQ(&90#G)>-K`U^^v(! zSn5&#PO|7Vibf%RTQoJj!ocV?=P`ZakNak^;XtEc20-buo*FNW62b;G9cAE;MPw6q z(r7&Jq#%1ob+OD8>qvu*?r_{YdBXqb>8dwfanW7g`s9oSeSABMtpI{u)J`fI<4 z7z|@C##ot8v7WkNadvRnh|oiHwqWJFM>`6fZ@f0RFh%d@WUF-mZV>QxW9&YPk#YG0 zcS~pku}BS(7svQ=@7fNykhrbM&ND}Ad(dnmXjI5$DL#$eZ3HGvqEl{5Ay*R7{M+&3 zE;NL;N6U}aN)PPvKfDGghKXZLB=IXEHP&X~swV=<*D+35?gV7e5Mv{vYGC@nZlaOS zH(~C|;AsNUympgt8;- zCUL~6)}QK+N=4mvvhdw^BAc{=&HJnr4!A6}lKwBKW>(nWJi(brz81PLA`K^`5 zZ4|-6aHs&SmobZvzUlX21y|#11$kAi;m$Ufj>BMXp)IEpxy|=#aic5V-HVf*TScGo z7M@|r>dg=_c`^E7kXP#U`Jw2klr=qU-uV(?)4Fh#kHuHc7#Z z1^|%`yR87I)%tqO;=8CmjYB~4iRLnBHM_I}oCZgO!DY3cv${3;t_iUwIbWdOWN}mZ zId_1#B`h=8w+1(}2dtmNbrfuqsR2mha`~4Ju%3LOxQ>c1NrEwDe!Vs_V1m(6slR4v zWCu6}81=i#FUmO%mN~#UmI3BQ_D~rei1%m#dJlDCOdwn4@9N*HuJylPnw@S9_qvx4 zJ=NViFE3pjwmLUA5d_^Yx;`)GQ};z~T`b>!;i-XmyuLqe0@Rwca~J;@5iIGASr2gW z`^}S10ITbEH7iS&9<)orf9$qTeZmk#U)*`K)v`>#!Dhvv9-QwIgy<($NnbSSY#f0U zSS@PVlOPgWUF#?8ZiK`UATZ1voBk&7eazcbf|F^oBfN43M>TmCRWXiPY9#T-%Dqzt zqb?u>hV8n^K(yjFOyEY6<9yb{D#bE6aXfNr(rm`is4?h-y^o>XtWrx~SdT6VGbKr=n*e1r zReF0mNfduuOEZ2c?h1xF3R5Jfh=$6qP$Lfk!W!O7&j`uL1Ci{@M#-G%e6!`424p;q zObhX34jEgFF$K_*Jamb?h%0X?<&s<_D{S+A*TN(W8zcl zD-API!1&GyEr!mH9n+;RED_M=krx8x*`!l|b;?_e&3Jt~6$c7_)3mVxd$DKT_ig2Z zyo>cmenF6mE%PVGjUXenHZO$N2%Wyhn-M_o^T6Za}Ax zzk|X=nFZJdwmb3M4e{o6qJDE%OLm<*_@mEptP$UWquxLsXET`1YVA?Pb12wElnS>z zcVa^$A;&)q?TlQaD>fIQ;;<+aNCN2QBIhna1bPs>oS%u%)wd5EEL)FH&*8LAzcLp8 z2DBX6oPXghjG;l_6F!Oqa(@Bmk5)DF(rIz!$XO4g#HN7j@KW~klDcjq; z|Ei{99)ELg-h${f`U90aEyiYjJN1OfqyTN%=`MFtp(I|MpUqV*|6)HiZgh5+iEMOa zpivo6nn*RQa$f=@uz$3K%nDY3KulYp@T2;G{6k26Vj*~T)vl0OqV#7gZV5t2`g%1j zf4dbC_0wrnYu1PLS|!XvrKx&eH%tE^n99s0DU z0*SSaqrT*I{D|So?QTCrmaX}REbsVxh*2`v3z*C$+x9EBhk;y{e3FK%s!c!E=`oj= z)Qj&sJ%iGQz$@uckp?V$V(g~Ov4q#PGB~V6(lCAhO^eK(eGTJCI0V>vC?4xkEY zueG2{YRwUH1eFtHI60LPzfye1TF9xyUw{0-3ZzdZi4L(4!!DfP#@|zd$Jp`0;vTdj z^#=e^Hq=2CXJRR~mW~m1Xy0A_r|4z?7VOGqF0!J2DMM7sDO1T}j1-FE<(KQjQ;Z6e zB{FprtcnDq$6EdHburqfeIs`-d8^6l@jnT^9n?+qQyA(6r2R@NIT_(;TND2OL|%{q#euQhW0>1R@pRYyi0(F4_CbBdh2Bcs?i+!RBi&cr+5 z%o7(y;ZgEzTsF+OTjf>A-wy22jw;)S=?W~5fyP37qqPXS+uzxsby@yvD!Ex&maK|J zHjOlm$>nW4(x5Cn@XGY?JhztWNI0vLJIa;w{TqDog-5k6-|&BruBWi` zO)?_Z;-eeqoxy5o^jz4?TzAERWCQO5pxAuy`PXvST*tX+z|{4r&@9&zCo2;4r+hP` z+XP^h^a&2?xJsH70NvbmHv3c{Js53h1%I7A4Fz%O+oRv7gb z^9L$swF*pf)j?JU7uZdyb_Rzj^9DPXCXYLzA7odtTNJXfDRIhf;D>h%ufvt?6oyL=pR4z7Ni8ub!*|y)(f9DO#J6c(P2P3Rvo&I}H+5DdR;+PF zE$|Brzl{sI!0$=@`*q;sWp%o<8*6L#`b`AY+A@r$eP8-*fBl64%h}gCOC4iQ(&UlM zc1B**yoaW3zKr(8J5edN&*WJv1BUoeh#thfDy5>?fOZ+PX;E{d-Cj+s?IzMf7zVy0 z|9AVIiU2MSqTy(t<2?G(jk1V9IV#9U-48T~K;c*afPk-wN}v@OaQET%AGial-cW0W zJer$R!n*|YK%zp2sP|p_stUpDIhMO9o8DB-4tf%$8DtH9%|&>7wiDODvEvdYGTlN# z(B$#dznNWjI^CE*U_V|Q-?pzWQA?@91k|Sp$AL9%*0N(2AiubU&IcA29%3j)-YNZI zVs3kbmsqp=&RIA;*v{(Mrc9_Anh1h6sKN8{8^;+6Y0&)J5o}h@+l${Q&WH0`PsgLE zfy{SXuaYFdcVn3aWzkP1h0%Fx2~0nxb*nNIt70KAax9k8-f5NO>K4^P(~5{_P&8%x zES4)Jmnh1+lxTF!wkIui1R^aVCvLP&xV+flh#Nt~z^&kPJ_af@J_OhrWA`TKbFeD= zq)??E+mtG+eIf5d#z=_~?DZ((x8d29<#+Ft`FBarge z?v9@1-?^T*eC+)erSK(unzd?!3`pV!qhE#PAsErP>MjLBtL#o*}b>K zc#mL$I2+9uAgL7Q`f_kt^w96FuMq!mcKbz@l->N6^NJDWk!9yVQ%dFL(nPei%*3u7 zpBT9Wygx#02mx`)&90AOgl11s0s}p&nZ#&`Z}OUQkxKG33-$3%itekh$ zRI1CuYKI@+HFzc%2iG7QNcyy2mG34^VNF)%`8u)+6fK9 zpf-mBI|N!$xIpzg7W4CexfK=lkE*M02Sb$x0RTi`0RY_pzN-75;}xupogB;!9slg<~PJ!87(NI4cKu}N!$m2RK(v#J- zXxfNZ8|RJwonblbj-?T4f28-kln_a6BM;;K- z2j)qlq<|S$S3VkO;zF`2Gp6)eVM=}pkr$;5_QCAlCrZj#SrUBEGtjT2Vm0wnDW9w& z+8fAfMdsKy=twO&VbrJCMCUjbp`01m@-3Qb z>QxhFQbR=DRZ~-=l47znY0I8Yk1S!>qOH;jwGs=s70i&5F-2 zioi7H$K&-o?#u1r`D*Ix;PNSxx6`-<;l;IKWNm;q`+Lmh`hRftj=`A*f4g98+qP|I zV%vCP+cqY)ZEIrNe8LGPn%H)B{=0k5yL)Q)e0cAw{@zu0b=TFGet{<6`P+XKGDR{Q zW4)L-_n@U^NHK6nZJB;R7?%lLW{Qj5 zP@!UE7QSCy>yn{_D=cxyUJ`!!{7q7i%UMwOvi>#g?@Nc|1HIshiioUg3V1cHi)C6J zgyyoX3^v+lj}o{HrU$>vOQi8o&@JNzJj>+PM#meClSLDin>uX<5;keqc>kAGW-is~ z@1M)@z2L(|Mt4x&Imb8^K(P(_b{;+qdwKQnGH9`nkyiDzBF4dkRl=CxF{PX*;^0!@ zP4JemDAyXKK5~kLji>K?N72u8uMXWqGVRxbFzt5sGZYs|@iYNEcU-Mnogce(J`G*u zT_OB7QXBW87qf24&`xF3Rb8{!9dB=igx}m>;mLrJB#)#e3D##PJR9w}wSe z*yXxK*fOR1>eTx8oRvuGT3$>Axnr6zj!`bklMJ3s^$l_wjkdFECdfRcj>V7b=HkVL6qZeQN;j^3lQfUg5D$zu`olL1; z;ESs=SRqex0dObC;lD9s;@ArOR84msl#-C`HOXxg2YwR+@FqBtjk+b}1<+;DU)m(` zU31eEwyCBK0a*wKsZs_0tBnHF3d;6}E0 z?yfV}60ACbA)kDP#x6>n99xDG;0PjSk~}Cgim=q`EOQ`b8ou1h*RGq8!dqjMq~+Swua6_R6z0)vlJ1vexRWeH)?ua58(oPDpY>}pgXrG>kmH!A_5c(&h+T5@x>W?_V#-6mpMLy+6#w2kD4 z0Ga2>uzHM=rjpaKUh=5IJ*Mlc>H}cV+sZx!jhemGgC12Yng)z+Z|RDjw$+~r!;wAm zME7^MK_cQSA}aOhj1i+Yt1H~oFG_Dz7=w1p%xcu$3zm8WPS5o&M81y2zBmhiS4>!*fio;G9eI-;(Sn> zt(-g9jm6o`5^u<cJKi`lvY5_Ps zfburDHGptAvUO*#rHd}h$~8lUrlAMy@p%2`oJHJ5y^%s&8ghbXM)%oh)N!}M1Ig?Cwl_YG47%6zv~TI! zf60oV_CuY193+IR#<=nS zg?aWn_^%uaM!1L?eOBF%Y6^3`&&TbRNYg)@FlX30`z{_99uxFlEbg`%P4_o6J<>fq zb0n*5aP=Yd+BDI>0Z?&s3n7007bn*D%P(TKX2}A)bg7gaqwKW4YUSLJ7{0b?I0_H?z2XMMWhAhRG)Mp5ZEBk-kfv)xVSl0* z?IB6b$xp0Nsn(ZSDvOcCfC=9yS)qWo`IY7(AT9O4CPu4K)^oF+p;cT=0yE4a-9bOp zHtDK6Pa`ZMhJT7bjS&|4JG*x(Q3TL%GjMh*?cg*IM{l}m9HN3M>YyMYvAL?&wDA?a zq(r{%r2EEKiFXW}A3`GAT2XnyBk_X{=S__ku@c}R&vZg!zWtWBsDjK8-PlAV^y~Q} ze=db%qxRtMJm|tT+@a$7`C3g%K1vMvKCtM8o3sw#Nsn}3og&v&J(=ovYfy+NxTtHY zWT5;dL5W)Hwt-X`YUT2?$5mlgfYnzHI#RW|bkeC7N10$R9YZHfmuI{L87vDR6bo|2 zuXWXBvkJM^dX4L6rH1z?oS97K6Z@XZlr{!{*eTsEkU<7tGrRi*-6DOzBJdc664-wr z9D6@bsrV-aORf5iYG&<87h~h~XhZG^cuS@u#Ed1_Jk^c)f+}hA`CMP4>_%Y%_i!66 z#Rj|P_Of`pSeohHw4z+hbV63#^t1t2@u#$w<908G>atxwnaGF+t_3~g`y+4qe(0?M zKH&bdY|GQ>UG-Q>g^TAz{~^1}bfkzi-l19005}L6{yymWJ>Da#MNPnSY?oE^J`~sS zkTe_eTw=c!grBF`?Q#81li%s*dwRW7S=&v?qe#20LhG2BoBTOd0S}Cv@h4@^56^)3 zg8Y9UW_rEqUZ(#un*stN_}_u{4(|3&-ja3zOIIeV|J`v`Q@&C>yO}*W$ z937180WPjq0K0$B5U>B;R_iqNln!`s{I45c+N{c9WvA`Km9aHVf>tzRbbkO)#VTFU zD;^z8EbStH-Jd_N%gD)PwO`6z`n9|E=`~AjGmXmeYc(r{YgSFXjD|9K+cvaHX;o|6 zu;%D>S)|7?>iytIbe@NIl$a^ogH2=G%f843nh%)vAxFUPB>*3_iPVmPKbhSo!Di&WnJCHdXYW6f)8bTIqD{|Oo7fT+D{DW>kt`j}q8Vdt-Hz5O2Gy4XCykl1+(!U3-YR#@ zI2hy!NvIecE+`p?_UY>B?Ls%7Z_Evj$-f&X7MGVmmLm zNIYxRg{k~SNsd*gS*?K-YQ4qIg}S7=B@M1WN5v&}bg^jd%ERk`RMEe`P+agemFC{b z#k}@pen7JIvz{c?dTXvy6Hqn{@H(lK%l>=uf&*Ay)dsu^x(nqI|0VepXFzeHD`SjP3Cx1yi zkD8W)9=>k*+BwVFrmbaWZp0~t7?)uR70T=_VxT-Az$R17LhKF?4laT8%`-$!`8fJ1 zsiHFJirPG%nK>e%y&X`%0(~VLE{0;I=smMFvX&D`M^$$#sP5`M6W!@RUT#Sw*}eVr z?LgV-Y?_rRQ%SXDeB{S0i4#DvLa39@IH{hgrR7@5AIGg&?~w`Pc9E{8bmhP+n)1C1 z;>ZBBB2R_seW^#C$Cbt-ERnBChY_%}tXLXQLmu{w;ls|>(<&)(wBlL5_A|I_9(Q5lnH2L#yvLM7v`60gNGn}|b1xm#Ny+yg zP|l_{GWnl>Wg4?b2O?pG_wLYv2ZU-DzakeOE0s!yOxizh;ba6CUvDxcGLy~h4N4%Yzrdb;spO@Rj>X=l6Ve@Kl$isv$HRwc&~L4;?42P^{nb^ zLGH%VJ7 z$IxvdI8b^z`nxx}GCRgg@k%2~P&DXd@}Kp;K&?vw`+wauOMgD;S+2||S`jXJCVO9)@}!#5#?;p4WZK@{*l^*f zNTU&r+$d=DJb~|*?cE@vA*E)#?ggD0ZmnLI&f?xpVEC3Xl9nYhx%`AWbJm!)_>ON!7B7GFxIEZy)VQY-jxnfbr_br14x9vt4E6Kks4iS)6q%S6=j-Q~n0Gh%xq4or z{R2Im&sacBz}WT!vPi!N%%aFJK&NcU!bS%eG!XvE?M1dMM7mSnhs;CfRM00;HWIX9 z=nx!rB=I(4gQU~MiceDZS~E^S-U}WkzAA0`V5cUYr5ld$-0KAOQx-4YagtDdxj_BP9gY9}j~RPI)=*-jUEuF3TW6 z`TFL!tMZ+OPcl3TJViVj@m-824I>e*>Xmm2Tp-#Q!z{Y`Ud>x)KA84h3_I}FMnkEb zOXLSB8_P-TH_VU9{`h=&vfPvFjrjcZf6HZ1*&?%yc|gM8(HZ{d-gm*m?hJE$Wm5qA zO{^S^@f36zm+e6>G93&vVE_>p*V1$2ziyw=)NYGG#!PnFc=~W>!jV^;jcHXC#`u%Y zt<$!n7<##w`(nZ5ku7?)4D+yYQ9mUAXR^^28EU=Nk3>kAC--cB1*1<44`%red3Le1 zVQ>+bHW>71IZ+DZUepXS*A2u3F%Gc8$Xv8$s>nu?-5ZIVVLm0;b+h^#de$LRS~io~ zFnA{_C5e6JwTy#v;-E45m@zlFP0Yvf zYovInLR_UEc3uHUsfK!~5rtMtVP7A2W?olWn4fdS>`WRdE&W*5nj3t3nON<4&EvCgB^N&T{G4B_hMpGtp4}ni9;BBYP(lP*LD^;UiN9`(R-PdZG4;Oef z@T85N1kqt6t$3K-CQn=%v>3O2tDNiwlXgc}*@m;u0L8?lBsBuTX3Sr@*n0!NHU2Xc zZ*WPDR1b41{Y6uNE{s}@d9MisFOYg23#a44bTu^4SCJbe3I=5Nj{?ZJSaM{kwC8B2DtIOG1zvUGUvDPmqCgv>>itcnD~Y zL@ZC}RJ=vwj!T1!`~q7L+jf4E^8+7Q|89PZ<%ma!-eAH;n-LRpb|?R>>}JtJ3uSOE z{2fp`C>7i%7TPCzl=(XZ|F?4BwEP5_-0zoKT;kJBYOJ|TRcaVK?GS^)7If5qtTFbD zJK161q>ZpymHa2i5~hKtsxO@d0L%~>O=tpULv6aF6tGG%SHe1|YII>LW#Hw<;^5D|$tvNlUe(%=M0sQW5neX>36_s4DaDo#s z3t7U5tiUvX)C@lQQdu{v@KO;1(}O7$IzQwY;l!;_9aA+0Wk^#292ZqtW`va% zW+d>LQPu>^y(kKFULBrn@QyvygiQn?nxwPIJX=!1y2pmHrbbJz)Onc{QTj9H1&fUL z&?>sa)TI}rk-QOXfeq$TePIZmAm^5aj&0Dml^3JO=fF=|9fHJwP@Jkk&-(GKHMD07OwTSIw1y^Ba>@I1Z}{(vjoaCF>=-&9X%O2p6xB|U zh@;4;!vKbZA;N`U-lPdGEC!*)&$KN)4?8tq{w_Q&So=}=TH&CF$QJl{2Jpl&W&idJ z^ORRo`5~OUd#W?St@v!=52GBa{=iJC{_9fQE7^y@2|p83?5gU&iyQOd55*6PX8D6l z&*TdW=2V&h-NV1P;+JNnhcMq`bO@EH=nHlm@)^COmn=-#p%SIeG?e++?tAGQU1YLw zRb?MgwP%J;T8EgLl_j0~J*fO!ZSuE;(2G&rTDT)@QSw;fV}E`YW9~YVwQ-`>d5?Z= zrDqM+DdAGH8RW}c_H)-=I?W@TkkpK~*^%?CY{3tfsUqB1pcWk45N~c;lAHhO1AWfx z=58PsL$1*q%Kt2%07r)6U6oN%5HZuq_71> z7zl9~GL{i|&%d46i$TrKyUIHFb+FzYNU; zc1C`$wAN&&xD7aC8G+K-XN3PRC&5O8?1WEC?{6KUb<6q8{6*J?jSgASNoMXwgfl)c zB1#}%Gi>j$EZQ%+d2+@F0k2}Vb=UP!`n7Cd#H>I3jL+G39yD$>se{3Kd{5#OD_5c% z)U9QQnkjYalSo&({Y{O{jX>Y+QaaR4c0H(tA?K{cS7CK3uRab_>OkIxmq8uf`NrV% zaNaZS=s?81Jk=2H;##m6v-@R~6}@_X$4tZ+ zG@9{r1B|z#krY*;Bl9yFz8bR?SYYb-S*}VRKUn?GNGPQo9$I8NiRhxajAlhQfVQJ@ zcOqOhBZ2|7AEY^Yo9sJ&mMx2GYmgwo^97m8i%cfw6|Bul<2*VjWp*&{FFPg9Y+E(s z^jT`6O8C#bbqjJhCvic7u_hR?1%eAs(2|pWdcVV5pcr9Os8^vA0*#0=uzIJ?*6jWf zZ$pz`-ym+g*~TG%S)N|s{9uDU=6XOotvfBM(KCpU1v7<2APJ`G6ipnLN}}XcbpsJB z0G0PQ$JI0#hfm#&x)pzNXitq}sXH2M-1zowXiML@O=lnc9VIlf@wR5mh~L=0 za~U#`4sG0I^r32-Y82iCg1@;T7^`qDyIHfcbNTB|4|Gf}(#Wb;|M=RC(bJ;1dDU;_)FS!}kNfK_@2(|qHbW?gT7ThXl5GKR{Q^CeTOhgb>N z$XzDgewzMYysB;k0IYI;Zg5-m%1Cl)=JWXQ#~N>#nxqqBL#=aRKyDQZqUT_(8i zvL%uNTDz;+^Nn(P{Z8`2L$*r5J_c67bqx>)?VS!$@g$0_nG*^m2Gc5c!wDE%-yn#4 z+?j*rm+k8(XoE$S5zHe?L$jZ zz71ZQJbnmPT<|WQstd{ExGk5=ieB})Ub!||^p?gZI8J%Dt>)-bifvl!?=v}f#+!#Q zo`8xjMSe@c>%r=iF3PFUSd2~M3hF{b)a6DWg|jUobT6t3#`YhLIz+7!jtKUYR+l~{ znj#02tm>Nf)qafT6Z83lCZDEayXE)#@r=hoD*u6x}+S4yu zJAV7$i9kLJr3Wp&*zY^<20p*~r(T(=m=PA?te9v_Pdq*g&17Zrgy( zObs|NV?DOlvbJ3vppy-@!$rV>ma$MlxcR@P&436P5XWcaYh>hvmp)E$!b7=>#iV7G z#!n%<{9&KS=XoTf?dNs&5$Qu{`#SqCz)mr0rjFbEif$;p3iaHrqQ8Ir#=k0O{GMl6 z%!)Z%$^z!<8ueSV6Hd4UHa7Qv{zTP|c%A-#se6w)?sNZOmunTM%yVyc9$vhw}9o1$G+>h^Yh-Af~0P;?)Jes_xGBb$5za9rY!YNm+@UEI*066MFsh~fQ~6)I$CW8+@LqH+4~aHnZga# zl0Ou;;}6SJXU%{OSzTNRT>yh|jkQNH{5@3XY_6LclfSI8`y(67SYc%~yt*0gRjW}& zTUHeyOvO>K3)*1veVc)+*K~+wW?W7Nl=>)`3?XV@Tb-MTMJ40GR@NmzUj;?igl&u8A{S!c(VA(Y8Q{Q|Z4H)apb zpV-xx@f~q1XnwPlu_5APuc^3FB~wgG*o{`M*=+^4%Ii=q57A4tHw836X)LLd=?u76 zybwF{2juV>j~-Ak3L)Rubfl(h36geOBg^C;Gs*Qio^9UqOE2X{eqxleg{ySt^ zOh{yR%BDz;ZfVI((VAQ7tiKA1>#eosCG?I)(;l#r<~g0J46mf6Z^wj2QQ{Dq34f#b zc;8fyPqyN$V7W`Inis_+JsjkqzNS!3|9K76SHvy2XBpH(jajG=-{X|?|9w*^ih3mD z2%5s>Ed*h;SFyR6#h7qt>p{-v5@34 z-s}9<5Akp{3`y?4?i5)-yi8oE&kD4{BidZGE%(1 zOcp+~74lJx7QQudvHViJe8@(kqMxm%N`S1chOm0RhjWB?O@O<00;1sy`rjj%{uz4!W&k(9{~(zDf1Ev+{>7p*qcXLZlAYLO zVM~p?42ipqmlU%gNS`we#~0rqC%yUsuSFe{2bv?zZiQp8R%rW$tF^wZ^*B2fBlvR8oAN)8SVW3;4vOf}Qg z@J#B$-p(%57wf<24A?ouHl4T=7=Sb-_ z(ufi!$s&hs!CgA%nI?cZq9j*uXtoxbuUKfL&vF*Zz_J|tsa^2t74_tP1K!PeS%mk{b-X*jq0eUkBED>gHBUGdih zG@%kv!zz6i_*k6v7if{%6E+BP(Q)HvAfMB$pINN1Q5qwo1L{z|prLETcn5D_#`{hv zJfBnRLU+u`HcFGVqlmo${&*AF#qjVf;q4u5yfQz9QGNf6fZi9IAtDNX-|#wx(N))} z?5bgYiXc_+$(~H|Nb>sy_vy+9aQ((EWFP@Y&4*q;tiLpz8#+! zXqlp+TQ&`H?4E*u(vURXyy05;z6;6%6`O66aBIHW6V#x?nJNF%ElYuF7#cZ2V`biABH{-#S)``)$5*(jEKPGkgFqC}N0*3TRgBTZU0t^!~`g1uPPf?kGzZ783$mUDWkga;!Z8R`Zh~@Zu zO1_c7l4SH&Z%|(~wHl6GB^5IfSM$pgx4&+!F>C5 z;+zsV1*OO}1KBc=OI|){nrAgb*9Q4XBVC~8VTHwSl7XUfQN&(V;%9NBNP2ClHAW7d}Q!Is<#xjCHoH0fbx-46f@l*@1g0R z6_wI9wI|k?{N*|I9no^Qb+8j|ad^&f7>qTQWA6>2o6){&+ z(0b^qX+b8VZ>ezdAfo$Rf)VSHSd@FAoLmcC*l7vSbP)vCICppcFh|1olFbY7`TBF< zs6o2@?OKR?LvEL6i|n~zlS9z0fPsMLAZoO_^e(wgi7iwoi$1bLKGP@i3;{pLG*_HO zGm3;Sxf>Qt+7Br#I_e8{k7^NrHE{ikX@{CJ31Pl!#ugOrD&|Oe7ueFx| zP0au(?-^-2lk5mWHoH(gb;RP|u#7VdrqS4BoLcxozt+fVzXa!DjGkKKJ3*#=0f$vA zFZ6m+a*j`<3GO|b8GKkpb)9-^lh7op%(EOCY;PBfY<6yX0d(T27crMVmmg*+1i-C~ zfWj!mu?yo*Fm>Q$G9fa?zvk9j6vw0sGyKum809mXp=umK?8quo`zbXhW{c{Km;+5vua?nHM^P1iT_5ml3i6>7VVgtRKRu}8AkUlgNkj0!lFP?jI&(aA>n=$`0AVt6InyvMW z-iG{`Sp+<9hiWXMNMzMYePk+}jr-0n;;Lx-7eh&15fm%nJci7Xb2U`A#A_#c=51KM z%&Ni`(FsDbxUSKmt+ItjFxQz(+TqemVQafRfwA|JNC(sy#X|>o;R)Y7gAzYGMk4J< zDVNR zYNKjL(Ml4OJ(?AIx9!RJ#Df|{Skba27~CN?J4jAOxs=Kw#kX%aBAwh=@|Ov+c(6HL zeni?FNZ0*x1_US{=>ZFPj@XQU4nN+p7&4A>Eo5rdD+&Y?Ubu>z@E0gX#kl&UO|Z%2 zhzq-AkCF`buyZ-{9$oYwsM{?1(;uzQw*x@Tz}@-ZoMgY^k)Q+uIMfiCKKMgW?IhFj zRoHk&Z*FmVRw2Q`M$?7aajGAJ&Zj$%;o0*8=V;aA*`(<*@cAGZjlc&QLJVjuG0qmBU&ZQ`f-wt@W`?JL?p+1yfBjuRZl zVFO;u(eK!@88!Eu*!B;T*g!taAFooePuu#za40#{FZhB=V!nV~{%ySPV=NcY-HC^t z2i)f5;U-fuwgZ94yNaKewc=>ePmIxf*J+@ll9e_q#_i3y z8L;$e!t5;5($a(yt>JOXHvi)eV|TIT6MH69FYYoL<(v=J=;Mf|!x>5peG&a`US5yp-v_rzr3&wS?LANI z!W_-iZJl90h^>CVQZy(|y5TpA9O~%=YOgyi7-tqdv*YCxRVW|lsa2NZss%!L-aFJP z5Jk__nZx;EE1ouJ*{M-m!xlJvqwdIn8QgAm+ zGggPez>-sK4Z9x)nXqZ-T_z|Q)C%K3+%`gcS>&cY_V#^~GuppHe+6GEAih>heE(xIy1F`kH zD$bvff%(S%z?<|9PzbUB-F)C!2yix=p^m1#xN&3WaI$vE>YvUzkH|0xas^951a#fSft`~(W`;CLv49GFCAZU(im1PguYvUA$9!ZY3c{$ORw*`vS zn14Slg?n6&6||Ro)Q9pDB*p+-c6f;TIY)iyv&j#|p+xkR$;|B|I=>P)2B@orvP#0!{$^ zh}8~DOXj>oRudYCbUmq3;0clUV^-~ql*q4eQvhh8UsvwtB`kpNU5fpIzE*wHBIw0v zu6tv)-(6De7V)&V6~YB45I-rBO3~Vk4v>g1qE3Qft1jkeJXs&i4n=UtOeA#d0x31?eAiU#+A`7B%XG>Rz=6DURho zm~Hf59Ro|2{)}!|ZhiPx*GnrTEGT|C84MLJbO;a0Z31OO`6hKG22K_sB(ne)3xYA{-R_jSVh?;_#av$0jTIM!;8KKkM$=o7Y;wRw zMI%7M`pU=avKSc2uv?6BL0&Muk=lwZ+9$!saTujl#5BX*j#WMbx39{_A?DWukVMXC z{X{xJK6A$rcnCNH3!DA(LREItD0*GPVPwW`fuz4kyuz@D;q~%~@pT=CmXKTgipqt0 zuW+|6CW!Ib#O9x6XlWBzD^2}0sq&cp%Y$@qqUu0ngMbOZsXO!}oYUkPsm+_ZiB~x^ z6*z4q%2y;1H6vw~FkwoLxal>XHHO+RHBTa16P2yt#=&@2blg>iT`*yB0YUzni71cH zlxiGz189BXq&n#G-TK-8$NxB-9&)sSbu0xb!mXHT3SxZykbu<=(&u-@*W>(y<^*x@ zg_K;NdgO4pJIMnT9UELFrh~1@JC@)d)7^SoG38a#i2D85<74eyaWYIRWSdiXW-xo1pSMQH)p&%w56W+89K071a-*@+&k9-_ zJaV=^^rudDodo=;yN}H^DVwk~`hA}HYs=GZo5h=t4=KWT=D?T7tEwx0ehv8YsVfQL z-YO1blK8=euT*WZycVZj(eMWZJ(t&cpcC~85gtyCPa=yO`=}i-RElZ_^&$cFmBFbv zHB@)(!#uWbQ$5wDsHESI3fGY*$9*R&kOhj2tpa2TC+8nl1s66YHUgo7aZ&ZJ%obGA ziOx%ca*Bf9cPbjwy#Qs4PSdTd9JG~XixT!Yn3r}7(Rj!v#S#r+d;H{DM&`2$ItB%m z2TAr27j}HGpdMl1g9{~|V;GN9QxV{t253C?$;b0t(>pwLeMl=mS?W@e^1L^~B+$d} z_#;UTB`E%7!^%!(!XP0*3IP77MCo?9gZCIerv>Q5^&;3*VJk=vmf5Cn>6NVBm26`rc3j~&i%rf z58Xv+pkP$98<+PF&7qJf4aaRjQ)e~9m8p1|LQ=c%y%2$eU;%3ZRXm>XYRkU%v@Q*fkj`z1*Md^ zlk^xVB{&73nBr#1;MJu|1Y7im1!6tH33t3mO2W}3^i}UgU3C76ItVX_p_6M6Z&S8M zJL*(<)_cN#vCaSk!>D6nARS#efFJ?T5+OYX$RP#7d*2>3>OHSo^2fR-a1X;p5$9zp z%J7FI=uN|!ywJPhM~jnPU0(hxhWQkuD=M|}#awZNclB5sM!q-FioTQ@YG^{tyJero za`03t$Pc%84eTOh&M*9l=s5jV?|7am2wqs_89zaLLG^2th-|BR-|RGQLVm*6x*5|a z4gyNDI`iMX>wluyRW{G>She{S=c7j$=-9T(JZ0V0vDqZDRes{ryHk<$SP*jWua_YH z{k3XWMiOrKcc7(H?}LEqmVoJ5!#*G zW!70mJ*$YUPqE9VxVj|8Kyx=m$BLcW3dc&1%MiO5HArPR=~gp(Cdbt{3!X{{_eJzT7^dV~P-zw(&#_ zXjw%;x&d!#)qlcI`(HEp0F8PjHa6RsYyODF)MoBImsDS5YcGkthCex?0wc(BcJU{{4{e0At*HeM){bmrxZ_1PmhPzf57HB&p5U!+xepq!EikX>vc$gNQBL3 zdn4J80PP1Rh=fm*F|oezEj`|u|Bl@K;jO*sU#>;h+3}PIK3pgbmiigg)l4AqP#RIGeC94XaZ|8crW5eU~H!HMq8{5?tPr z)Oj+mg(w9an{_^jV<#ChedI0Y;XbCInu+X6Rzg>_MwxQtS!m);!Z^Ev6(J~q#54CF z?00&RSO30J^P;L%wRqE%Cy}pzVUT%rzt-6%fWfA_L97ZrS1m;I$_IThfhJ?v7PBxP-lI|7 zl6B~*GVDEuae&7m${;;?h4Ohe3zDZ;kraZXHuR_2_sYF;f5a>^*$VjBgpY-d_vi$A z4fnX9l~O)Q#L^`GU}2DLock(jK^#2V-xq?xY0J=iI3urOz?`GqW%>fx zLeA7^w>*V5uKu*Teo3Px=yoU?+YkweBUv05Cpsxp;wU`5R0drn353}33v2lJy}

Zd%*!Mqea-qFa2_;g~WP z{Bh9<+fih6wbF}Nd7?5Jyx6TZs(MkWCU{)q~e(Sa1vu#d+6*r z`{p0CF#D*7k}9Dy4cf9;WU~+f0&ZqRdFcR<`32dG?o*yWoQ|>@FB<3s3 z0ULw2YaEi+#BkGFN$Y0|W8odhR?bkN`WihHJ%#fM=s#Z%ySbr{k5|JC>G790m09}- zS#-2Py|ma$2#$_c2{}G4jWIa9P1=;!?W=dU)!kG@V!&*9TdyuHhC8f>!L*Efx@s$$ zP8lpL)jBJJ(fEp+JC`&y4kCYutouVAH!H1JrCc6ud8O!o#VbhGx^Ug(R@9P=NEffC zJL{XdY}r+cUlx1KXz}o@nJLg>wj<`akaU`o2ZCZM&YOgDYW!tp+UjE_+Q8VO^PYOb z|BG185_x>%%A+)=V2fcGAED8_0mhqMaqLCs8sB7IsJxF5q(G*!IT-;R=ZvR0RUMcY z-Qy{@y1+%jjqPijirA<{I85bpQ@LY7Y`z0TDmv8lzM*qU>r$Sn zz~ED#d^vVU-~r`q$q^yMfinPb zgNPi|VZf`Lsoga5Nk6`tLuhqHuZUo~&gNVqsIhC6{;ft}Uu|!|_T2yVcCu^xf~Cj( z49l7JC(dhX%V856X4V)Ja+Kgh-n;8yf8nZxa8`=S2NpO*%O)6paKD;Im zG5?#`p$JC}YJQ83X*$}dKeOHx`1Vg}WLAWZX9WXh&+5(!cH&^wZI!PrlA&?=Ude>8a zW2eCAj3Oc`;2`>4kIq)Icfmf8rB{ep;j=J)pz-1+lv6q*i%hrFI4B8s6m`)XgYxW; zQLx{M(g0{qU!Ta)_x?0DI!psFgKnh>k>>ciev?k#C0TZ>e*`SbOc; ziy&Qrz9>zlt>VqOBt^Fy_|2hx^92y zWFi8jps5`b3<;?lcKW;);U9YQD<*&0Y-i6I$a*gMawyXvoI?Zw{twR1u}RQ?$+Bg2 z*|u%lwq0Fzmu=g&ZQHhO+t$`j#D18K*qw;?N92Q>H_yo!+pA!+mE$}&1CeOBk)Kc4 zj`&j3>)HCjL~$zAG+p~TIArT_W9A2fz1aCARX+ZaK+z@>+#N9F!{mAc?qYJcyQ!;j zJccUicAD%^f`&9iZ1Y1~KpgR!4*;2(86R@j5Z;`o5qX?yQ=$x)%?Fum>q6uDJ0Okx zHWPX?@c9% zZs8miftT%&k)oz)1hUbHaui{~4!mY{3fz*y7IYEs)MO_AF}707d52NPvWYqU33}x> zy-lt-P7XbqW6w;@2iL(Z(~9L4ygh;nuFVA@an|6c-4`_!j%lVfj-7ZIci2420{0(l zhFsc>kJFD*y?!sskG6;JKZS2X3HsL zSd7N0z#&ffg>oya@71bff17|iq@f`x6u0$X5b(1;X5Vvt=o3>G-E2_xb(oPT^hF`{ zX?}_X>G^tgbxy9&6Spny=g0YR?XhKqT;tK%kRzjV)cgqVldil(HibuuU;gDqOVFwK z5VgN0IcO#8edH@l6&64OQWhK=i8t;Dkl2VvwydZYBm}B)Cvx%kHu`3ZC z68SZ&v))X#SUk`Onqpvv`w+AqtsB--P~+rbRc;BBvR{&MB{@xU^$NyfVUX&rn+kRo z1V5j_&SJ+w@xR2&bwxr*Qdf)uEnw(bZ3I5l!cBy?%Ltk$gQ*JP-1ZTR#jn*j#ZhvvHG@`NdC364ICaWUL`RS{+*SmL!oHfPKCKhe(+9k@XZ4xW`7eX&`jYE8KBD3W#%!DW@8s{K^yW8}30cMC(eDPc{vDuW4hu zG087@myYEIqiO52g~_#LYcbAh^7YaI!Da77Uq({#{#vEokJ_ck2Ds~@y#?!dhSVN8 zsGNPe?|0d;jKCdAID~bcKX9lAjGTC0IkE3wu)LIVcCK!#(LH4o5C+bFN*!R3qZ=Zy zh^+&AG+{`mCv=zGou-Q_xach^#2OEtuToGEc|p~D!% zmf|vgaL^p$}nNi|V>k!>w@LAryUq6njWzPwLE#pmm$1Urk7JPt=0GXi8 z#d9#O;Y9aH5l|EJCchV;C^PvliO+x#?c#Did~9~3^=|2aa50t4o(C4LHlJey z7z!K!MaS+c)!3+8Fo7n=R=CY{!Sy8|5_sI`0H6;93k2kbd`J^_-wa8|X2=PuH_!s$ zTP1K0dT>iLGY&$GlXV>`>hvF%%!klqfIjVlJ!hFFEaFk={4xi3`lg7U&$7~Tm9D2; znJLi^Xrz0d6jONi;z?1L|Z){ju9z~5|HEyDZk zgchc(4j#AfPf&!pcf*ep8pD!nA^yK<1%cZ>4x@*3w!>pgS8wTEdVuJ>D+7V#VA1xL zy6c8hhj#~sBbK3TVIEeh6Wb+b@HS}Y-ATTO9Q;O0pgr2u;}OYKZ%n@>rL1b!ar0k+ zvs;^bBHJ=}2+yFy3f5g}lxONScaicnU-bj$h>WKy&4WK2|AkfBK)Q}UD_`VIdR8E? zbuzYB0L@akx>IKYqyywtfAal@jt|C&CaNn^xQj_?ZgG` z5cyd*tPb$33gbYTUe1b$EXR{mTzbb6T`g|^XtBb|7?XAwVasDy8O~qAx()xl(4t9} zLZeP`3)&iSMM>GXH;ep6Yp%VFtwqaw*@>@3!=wBj6I?X12eeqYnHqNm>ien5+KRDA zzF9e|;DH9PBFJ)=iLf{1KxjlkTOz{p2v#SO_H$mz3Jawm#<_g{^Lwf2%SS7k8$iuH zMtJrN8Cz}?@I`!3LSlqx98CH550?zy&(q~&!pIM#51A?6S8+`kLAd$rJj&;p_&!4jP~@mYM+(o?EGOC<+cEM+1ykuWI9xY6WI&C zL!$;&LSe47-89;EK**GtsxA>2Yy}Oir}h5SS#p`1wH>J!I!8ezuwI@%4pPYt^Gxa) zZP(atsRvMm!5RGrhW5e_@5pR8K4Z_Gz6|E4KNei#0JP7`8+oSTaQ_l`rC)=|_Bw?= zELxHvixZs1DUvluJ_cp3i@qclW&1QWU}V<(8RNeoA}Lv86cG1E< zYydeHk3Hmt;k7^Gi2<*bks(XEYOP*jw=pTW3~~6zvPi9h1#-wvs7z}wV@{Rp z)m^6;((Yo|c(|cBdTz*=ZjXV5O~T3fBhuPq=SDP2?;l$wDx?^L&j3ldkoc}R+i%{Hj}D`wrZ}G6Pvb)t zzfI<4doBFglDUMWOsEZ#9NREtHpRM_;DhMvZ%Ib)lgSJ>yCM_v#C>>&3AV$*su&kw zpfN2)Sn!XZqqBG4^XvdDpbF(W0cy?B{@QwIV!@t}8*ubrf$5S~yA!#u)!*Q(jyV%) zE8f6Q5OI&tTPhD6eSfFkG8lvr)>*rvp(j)flm|qyNv>LyZKGz_s^-9%q{N5~FuL1X zBU&LFQjllMcdx3jG@oP{Iez*rl>f4feJMwzEU1I|s~Aej&{`5MuBnn`DpTxE4+osMU@7sq0?>=eTcYiRIw7d>M71=g@ia z$wsgH*IQ@p1NNP+jMkgXby>1$m+#a~rDOEO7;na7JP7Kbj9Ft&Nr6mP|CVN++ammW z3YGE?TbhtgPz}Ko$MrmNy%Gjsc@l9;)f{gD^b7#iscqs0FdZ+ll1`UB671V!{xG}N zs0|b7up89;QHK-&%EKP8t92XSWTms2ORlQMFLpfT7en8ksLb`~8W2yUfg~u-`=%bB zsB@aW_8O{_z9bHLQK|TdJfo!Gj+@WNcp}Lb%#|du)y4kZ4-<=l+v?(F5tHp;h_$Eoo$eYU?a@-DG^N- zcPBQgk%R%wWiVkEY4b#!?;V#%MMppzdSPf7boTMA84bc`jVwM{@6)wYgD^^N0qu>F zF93uBZG~~dSA*1u1btv8O4egD$Fa1M%&tZt9qTG>(C?T~#>k~>AK&<5bQ!Z(-@5U1 zchn98puwr?E3!Jmsun#zjXnYNpe1tsdnQ73)m#v{g}nRgm6Q+*XaF!yJgc!Ya=V{# zV8qYxUHc$UxedS2>j7^oP9;ZskJ-Yk;_bE+$-e7uv_Gtb5*5$v_Dt-Nqxfk8Ov0^R z;pra5c7flKs|1x>Pw9{$7nZv*fGIyio(n3#5!*VS9kAFt$uTwj?JTjh-Y^}o?NpE~ z!XcC(b*`jz^oVrRe5p#-YeOG6n|81#Fe2m%@EB5THH#(8!7!#H9g%edG=w6ro6|aG zj=r?Jk*K0KdJ636zP2aqn6#{x;MR9Cc`A>-$|^(e-pIQ}72x-B1+WXlp?p?cE1#m5 z&{-awTi~@Y~uxN zAqdx0A}hH3zq`5Ca)8KPshGjEeH;KaFs=L9ITe())S@T9<@9O?Y{OFwi;k07lr+x_ zKg!bM8tS|^v_74@qQ!p*Z`>BwXNnbz+RNI^o0{uudnND)nkAU_=C&Mo`eo-wJ&1@4 zk!k1^Cp{a+nbG=G>a;g*seya|zmTMx5tw&ix3S2(fxfxhn^c&ffbWfj8r&Z-zSeu_ z#hktlbg}$$ob7!Xyj&qGftVW6G*s_blkZ4X% zDyez!$oE>QDcAa?3luSQU`DiyH|~nlbZYy#v!=-IOItRq(fDDbK|UD6!YAe#P{Z28 zkAn*~>f_nhKZ~(xq@W+Up-hVwNRGPBaXPTvPUDE%|2A_R{DVLGwjrTnOS^=gF@pxZ z4r|O4lD7N0qG6AaMyn@(7sr!Qxi)}O5n0nvU9lYn12sLo?Do^)uc>AouN_tW*QuEu zLFaZ^(fOA^q!f1RWYw2{m^3e+XFSBMPI3yRqsOS=h#5w?xj;A2O*K(PitS0+-WaI{ z97~!Z+)PjLaIx%k4iUdeH4BYu*8-KgEhPgrO;JNdTqxkLWUe9{b}- z@BMYo|D3BU0O(;|X`Ff|ll~CYT0HB!fsMJnkub@ePb`;8lu&P0+5ob!$X^fNhZVo3 z(Fb;K-KS_V2vkgn9$fX;)71i!3t%74ssE681(@7zMrX|lxZtoBUv8iP8bX0>yFS`v zU&Fbs8~c%bio+wXVX)Y<`=~uNDCBlrvJUhB2~pHheC+OIh8K%HUpSAIgZDQayEMwT zRUUS@z~z(TmF(tp|56mGid^GbT|GR{R{B}HK(p2BPWNp3xxXFSGaw^H))rk~?x*>_9kOE#VAB z7U|8IwaBP8L7igJxaLk863(CBOX>sz4{1pwnf&f_mvBD*alKD+@imoDNJALScA_O^ zVT{CNJFRv_mu5$YT5@%F@&R<%eNqi!b-^+GWz8l9`JWKM5)T=bffnU}H6xn)R8XC& zx6~z-5Ed4^r8m}>ds!yo}RDHygrWsMmA3xmdWmZNT{ER5R6+L7vAqx^?RNMuQ2b5H7Hcc{4)Vewq!Fv7 z0eMtV3a|l{@yN@p)S&5^rRhGbtDB_7OMqNrLovwqMrsZI5b|1*zF!@Ec!J8uc^JTMiU#3GmDjS zPAW*8C3u4$>~#BGOb+HR+f#pP^fY;4TI;MijfB_FI?=H}Av>7p_Hh)DI4;Ao_c~la zEZlh*{yOI7+AN?^dC2WmY+$+W@~>X5A8>TXRw2h!pruJ%T4pX{&!9cJFi@za98qhf z7Rs@{?5n@lt78UrJ({G5^FXIRsQ+NQB3&fgusWT$BgPdMo-`_OePgNjLh1*R8h)57 z`BP7pe(n+QWtXBcsrH-;(7jEtfs26)K#UfMu^4CX@>^q6?4NxRtvbVb9*Tz9bD(0d z7C%UYbomOjSY7ZsyTdb)Nu%ZG+1D9Lbu{ZYCRt}NU=41qV*pG=x&d`l5v_zGl83RC z+NP?3aXf4*RR`;ej#f!iyFZF1cM`?Gx-j;XqD(fqxfOS_qh#J~1LOv2GM*u*?IH1H zA+4;7N9Keh4ZobdGMQud`}01HO(|s$w;I;)axm|Wm=9OL-xi#cUOfLa^` zqLU1SAoiYFHx|4{*P&WIZgre1#5+R-h@KI_U?|X|hCygbnuuGvGrupOm|y%!NBpv=qO@&-<+j||O^AWG*3O1Xn%>yc zTbT&7uWn?WN**TEfzFhXe|Z*}gSE(f#`b@nCY9vt*OPlOBuV@?_I0jm&@5Xv>7A&muyiLEdAO< zd&P?~q;t?gd!2SjG*lq*u-U$@=z|(t6lAAOEQV`Skb6gClESh&TWMd(nq2L$`IP=4 zNfCy2&cfXFxQD(hIPekk2W`?^Z}B<@CSZC$X@abK?6mPFQtxLH++vZ}b*UX-@D_gt zE0An<-CthRc+a!IUBdqCP}hB3Fx#)va~;j;oq#Mgqh?DI&~9@jx_LnazXWq{qiq!o zrVY)?x#>{5^#W+l#XR_{*8{&rvGcS}3gP2VPE z(1b#vfF&wb?_{k^<#dVq5>3Y#N+0?NT>5;rO|YkK>3=* zo~pZ%c@_7aJ7j5^KNLDBH-y_R(&j>=0HoDI2#!nOrJ4Lgytb!?-?_Sep^Y4ci90-j z%Wl`C`<(cQ#?+xiw!^e!Gt+vTYu(?7vqpylW8 z#t+kJfNccY-Di-X4h<9Yx812u=afyx$t#!@o{8^qRk?4RYoI+8J;V`ZPnW)>zNA)> zRZ+e_AG6w@eBmZAh>G>HP4zJ#WuYTyvS6~GIAxjDXrP}HgUmZ)n}_uFh@pEF)=)`?P@r;KGl5Rvwvb1Q-cO&ey~-7%)86B-eb7!;O4#B+N=wO?hP21^|vGa;jXg z>Es_@62r&i0e=Ax@!hyA$(Y6aw8_?m{~c$s!F*r=lwGgpQdG^5JCf0Sb^1Ii;V&_I zK|Pc>wLE5cebu$N1dNMF^8y=KT7KEM{b$|j=B|z%@t`bEd?ym*qxtd6==j*1Luh)s zvJ3ER=)2Dgt+~J`o?Iv%l7RTP?4LS@v1iQW`9ljH!1uk?e!4A)eEtWr+P1TDw3QSsqwOzlVD(UK&$fA{C}ycF%3 z9ru1>Yu2n?4YBz-h!$jP5yduK-sG%UYx8P;WY*lb!(N;AZnt9mq@0W8#C=)9Q~a=e0-^rPhGO~bN1PCgqRSfJ$WTeD5{{yM$Ik7iN`=XnUCLj?A5smNriaksf` z()d9?NO2Vl?-Pd{>n?g!!1gFID|oqPbGqFniz}z~@?loAozn$;Z-i5E+w19Cy))~> z$$hWicYj(W&-e=VR#~d$;}clbrW9StZ_se&hBt@HiF3sbtyTSSC?XGTF2=M^E>jJc zG~~aoZ)3#HzojG*{7w$I2}ze|kw3K2IwW@t2Ce#2)T7yUI|YFMh3rui(`02*O>GcV zyCIO(RJFwQW;tVmt1b$Rl0Uq6l#FLMDfh(Ope)v&32s2=Sba7c7Yr@3@c=w;#lJfC zklcKQ@uROp^@lNHr&TW^P*j_g{Bq(+-Lw@Uo8^&}on#+w_l04j1>SDQSZ8Bp`l%?_ zi%ir%<|1j&q0v;fiQL?pud0`3LMqL~kC_jUX5!%3#Rp<*GI8kjFv$3`>j>vmCJ=Iu zs|)NI9z{AwA+9D|{{n_9_H#1>+jg1Zx$D7g**|Vs*nzFuU%ryT?ck4K?Gts^IEoj- zg!ehK&3BGc;WX%Ay2eS{%d(nhm+z-#)!@)OA9ADxY!q&y?)ZRZJGgA>NRNy|Mlnam zw270)9*V|6ie=$4HF$;s3bZU*h{(INDL8Z!=AZB16AGhQm&obz`N*iH?)SSV$<#Pi zLrzBpLITkuaRbNL#tYxX=CnQG90zFaP-pJj{ZmR_p@0<7!;W$9l6#Rck$HR5c#LvQ z@nXJ<9+pQ4g8)UzMq1+XPw5&}&IzH{92sRs(CY%`H*mP3-i_XyHf+M^P(_roCaQOK zmYR$1wn_8_BBBB;5F(bekxC48MV4=LeFVWd@>JxYc-pG$!>lk$Le{Dx?Gf({bw$!b z+W-D=uCUzPVH+xYt?(~%H9?cVcZ%`b>+OPgeg^%xQfJ3mt-xJvzZwSd>uJyZT2y5^ z-4M$|rH1tyRa3|W44UF}Y4-5r*;k>L2eLRBuf(yC%d*^ZVmlO~>m2j1)g{YQtBLl2 znT<7qwqY+)Lin0hzIcvD1mN1IG>X#sihMndTnHfNBjN!~no)Q(o8Wqc-g>7#{*BSn zLysYrCgdEU|EYQ!t7x!AY>bF#a33#N1vR$8yTs`o2uj5Du--_G$bbjf8!cBTT0cu7 z(JW#*L}{E#EV%6N<08frV!^aup|E~c))V`?&tf>!?CO*zUY>u(>l51JFnmhr{sFi$ zSUn(qlUmlJmS$v?{!$^k3YF;e^(e+5k+@k&tvUj7cb*vAoZ}0%EjW4XUA+Ue5X{X$ z#vp%mE2yDB0La?dJEQj*!^pF>o!7Dx!G-VclG@J~bGh&BPjF44X#}z(i<9MAV9__N zxd$sjDyz*;j>YzHlq+3Sd?b zo!{{m?2B_)*5-^JGLE8~-z|W1JX0vCCBO7CRwBb!Yx$*`rkJFj*0w!8G|eT>r)r?^pyCUCtF;O)IJunjCaq47H1mhQqFQq+ukG5=LFTic znVZ9F9#%bR`>l~@Q#KEXfD+G0fmKWY*T+c^wPn%{7y&7caoHx2a>Mg z>_s+G+pR1peVFwO8li7sP$Hl;A-X!*0;=5W3^XG+d9*+4Z(9jBECJjZL2YZ6t+&V7 z$q|#b4(_U~HE*~q3y!MtnAIz$>U@DmCj1QO9{yUE@i))7m#P&9>^9DS87(%5Fk9fc zHkP^Tgs3&!vE>%lw8>=v6rZ^C?b%JzBIX)?ftSRaXl62>Tw`v!jW|=hK zE9zXuo~)uPS(TE_64*ozumZBu%#Rl15_k}7yg@kr>dluQ^zbtxJh%o;k2-75u_^Dp zmx((xRn*&|XxSDE-?>iKzR?81(97W0i&QMpFanMh$)xXW20YJ-yGF*dG}v5+7NCXB zz#c_@egJJl5iZA9A`-_tR9z#y)Gb7@8sluFu;-GMMN7_!puh#%7Vt*Xj%`1iMtH8> z+&226TrjUtm#dq5)XRPHL5}Dx!Nwtl);quAT_Rd>4-~kwNf*8GO08m8-0lRK;0k&? z5Hm80Jj4t=GJI6x1GDa?nP;giaQwG9pMQr(b^&g4r@U{(1wuNZAmjMoS%>cx)^z`(Y(=&?UTt8;jOYS#AryER z2jW1x_65|DupROb{C?xxHlFON@o6shu_0V2ro?M)jIoW!}r4qzWv7*)8H zD+ySbJ4v5#zh#k-YM{QqWn1)bS<1P%Ag9vkO#aC!EbZ~cf_m$6n)Bhm8L)LsIBYsk z5N+#$n9&_pSMP{bh89cc^+0p$EeaHjO#yj=lIx^wQZ^mSg>UBWm18Htw&i8u=WmQw z3Ebx97~c-ZA9wZgp-%)z?Icq~UHZFgQ;c!iIZVBZeLlmnPQI6fvFr3HEvIQ4mrmhT zSKQ`0#);Cbc$bUv1@(KIc1=%*hVj5Qt$o{ohwHE&nWc@b1n-M>7uokJz|q1T#Qq#O zxbOP9`ws4oGjUX#9IGsnO_`bosq$NI?jYwKO|AEH{4F=;3{F0dt=1QDlFsLx^2_Pz z0}D=q2G7E$XC$=}Wn#WY&l@s?l$-v|wdsxL%VpU*e4ZB1jVRC9zz#Z;bht^YuQwb- ztqO{*Y=(6$+t!069wX=Bvbk*gHF#q~f1@;pkgKKEODB1o?^m;mu~0E~B6J`ew-jd4ZB+ zl4`9RZ%aDN9@>8vWfQ%*gnD1u^HvgJwx&iB4tJi0>pP=BI9H>rVn03M zMqb$`)JyYduenmO;*vJYgWL8AH{itt>(NPG;4-;#!YmmcJ)6Gti zWsp2{mYWxVS>i8Z*?D5?%|&&-|MvSgiH55JT+`Y*@lUzL_L@sT?D&W6pE5aAYm%)9 zNMTYl0d-YpJ*HpK>07LK(Wal!4edW0i+Da`)o;i%JmD)=$LFkYx$Yv+HhEW#d$mM0 zWgK(s4^1M0?C!2U)*xMq%$c7vl@#mqL;D2~mqp>v2i(WAqvkXpxmhPc#UbS|ih4c8 zrk3tySWWWZ{z`PEUS^)j_U$4$jh86UI-mqxOd4lw1{~>XP}@k%*Y_M5^EP2X_v7b; zjE$l61MIa*S6oCgs79@jfD+}AFA-Sai>XzVYpZa$`@H|P_ti+gMQ)Tl0ncT z_*Z3M4d&Znb6tdzL!|QrtZ+(L)gYk{cIp;$#HiOBTcLdI4x^kzSR*WQ!|rb}YRXvh z5|V)-t;geDTK0js;v&n&_rb)?rph9h!lEzATjb1KDQ3oW-r7UDx$Pk~sE{=OD1W{w z@Y1s7`|4f?SQ`Xc$CEX>wcjja#3E*FU{tL?u8z(U{)y9H+U7*{BY|;|6}3XeO==1; z0~ek1M4a*MnYZ|YfVU;{1P2jAI)9CfekCj*C4bj@YF3!Hfx|aA>#>2Vb&5ZK1Xn4~ z|NO=>Nbqg@xwi$hGn%mHGXUEX`1}IHwF#4Yf82_ue&~JM(-+(GtuJ2E%AU1zSuS8x zbba>pa|thn*5u)$8!BQkWZ?WS_YV!C6iQGlOhtRFA%n2O>;^;AA5T-Y1RFi1a+e?a*CcZ9j$57WxpFxgIG> zF6o8QGt$B+3f5uv*;US}GrwF{a~obuU8BU`DL6++)^lf zoV~5G9A%QZ$1f$N@AWsU5`8`zgkAT zd_CG)g2bm~3K7o<@pSoNV~bv)W;~!l$HE0zvvtowrOS8)L?sjFni)G`u#Y9DBU#o-}X@ynYnvRLlS z+AL%t;|${XrA?)hvoA!NBD?J3F?BSQbW`Tr2QoUU128#p3WW)bjH#)#1&*K;57WzH z7c+DS+Y;tDX4MJ@20w%=bxz$0i+;$P72}IfqQlbQcsR^9;{H+3JvUF%yS+tu__Wt4 zB7MOyG$LrD5hfgHW;?Vs%vdpD$^+yY&qh?Awrwe;G`Rh@6I?&zv5)!^l#UmR3K5>` zYhb=tK@aiFP{#r9cCPy1s`>5sroinOb?IDOm(TmuVweLde76W}|B~CMZ461QLIXn|t*7@f*Zjy`(~A zUv{ib5gVe6RMT3dmhe-82|ZEMAoTz`ti;=GzE@cT3MZ$x-fj%9o25l0JB=CNSw??4 zAQByp>1dU6CCWh1QCliD)3{bZL3F_KPiU>VF^cH)xr?s5JXlT4%hx2V;dB;7k|Y;V z{;E6D@bbee!>+=HP^PLy|7Bzg6G>J9!Xs7~a+s^Aj?h3Cpxct)b7>ueaX%B3N0GH!!KRJoNcmCFwT9 z;<-*kHSHhAOT)zZbVmrm}>_~#d1~oJs(jUy+ zT*iX)`qm(Y3$#m*n5XY!ts5xHPXKtvcgD+wi_;-#d7$yA1rD&Eih?IRG|fEpw^M+` z*DQ2C+I=kvBiT%p9kd-Zix3E7u|5Or&$jF;#50QP#nC31Fyib%HbU41-X%|uh+{4S20)p_J^uhEUzXa z*q4t)Y=ke8$>69J74!`jt1f_}?T*HD-6E6#!3Hq$A3b`0<**Do55p3c7hJ)BgznH= zaW{I`6hccjFr7__=uPJ&9V2;Q_9~|zzW+`(OcqKZ7XI~gnSlL=_6T!ZCr5Lu|4lXg z|LStAmFH|W*ndS`YSh@f7HH-1mLG|_3hZ??WX!b@?B)vCP=3X#h<1o$@ybR|7afI0 zT)ZoA#S+ug)7P+GfGv_QqKQcXpbl0CtF*z&_r&d?w7atZ5>Xw@jHAJ{gXmWx7Z|q( zZ$YC_B>OK8p^9q@T%yQkDI}ZCH|OdDXKTXCIYHr7IKwzE>A;{8DRvd1YM*G{2dE&61CEDnJ?{P zEPvtf)f^(0KE@JfLuyU4otKfu)>{biS&o6bb@KM%)8i?efT_IMbMDeCggbK=>F6=& zeh15$Um^r6WL3c2G(UJs5Z}`ZZM#_kyA$Tm0~1gsLJQhePNCLPB}_brj4HI}FE9^$ z3TPyzBTLp~?q@)goer1G7Z=xU;pqRf-61(vX)vr>DeKtdfq){)(wo*aXslju9pMkb z9;Q6pC$l=dtB~eQN~CNuBo`Unt7+x-SBJKcYjF04t%k{Ec$15GNz9^0ZslcE+zwfU zd&K{*CBd8$e;6$h%K{Si)DGPpIkYY!p;A@G70>(m%mu_LQ<`-DI|9y{Jk?)fcCU)r z%sOf7#e`9Cwya|eG6Z(wYwW|vLwQzHWb-2Op3?qATKqUi3o{+85+K8^GFFUjF3l_~yrb}&JN6k9R&{`LJII`Hp7x`h+z27_8tIep2DS7K9W53XQ{fNB=R z>gmp_oRPZf9gNT26F9YwA^YxDUR55>9L69f%>;)pThow6 z3t&1j0L59K&T0mX1EX;<9z(n^xlr`L)2p1~t!YjK@k2#tX^P9ec53eI9{yATv{xY)yfkehMpTk44?&k&dgTDi3bVBRzD$l|HJZhhSi?S-FA; z1FQ5Y5_w%IkBZcLsui*j#<j+0J4;p|}hefN3R=a%wKB(o1<7{zeT#LwQqM^}P6P)bL#y_D&NR0O`X8@J)S=a77N2X%A2buOXf{_%* zkCk&bDZT&l>&c*DX;z*6`t_t+&Md-uW|;N7i+UE04kFQ+^PqcJWq*BEs8Aee-K>*r z08ksi_b{gIvl#D;#ubvq^WC*fW+9l)P0xgd;GNhSlsXr=g4}OqWk$^7_O#>`$m2eo zV9#e3Q&2W|Vu7aIxFavBhu2-0rh1Ok*{uK&Q>3xB8N==H2%N%y;=70o`qwZed%}Ed z&2~^hJ2a#rqp(AoN3w7d3-nG!RA~OpL^sIe9H6hU#+FlNP#{3BgFcS7sroTU?(Ci> zFfG3l;9NPcpVn^3aH|bj@dw;5bkn&F#_xuH!w+25t?Xc&pk#!Bd?7bj`UiYHZme9D zMP@<#5jyjU=At`u_JbDO{^I1Rn!JBII|Bm04RW&rSo(eV`u!WqA0_?c0-HLRw(|8Y zF2H%ZYwicR!Bn_wun^cJRd3G7X)6A)YNlLmWr=_!h(#;#{AYq+GZOCDYgx?~KJmc2 zj~rA-qRN4h=7X#voeVt%$ zRojCbvtOmLmDwT(eR$M}*B@HW;>}^KCMA5`!cqR(%F>}DGj5PP(KoYqJz3Vbv-8W8 z^Wp2iBJ|C%;R$yEjEba^u9B`RyYdq1@0__!UR9f z|NDf(b#5=zzAT-MmZaJf)X*%Kj`x&39FC(TpL-)!$`>cLPSW}?wD@(^-B_Zz?}1c; zUb4M5TW!Ss!@XT^n_3zga_)c$0>`vyXdk5{^04FUYV%0l$^x!r^U*Sz*JQlUQN##r zTa8-r&*6!xnnl)V#1NZt&rNh@*!dDE} z0w^&XXn6QAOz-WhES)EvR+E;910baXSdW!fS|katR!2D`IWOsxSj^Af7)Pp>jARZ- z7nCA>G!A)^gAIxT3pTL#kN(8yn3G08HUEz+Tx8aGZ(QT|ZW47Nf;0w4lGKH8cbO{U zF3o0}N*Ok@6pXl!{iz%VKC3mhv3rw$&X?LYlR9P=aB3Xk5z%>(N0ui#w7y)zSM@)V zEgoJ;rto~hL&oso0D*YO*%wivtakY^Pq>2?s3umTu2(Hm8&4MePIQ?+NtjKJsBi?z&-K)GdiJboz*6v3CF{Z4@7EZn103_e z&-HoNZUvDhS(<(dGlC4K)c18{6P4r93=tkzF*|ILa(UWtUubjCbt4!*J5J4O%C&|( z#I*O6KYdIPBy}nDg)u5b2*0buKEG986Yz%{oGw_Qscq3iD zrmR+yaW3UXo~623F4{nQQMk~Ua@b3^y>;tzpQ9&`i?k|X&RLu|yocxTVdC?sH(|CZbkgq=EYuU#F+6vWPPsgQUy`KOYfnDbtBCW{wKJNk$tS!fbehA4 z2+W|RK~(pMaX|o@^k=t|6{W3Lbv1A&`w(IPNlVrHiJT2fiSEOgK>wmDslt4l@sGiiKV;cEyFEMn+^rX0xY9;mRCeJU*QG8)A z^)~k^XoHi0q9Sh|*uVRq`e^F5jy<*3Ac*XT`y+K#V6(LEU@EY}g+tKb$naY>$ieC^ z$S1_Jf%K2)*0xErHSCknyXd?sT&Mpww7@O&h}$^U^9DtQUHfuz9v>+??l#TaE3(1^ zyXLMeh0&=ZS-C4b*kz3{HrxjWKnxrRzL#V5dHVOTp;sOi_vEd`-{IX$)Zb6s^J8 zM`*J{z&5B>=;>QOLQZ<(4m1?WVK13t;(nYpCI)F!*_3`{*V^V^5gY}^%#k&JbHmC) zvoxpU;sY9B|KQ5&zLIsm5~PyNpOayFucAMie1eR>92IZUVh5l8wEY`Nysfx~@>81& z`~mg)%05N+;(+-E{_kIeSBF&^p1;23FEjuE_W$HHV&LRx_J1Jl{|`LzI4k=UX<% z3#2G$a&_XA<077aW5?pfTL^|rqZVeTP8w6yj2}g?p-9}X&lODeZlPBiPwytMto!r30#GVl=Z%%!L6l;5G zC+5sB8d9hGRB*z~Wz5~7N)BLupvf+)+94k&1%#kDL=6T1fM1a#>>2joI%J07q$;Rx zpyqhCTt?`Sqg+-&B|?_=l*Y()7)@wCDH68dRRHvgIdVbNWGq7&bOvn6jp%bqvfT0N zHoKZ6rAsHGfvFWClUdV#V!^~ff8`F98xeeUwe}}y-B}yamGj0o9-;i7!KjhoHYSrP z7%6=U##vUvI1G6y+<=ZI6!URZey3#lMOB$fU!~#IUP=t@FRVE0M~DWi!gfKLb@@B< z=AoRl$v(mOzGaG7jo@t!vsy6GQV;}1Zi}@Ig+UwbKexnwM)bL|z4uUgt8}N#XBiLe zN7ccOYo?ZELZ?r+X953;Yx{sV(^^(_Cilx$_jdrPIAJ#`@?VeZFwstk!LUcO!_EJ#m^RDDUv!3$O7LgIraQKca>EE!=a2IO0i zYr?={`Db&(zROJvU{n$VUakW=>8Ba)rltV>lr&kXJK^ejl@5|OV#F7UM0&`c*>_r**xZ;(}ll^Nx0CurQ^eE3EsU|Tg><}gzh{}q1^Uqe1h-;Er_Q}>Cn+UY?+5p z(b$u?dB|WLz!XIg*W4gl@VxOCXXb{Lro!j%sH~2$->Xs9x_+YTMz#E2)2m(Ag*6}e zwi|lWJgj;hC-dc}$jh4Zg@*j4&bNA&CH|Wr=svjiU!ocTvVU2e56vsVDrk!zNv(jR z<{duO!oA($RiSA8=d@kzVlFE(d^Ar%=yPuCbS~fOwS6Y9DSRH`gMiHgyLjr*vu@j`3jZH<(emuQB~gqoif}z5K(>77)%bJX0>jzc=6Cp4LG+VTh5QU| zkB3!R6gC(5(n4woBlza&)*<27Wm54h;Li*GIRo<=sbrliac=pb3(vRw zZGLfz2Swmq-U{!}{nrce$MZm)Ri`3afyV0Y_TGt5DW^rizFNrUpM1M;wTrtdsC+|= zD4NZ#R^Rvb0#9kt2E0#j> zw}T%Fc#)stO+)l(jlP8awh!>%ZQgQ|7HKNi7ur<3s6u3NPV7kDFJA84zcf-?qh3a< z(&*LCjvx#!9-RH7f=bi>24lHT@7dhyR+bBO@)Bh5WPoOBFFs-B#IAW-^cTi8H#`Nz zY}2F)f9PV};ywFrpNhw1EExtN+N+=3;}W`HX!iea?(hFFY1Z*J{JhA3fQC~4KcoHs z^lkr_Xn(}}{U7r;t?#m4F!?-F)>6!=DP2X|TfQ+>ygBZ=G4+IdBStteVK|EzEh(z> z#qIaS`#%)|Fmh5V>-F{x7nU}0VEF4dcml$PW#_iZf{k7SOW&P6jQ@3`A##g0f;Pg2 zC+Z#PDUmiA(VPa;X-(e~=o`tc#}!51@!zb5Vz+QE&b`3y=l*XR!SxcFwmS}5sXuL# zTmcbI?6F3(Z`^xRL1NH<7Dt%lzDW13=&4UyV!%Kw8uG~^T$J7zH^c8`-q=^hnED7q zPQ@Cp`22j_-Pc~A7u>RD@W+f4f8KmNxH~$qa6y__zvPe~dCjqYJEE8gotq4QkR71E zpbaoT-Dzhs1g1E*X_SVa>7%14Gd*b)%13hZ9RrZXJrl)}RSg-X4I!Q`Km%0$2JxTl z*Y8e%y%+kh03JYx*A7n?4>J)x*ZTUe4B|rra9!PI<>?saLko8gAiX^%-_iLSyI~mS z_=o-E8e#S6SO*A_;^V^j#C>+09EcfZAcZ~?!4}xyg6hoL?<})s=M;Hlg7Df8Hb}>m zyaS2wwXB(Ht|@eY8(YUSHN%~=)G|i_gqjwhHypqURHr9(#k%Xx899rf6R?x_jIk0R8aJa?Ai0HsCy1 zW8Dw`p_O_BIVv+?I?#d+SO^lj;4;Yq&z?g9 zqn>R9pY>Dn-IDWmWH|=jGmzO+n*WXs&L?P=DtdVlHAbBU`x63wx1Dn9p6YY2K7Kgp zezbiUHOI<#SWpKoXbFA4S9K`OUm>Ln=^r16 z7cp)DTtP~$KN5h^F&t0Bh?OS;#R-*A+wf%`T1EvU<}@dr!#=YYabz~ZNF0FIG~2o4 z>$uFv!*jv!3Gm%KchPytaD8u1o1t`L@%8u4u>Od;C|)P|#_#;EqfX$4h5Udvv1@kF zta~ zInLz2FC?U?53U|mL5%F|b8Hltd*n-Nw~c1~8llrFI@eY)N)!w_?k2|Pgri9L2HT)s z@@GW{Nk5<8Jh@_ghaE?rYKQ(iPeQXj1Qg={Q6x&TyhxwPvgWRYOfyG#v(#8gTDQn~Edb2qP+DmUx)d2j3Ae`BMc}D#TEU%c~73D6S?Fwd3l;_t4_;z0-k?(f=lrg8i>!iq3ICa}RBs)f`^NYPyx&J5 zG||rFfPzN4s1^_!;*GE7f=wwC&|V{=o8~(tc@we~<&gi=2t@L!gcDvBta(O}ZKf*t zL2$-}+d?7ddNR34gB31cID*q{Z8|=8=2HJ-VbTo$p7^!?*XdPy!^?HU6JiKkfIW*h z8IQQ`U!L@PTq7oLpASz5R%~TAAr0s1JkQy9zR1VQh?gbZI1AY#CD2)c#Huo?Y zWo=_|!6Nh|hEs&vvrXW(>gCGam^x2_VyC}kU+0@Oh~bZ7{5Ll8qXqjfGOda#dM2eFj7ri{MLNm@@=)1P$!UdJXh}B zW^4auAc6tCdk`LaUKEkzkm@GGz$i$hTu^Y9`38`~9oArnyqzo6!C<@h<4tN?4(kvpKw_trX` zFy_)rk>e@2Vj8~45n+dOXfR6QN*+`RBv#0e`cWjplH}Z;F`ZFe42$KX<$nBkmKI9Y zpfnJ;Le(5}$!Q5QnF1hc#DpYXtjZ%Y_!7E)rZ}gNABOo3Kn1BtK!QrE;a0y;<9`jT zfnEcG27qX-<1JXsA~KA3B@u%dPCPXZM7yChs)M>YzdJp(`STeJ;kdc_zAxR`IFx;O z>GkBlHZSxo5yJg^UU)f(%=GSGRzL0o)i`-Ldi|W(c-)hC10&MYOI-Q9d2xc=70YOK zS|M8{7zC~bUoNRT6#p7|@Z)}8HKq?OfD8LW?|oiX`SW4t^>W?nH&=74si$**3ZBb? zIy!nXb$fWi(IDs{(2FND>Grd+zd`plWUGHaI&1a*LRfUA!A(}@*AHmNvQpO5&&@$N ze2xI!-Slx1L>vdcEj|9+wT{hx)PD*@L7bBHca<4}7{g6(se^Vxb3X0=Qnd*sjxS9` zFR=MM2jHv~Bad(G&-5FAU+({>_zv{+^o5owqBOW63j~G200l-EF_A_I+k55Uc>8P4 zK;+DRmJj-c16HhHt?u_AH_P(ix+ryUy0*YAts)4KUi2wS5u&fhHhsOHB=xbt0E+AN z!2c(-@2(jRQy+|K4de^sgE4V;K%ah!z)CC#dG|q%KKCC;s42h$)~_Bjo`@#4P|^pC zpE+D0-tIp!E!$&4eFyW#eu@zJ^*%^~G8#AS{>zo%352q49v*{$0Z0U%jC?}pgUdltxYUx`M76blxK%qr4AQM6 z2E>}zE2&`W!&-^bHy)cQfd;eaTn7pblgtCd?mMrmUAJ+7bWm!Ieh5;OdMGEy3InI) z&0Nuya|BNv0xc4q!4^2y(UoBX!d<;GnQ|sj$Uk0=L5_bIpy2lk!iobnC5e?p+{(dN zZwNAdK!1U;D0jm5VwuB)}czAWIAYo9{$pcC^7_+R$na)*a ztmiTIL|M^^fvTn^)*9FQ;_lS{YMq<01 zNI&N*7Q}zgljIke8a4l)U{kDR4V?vRz(7lp4R2Z#b)l9CL%R_v)Fkt$aXi;B%-nO; zgc`qnwz(7DP-pBeGSp~V=OZKnG|RWDERH+%gr$1=R3fnxcJXkj6BuQbBTjGntE7a} zVOxhvt8R&s3g}sspMVI+rN~(k(`1vH>jtK!uCk8p#iLVn^^WaR&@<%jNnEt2q8#X+ z@&QA?EN}I?<7WHDO{6dfl=)a~IdjGQ7TO?~rh4pe(d9a)f(R(}$wG;h5G)s}6YQL^ zq(H$fPfBt{bLyIG^c9{Dl16vX!jXQ(V2=YcI`Lp<$=et+dwECZJgRz1aqE60dbqfU z*WfLCVuB~reYmf)tiDZwbw)wgQl=@S)e(Cg+_Jgsbz=1_fR$*#mOF0VfX8sBhb5}cUoC< zMzFpfuO5GIFc(&SkhRmwJG!UTiqjoZK{T=*uW0Td(C6XgoUnuq-oL6#GWMLsl73j* zeG2}YEfb%xG3F**)Fg+rZ=z`^htK%s7D<+spo9j*KSAJR7}UzWLDGN;c==PTu=)4&e3u9<6@{4Gsj41GXM%$AUga0BaM7;+Ewl^hRM#fNn#td=)X>ik^u;#{*>EoBdb4&5cdbf z#Q&Wo*0fn2Npj{a5z1jKwbfaIuYArIDnh2AQ=bWLY+OaGGsLtWtsa#LOHPpH4mm+yge%YPEukg zB%yK{3n&5IR137N6|4H_ch25&XVIH~=@z^&kR6%y@YzFBm2Nq_-<~@?WMaf+P=BK& z*eNJ)`R;t#$g-ESx7el_8BRrY=N$@j{cd)9@8zbiLeQkMo*2KK88rnxKp0fRq&Evb z;kC9`%gnif9&ec$4YpXR(i{8kL8Qf_DlKM@fU$jc8>bfmOhu5D%l%nBcp#|HacW%j zVuPm9e?KKGQfoFOK8)aN|CP?rm||Q4v$=L3)IFfEBHI^SDKKj(7x~AO>n@=#mE3lP zU2vKpG9#I4rY(cVcQ?shS!Sz>{E^UeaF2`-v7hL*IVF1K4QuO;cmr{vG?vPP z3U)~4&2Xl_cNE>GvSVOe?(u=(%#>Qh{|{Vh)a8o zGq2fTG~W@en)>0bE^<_`@VJ&d(`8sKY&FA#A0qg^-(vsB9p(J#UsCfcp{1H4DcQx! zEMEv!ihPlk+0mQyQ2xkyF)dgo5aaE&ix-H_vHW1WUQNN0=^}$K)xmVQoTM@AB^0mK zS}rpV;oqhKTMQ=euA1k#mYdx~x9DC=F8_y5V!Ro3Y$~**x^qA1s5pHkUejJ@yw)|N z4$sVlPK(gov~8daQ4jUj-w}?x>ZKx^B*6UphIsbV-d@1Z%L=gev^IqweI8KS+8M39 zvqp(eDCZ{?{_md=Jz%~OkksoCNPd1W6tn|JDhGKwAtJuXn9n~|lz(l9EE+Y?I-p-L zc_`n6*l!8+N;ZpluPSTTo1*%64o$Bhb4q3YiTqWnr0E8rscGAdjj}6mgzKzhrzlM< ztGMsdYfvcyZ?GNy=UqUtS?13Ato!uygKngHY_6TY;|EiERd0DpeT8U)Gnf{u9X(`m zbD#yWk+!|DV9ol($_X$RU1W$Z3cDEZ;eYoH*KTIdWNk${BkJ!&j(ps8yrN`~h;s|y zA^{UMuHxe}CE8_&_IO=Z(Dz6>r2UW-cX^gCZtcU{mccd0P0X!GnTt8h^ZUF$FZJ>vs_ ziCCH5o=|&-{WJ_`p?M^UNqwf9{Kz!ro2ZJD-DIZi$UbzIT2YDr zZnAc}#jyQ(k?Z@e94oLY>~^fE9u2hz!)SB)80l#9U>%ISu(qS>JEA@85K z;AlPcm4#!9%Q9yGvqh_H_1(8;p)I^fD{H03Ph@n`#%!nT#&2@3;>E6B$5f7V0PGE# zmwia1=Z>z3FWRbR+CD5>B7=zi+U09asOPsHXH;edX80PH@en>=J72^AE=-^EughDo z_uBGanuaVK)_P+!f^Zw?-d43WXRZs&$g#1Ut~Y`{(PZV-upZmKvPP?JINJHV87DgY zwYvZto1eu3;@^};m~@xa z=aY7@i4_4=#EXsT#E`S;-h>_?mIsjS?tbVe;ZMRX#95+>ICjf%QO9h=^??i?S-Cf- z3!Sn5%se?dqJp+E(sAe?q_8uN#v`BUFy3*w5wQ4|b*8_A-hp4x9J5KeRJsTb;H)pG z3cF{*80lw~AJ5%DEIkKxt5!qiz7?L3kq!&}8Z{3OQAk+!#BlUY6A!JA(~9cGUQPF8 zxT9^z@a@W9d*6vZO;a(TXr!7$2@XTZdM=RuK4a<|yQctwTH*N6(FZ2?00HqtM!^!i zEpNp1i(xX4V@Ofggf72&tyDfGC!?etXxhZC+~(}UFNP32-IZ2Cj_AVV$Nc~koHtg2 zUUS-a?buzQPLGyLI{ctyu*#`p`S@soaB@5Bc0quoaF-iq=YcNe;rqU3YvG+p#{f;E zzund{js$x_s7OhK#@Bd;ZcG1N5MhAT?;r&@f{+nVSq?g*78hmNRiBhpeR<@X6Jmb!ZJoy9}gh}v)Vrw1SW6>wI7 z5;gL{P}mK!`zCC70~Qc}^CNw8bvlSthqz0w71SBQ-Hs!z9#(%YT1cxuaGAUoSI6SK z)Q8);N@|mGem_&*p%ew8>6dfzf{g^&i!iX45&8RvwEelgl1F1 zI;f~ih+`q&;4R(i`ncFcOIYqD1(ieaFZTnV0O9^^3WsR??JIg_mWhLi&&X8>Hh~a- zBKU~{_GHdnWip4|*B~liqM*DSL;r5yddQ8gd5DQaJG}bQ65>9v^XlSztFr~{v*G&YEGMb3gvzcfx zF~4d1{oa9;N<<~wvc!M+7&=k>;>3?1>pHx*V-{k*EA{sW$%b`rb-Hq|R(Vq5LHgYF zRg%J#`s@OI@o;qiS2+r?lFmA4rpv&V^WXLFkUmb<>uXE0)rE$Rj>;s?A^MMnRss7i z#rFyZw`h^%C)2R(!A;r?;)sMGWh*1rHRjcZvHvP-YA$F6mjMN%0iTo%F?D_$-|0Lb3KVo$a zIt?6V77e>>*G!(H{4wNEA!h8HQmp)OKS9zdWI(++h1lX`!=!cP3DnZqrLe&9UGf;> z8fbz$b&=Iy2s=<9oY{}ut5mR3>lY0 zaTjaM5EPhlr;DxGp20RkXv#BT;l*c%UPv zxr6%L?_QuyIuZ2s?-DR!vUimPr}As2NoDyl6;`glJ)?vK9TO`bf$?G^Sa@5VeH?p` zy#~#rQgg*JBVuBJg&!x1QX+{{@EzSg9>-oWTk%9lod+2Da-u+Psr<_kPVE`JV}8J? zqsaUGAY$*=d4`?pBnSXKzh8gRoyn;Ed>_!HUl9sWtr-c?YEWqbEOlD7bVewNF%xo> zK~ddS&V}hs*fo|NZ}ybN`Qf-XJe}lUq}aGhrJ^lbwy|JD5`bso+?1gd&^K*WCIc%s z5(y-Kr)|$dYmn$%uUz-rxY^mb**S(FD$34iu{6_kyi?ej$D9*RgC3#mL~Y9kX6fZ& z2lKU2ldZ0u1_>~=p(!nmMzbDkZS#ur#*TRE8CAOJ%{(7$g7O4`6CY=xYakaY&u7CP)l0$wGF<;b;Z%#S=ggat;_!t zr-2y}KmnlwuxAP2XLb=b1=+cu9oKM|Xz$hp6JXQtZUg(#)9-KD3b$HefydCpSffIn z<w0JAHa4qT0O2D^op-3tXgi|Q$x@*kOc#aiDuL-@V<_mGbMLP4>YUIAw_{DI_{0Dz#5aR2VnkrI5a2))X3zSW1jzTSj;CJHc#sd`ki);eThV65{X8)D5 z?mMc<90T0q2_;rZyzFWMyQmT`cj3JLOlA@7QQN;^D``&);d{#OWK&4)L>UY@FGz(s zyn3kzLvK{~97aMts|07wgp8zLb`2d98g$!u=~FdjVQU`De9b9jfoq!E@^2!6C%ZtH znZK|sKuOT7XjiUz;P(w`^%vKuM|g^(f|$LJmP z4&qFi(c?T?<`Yv$54zP`U<0b14Tji&*m?++E6lx9KXl`=9?SDk&GjW<$|%SgLc=;x zcp3!Po&OH6XC??&!yhZg%`A2eiH)pwXU~}nfB4oo1L+!*&18qu;=7x1)Stt$fx7wZ zKN}{V3-0CCGS{ zxLEkL|Lt-u6tg~}*oz7$G8ckD7iT~64RvqK`!e(}PW<3c%i5sb#r<`1h)@+2| z+%9D5aFr_QQQj(H>awdRt*s&cKp|J$D!Z!AtwHO1#ir7Pi+)yID@uJc-G5;a@35`PtVvEWGGlut&rq$9H(i zZR|!4^f=1&ffoYuG;?M=Q*p`WHiiSx7dY+iH9g5-=Vr!P^;8Ej!4BDRTv}_GZ z#u{q<4?DQVi6^VST?>mDxo(9eCpnK=umka5+&SML)SP7ZOCtcOOa88snYCMxDMI|l zLx2N_0DN%NmZW64SPrze3_m^e0VxiB)M@e)HeU*o)$qq7C)Ev5>inKQWEUg@!uRog z6O`XqA%W<~ReaT9bv0v-*gaqc&EOe7^bJXgsGM-f(n`=rNPX@S>~Bv3^XN)y4lCj1 z-Gvnp8D}J7bc1q+Km;ev2HkSF*M7zWYMiqTko1<35tyEWWiI0ZxdkZ))(Ft?03uzH zbvr7GQ=PJlMb{mV7;0RxCAX$4gwemCzZ%)$Z^c*4xR7T7TON{u0hU?eczaVV7!`lqlcderR1W(AoyI~GeodcjydW!%-M;rAo7U<2?0pdqxJ zSrV<9u|Zo*pS+$pc|RW~$=Z|)oHilBs=4Z0INssQp*S$h z&6mVVflncnsxFUd!%qA5rbPxjgZSL)YDQ^nUlpW?pd-2vN$yfOE)OzqmSW#&J2%DdK1y#>+637;d>T)T90~ z8F1p{3Y!%ZtIaSoaAI7Yj#*`PL*OVA_G22lm`Thz(uA=Sst40T+@phW+;{?;!ty=S z<6iglnwv*Y*^Rs?)CTOCh3(p&UFdl!4)W}Q0Gk^w@%Csc?GBiN3MUafFGuWTKdA=+ z>C{Is?>q8$Iqpp&U)wYcDxScQFC=y-pmY$=0%Zbz)hvvhkvyG{QboZ&5!nm6AmKCB z3J4U7oG+$=$tPAaZCR#Gqk}U=#klypB9s?O$!H4@IG?B`%5?y=FdGe&fdlcVB#&08b%VpU!Wi_-W@w! zB^!8Jz<=!F)__v;`~|3buwF9lngLLBrKsBRRuy%vyDlY~m6Hs8j0vC22t6xLcdhiE zi}_(JiscGJ`21(=;uWqk(5+lMq$H{SA2U-})~>SG0k>$sCFaPkcJT6TrrX^Tc@@6h zwUST+W^2aRC?ioVF>ln?sLuyXUiCt*JxZt-7kPBLdfAnoKjbRY(-f z9XuPB%VIB^o)d6iw0xOCD!I4=`{!Jt6}Y4o4$#fcXrac?Yni(H{H{0Z^t_Xs- zQDNj&!u~y?%n368&$W0@V?k+vT+`U5R`jr}!hg&CzkQgAztQ*;6CAMY)L~K8-s%Jv{$vl--J1)aijXp)JFr`U;^Y9WDmM91WcOMM)SGSF- zMZWMGx%WRxf#@dkAct+&w)ex)%?-Ug^g>5feLm`Th3CV~+uhmGmjy;b=?TFku{HKz zbN`zM>@v@P$mhronWgCghKYd4D)e*L+=1pcVV&EH?wP5}NQ+)mt_6hTKM(E79Epp*BE0T!K^&Vt_Kb@5OAD?p8MJ*fx3xCXAJCP@^4{ zk#3}BV%vau))QS-fARH}vVb#YF=P%PR&5j0DNrUQ8XJz_!T?20F6~I-t{E6B;OM1Q zn#LLfRVQdJhSPUj{@X>P%Q;=Op*g1gW>;19KLN_98!=3fO)9Q5lP-xfTXvcz+a~aw zE$_t2b1Scr{l9a~MRM&&BpQbA7&V3*)+-z~75PW{nDk#r3;`8hd)Sh2ioZz=XikCz z8LGKe=;3Cow+@Rajxf{!)nPU^ryD>2vw-*T8O}e<-_f z@6|(n>T`E}HPvmwMOD9(5gJ12lLTOPaKA7s4DAHYB*Gv@P5oYHQ%A$qf0 z7`u0MTwHuo@c&h>H-ua~A3F>Fz?zq|;)SqA+?(%j0_QjS@NU1K=Qn-6n^^X_`nWl| zr1b~@Tt9aQ^s74oxA?L7qou=NNWQit&BRi&`i~gY+1|Iy-L|&2la15_ z1Vi|ppe@n2zI#Ag0D|P7A5Z~8-p)=`J_myPIfZ4}i^AV&H3?VUl`@9sKu5lZVL5lK z*rWwfFv$UgeNFq>59mT}{7XX!hv2~TKmoL$L}(#ox_{ED(zP??4aN{v9s|2JUl(bOvBxv zG;eRW55Ak(PNdtoK7n4uIq{qEDQ*Vspx66zif__QPhH&`+2QC@WAexiyJj5J%kKTW z!7ycA#izMUlch?cOJs$>C&V#Gzp|r zXi&9DV2jQTsq~jEdVGy77CuY#Q4%i*5X;q;$V&w#fBO9`$beQY)e)sahdF<&=>N$6 z5)4~si+{Xx!sizvrqG`98fSY#6j9{P%y)v?swG&$l+0LTn=$nFk$4fd;S2)*WW*1* zML1Ms{4EyP&m0Kt0?MD_XUO~P2tbD^T*CB8Q%xpOkEj?IFqf|Acw{TtprE~p@4fkl zZs||U71NV@pD46<*sLJH0skwCwXozro)+zb^~`oCO8J-R#(IopJ^wS!5%;>Ef$-tu zZmg|}x5(chF0by1iMqcomc57#z2B^v8`L?O8m6WB zE@PhVt-&pIZJx+>09m^Y8>@l9m@{jCfE>FAddur_+zafJB?`OY9*_)+#uJvo0}^5#j!umA$eLj~j+DLX;AAy=k9foq zi=&Y18qCV_JSy`?+yilXz!p4>Ft zSlu5RnzKHLFOz-U&Hytb-!)5y?yW5&v@%$*az8~{36)&~F{^YRWSbK(J~;3wjtXX4 zy%Rsr;|2Q`m&pa>H9Ps1ygcg6Is1jm7W3VwhqYP!8{D$Dhxls|d|AH({{#>+gz1*# zf$nuN3H!d*Hwf|-l@@hyaZmT;9mhr^@JRk_9_Lxk@Q$wO*~tFc5ut7ykfk_ivL-Xb zmr0$K%HCr~+b|4Cct~MdOO6fGE3WAkb6@BSs6lUgExp+)jvl&AdOZB)KL8bIAhHgH zYP&_}${qg4ogncwPVx+l$(Syu4a&D61OA4HH<>nq1|u*JE$hu_4lMBi+`_>&?#|gG zwTm9SQ*7_O^oGl4pNWr)E-m1t>w3KH7j#4W$ZBiOaZ=e-1WeM@Cb-Su$m_%_jUGAI z)5DRrbB4)asZ)>al(w(OUpYBlF098+_pTMlpcET-e3O{h_6BR%L!qeL0t$@&GO-or zDXUDHwkjZ*(^enuuu|Y6;a#GXK&qM zxq<;#p*c1N-QK;k3^flO9u?Z^p^5{Ic5_c2GSdz?=pT~X{FfBM);2#p>m?c9)7o7b!QNIuizB6sJ zz*H8-n?fz~EN^mKY0LU-lIrM1B~?mQfeJuuFW}MJ43g%(pl{Kb?j)pfC~M^3Zi+`B zzZehb8^c&Rt>Nu=2-sxk#;bd{tTXF9#0P{#Q|D5oTLOO~-6ejiCB1iu5uPQG?*7o) z|7ptWmG5rQ6U1ve8y_Vf-$*Ak(J!kd`fh&HwjTp|vr430LX)R&0rGZr0|yziEO)C! z@jT@5+w(a#M_*3a(Yj{4l?d$QGHFG=5-4dBf-|8$EvRfSuS)aduW%e zp@wGM7lI@ZO3>ZMy}&Mgl&WLNm)}Si_$-`w-1PHp6d($E|D#!Fm>L9j&x@7=Q5jz% z)}(<+?z})u23yzomPZ)Z&pI;rLjjGe5>*{pO!CrrT*6>>e2cE~Q)*w0U`RoPdER&{ z(-aAChWQgt76##>W}d%lW&>aJo>~p+s9ffIL@-Mv@53=W^ z>>`Tm+QBUI*=wD7?>iEtexEnd!W$l<x!*USSnqw6sU`UK~N^$bl-VCMM1J^c3x#lWP#0auzvt_xG9H;BK z_^e|aM6iv040+ejN9q;6Jpi(`Rq^v=g42K{ntZ ziEpC-<-j_$7K{?Aj+#d?n3H_a%y{4jlT|4a@Km9m@hMq;`s#l%xI_EMOOn{Rz5}Py zhv!^r##FKntlLXer9%a7LiRA;qDWv|w5iwRGbK8kFmGJK=|F=8H)M7ul{y7{S7MxP z&?itxx+i4Q@$#F|Eh9SUtmQE*>OvN@h&Id|jdE80Jn$3it=`Uxj9oacbXOAv9_`{I zZzD~m(|Nu~jT8I%=!cKR&441Rfak4)QhONPp6U5`xr&dB@4kp*yp?98NRIA?8OGc1AJir zKyI02NugIC@G%%mO*a`?S!M`R)?EBxG$%Kv&?SJ&$NqPAaCGr`O3e28)67B_$%pg; zFrv!*2c6Ff;uuqo4mFAoXfis>no0rt7r6NRF#CeX8Jx`Bf^F^C%u z-4cT;%YptVCFP%41)9Gfn3ct1t^}b%uqh6BXqsB-PO!~To;f3ogda9)GTi6IjrO=? zYj_zqrZml;*qIz-Di}GanSfFnlE(lq2YHhEZ?IRt3tEg52s4jG!M@F`4Gq+#8L#Yz zR|5GtexLtpD0+H)QXZH-J=c+pG+3}H(1N{gBx#FLLHxmO*+qn;5*u>GIsj!%+uP40@rg`yI7Ze%kE+!$>lE4 z0tfo)n-4Ra^Dkhb0CDiY`^?JntFg*UjMZLp#@bB^#`hAN-bcqsGDkU~GH%k1me=)g4Y&jh@>*ylip14O2 zLpYZs3+4{XcQHvyb5PLDE+ei>NZ#?HVMlyhFJlVFg`KfNS+$xL^n*tvR-^cd^ zo<>1|rq_<;3XBf80WpF3ghF!YC2S|$7g ziw*y;yFzo2{EA@>cr_gFA;Q)nU?Xg7Iq!9u`Q%;)SQh+`$J?iS+K^KDt~vF{(0C@z z)IuvQkmdB-C?>;{ADVZ7|6dTmGUXRy+JN}0eZNU%Jrze#$WGQ-+SK@>kS*_0#^*rh zbF9&IP3Fq`BL7_DHgoZqkdbAx9S^Qx1V zbWM@nX(6Fws|xzcZ`Hn`DY~r_=;W9G>u|R8E?bs7K`>cT z`ap?qgZ;T3ver9ToHD^1EKAn}dL=T!6?4b5gRswziTpdTk=|*+sidiMNz?=Lk?5lm zKy47UO(p}h?p>I>prj`05E1B-8+c-=#%yB47{zOvZ=s0fGxb5r6C^f_2dSaYsB?*R zOdA0#jv0WyLRSxe8X{X`fRhkS4_}}w!A~OS9~npBqlmVQz_e%FF?`T<0pDgL=V0~V zG}npE(g)N1Mz#kP`T{;lkReJlhwDXHYC4}Uj%u6GjYn>%SJ<@RasbWTmh@8M_TZ$p zGJOR`DPfI#wkqu%7mKAK$Er)E4*f0+z-0#OoiDcEPY$DWjto%4Kt zJ4X3%H=ybEaeAL@EiehnIZ<=#fx16Cc$zuKZx}YmA|Lx?EyD_6Xl9_}{r&5O6}I?y zb(A@rWIP*%q`P0BG;PvX_xv9PIleM;QvhX@UhAQb%+_cCnmGUMZy8(tPcGJ6xzSl7 zlWpz zx%f4DYg>!txqORIcDKKtsG~0QxJ?72zZe2|^}+u^@%ldDb!ziOHsmMXuH2y$bSuUL z!r@Mv1x}Y$CtzBpzS%;Jl|B23)8c)jA$nStO<`!1*;&=U9T1*9h`82-i3wuC=#k;w z>($2d$-bL~Hx%BQ&zy+}&}IjOb>MdjL2%T%9{RC5#9ldW{ypI{Q+jnTCYb!%o+nAp zs?olHTbdUV99r&MA{vNAHRj9(ei_Vzz#* z^|Om`q=7m{vQ}b88sL4ePR;10i1_d)kx5+iJ>CI3=X1Wc`Yq1avX?fbX2b&rN;PHCHjDYW0}u1QBN>kfb$m0q2Tuat2po<)cI)S zxQ^%DQ3wuz;FrwGoHRJ2bHU8Afg`Fhy?ft{38QMJgI-U_pKMc|x-KcB9-gj0;` znTUNsrjvtUE60<^a+%W{DYt2Nj=^liNW1wd_+(>9H>grT%dlMMBU_sSAM3>c!JdH5 znh4grrOO}{zq#nsg9tZOJDE|8S}4HVq%4;Y&>!{cMocf_FS_ANSCBtX8XEdh+E(iM zeo}Ck^~7bqQn)1m=R;T_6Bf>-17C|F4(u2ue9GLp(*AR{4gHo+=`Ge7oO)U*L1wlO zaD}*BuU6bn_$9bgKfk7Kehnlow$Fdp^`HKR?>Yh zNSz_WH)i~1g2KO~zcKz3A6X!agvC0auBgICm{kIg|S481bs);+Tss=Qg6SU@}=hJ*z^o>czk3OH&AZMla0?uhM`;YZc z_oQe`8{&~htM1PP__X_5shkqsgW9=QE2W>mval`PeP5{xggK)pSBRp7I$n!42U+-^bSJ* z$|SvJO;r+NH9KU#vyOQ1S!ure0B6f_A|kqdmw(ewOfe4M5qabf0#CWJ!K&p~2%T>^ zsXna?da&MNI-<0zHwX%pVD*?42;~$sUmp__FXpH7%QJ{ewh84OaCH%5cxLe;6!@Pg zR7o3t6#Us}npC%O70Gp8b*s}>MSf1-r4aKp<^PAYbBYx$>e}p)TWPDfl| zl{fI>8?Z{XM+r3Bd~_(^6E&W(JUFmh+E3_}@J=}yxj{XFLbx1kfN6TtPrBLZjThYC zM3M7Mx6 zmlVniti~%qQxXw49YTclN`|H{pQy&Hb1RFSCJ$4IZzoUcb9QvfG7W_tKcea54j=Dz zg>)c@{LeQ~905e+CcpE%Zsgog8nV2|%^kqJ{sD9*}F4% z9&4EmP_ybensG`ecdNfL?#Qvr2)?rL49IioiTr{HFCv_ABYd7LT9(U-n?qOL_nWWe z?_<3|3UN`WdG%GcNfS+9Ng#qOTmCfkezH$RhC)X^I4v4?7SABJxZ=7aZr!d5?a#fI zdW>mz&`LK;i!&~jaCZbRe)47$K+oL}P7x*AM1;qF2P{uCXd;KV>GE1FB7ARVl6-@j zA8oumWQw$)FzcY$A^0uEZb5_$t++~l+^O-%6MU+cB0M;GAy5DQqtULJO}?j+XM4)W zLjOmzu5dKmQXKj^##JZ=aMl{*Qn>-Yg44{g6lRaF zKOdx6>-M@l1qp-quKTx)mL;JdOF<|2o|RDu%`qVKb`$KjM2p1l0`UF6Hs%n_YZpP+ z`C<*P=1+uhjrRPwC|~z!C>_nO6fw4~_5h^Soycoj7PtrpP}D6LFOSk`fdax4X1+I` zrB_u?)S}^OuajD z%wpKJ^RGP<41n*t%q{TI-P6#-$btq0*g}X>yjee@yZO9)=FC2|gAYyE42L#rdWX<7gt@eC59J!>Wu)~Lu{2ha6fE?6riO=O}MI> z4^&P5CN1nvt(fDu`IqhqMuRqMb6|Sma${J`mbEXhR#A$ngvB z2P%5_4chCZCxL)MyK8POxJ5AG;t{XrclBr)w5 zdpXR{fE6b0(Ldqky}={FIy&hXX50J6msuB?_8m+Ggy$AVk;~Cp@$IY)l(?mY-@!(b0UQT-iGiwY0>bRi9BIH?}?lcsmzV0oDBo+ zL2o0*kp3$t;3{6-=2Q*ied?ki3TQ07WWb~O@hAfxdxJ3Z^a9421%3|Cxzu4|53ayg zrwPZoLG}sYbPZ9b;yfgG`hH(GXIJ6XIn^@6(+-QF*`{3qzu$ zPn4M{8$Jg)fOowJ5B`;_Jt%NvOpJP4Vpgp6$LcC3QdqcFR@w3!Jwjif*azRAR+2Fw zd_k5T9N7I!BR6W)iE*UOR=a_&RXO(2ftE&FrY5ou-bw;-0Ohonp!XdZUgGp=iuHI& z2ky}-z*|zW#S;L%9sfY00_t-eLrf}29XSPU7yVtc$}g&2eIw?%^3Dpk9qjSJ?4ZBx z0o;5y9^HskVwLz{lcp7?lhB8BBcqyjjCJuX#uO69DI`fsrU<%bZUbUX%mzl60@3@K zmK7HO>hD+qBtW;Ye{=)eYu;)`$`005_aTKHu)TD^2IN-j&&zK~C!;DcNpr)})?*~K zP=?aH8k_?E=^BhCuQOg%iaa6LdOd$tjTH@Ap5hVFQoyZv7wJzQ_BS-lopIyh%?tMw zN^OZc5snz-j+B$Ji`D71=7=*~l%`&Rj%|^6LgI6xxt9VAoup-= zXmJ!XcZHG7ni31dcO>A6mPNNsTzk90t4<;Ggt^u>Ul>$3aju8w&Uy$}r9V1dOgLNX zd1*rbY4AVSQgV$u-%c$2SNTIGU6H2z|4muZ$G!Ivza?H~7Rz>TRr7ax*6FW@e@r95 z;}Nle-0#%7r6lnUeNH+U#@$J!7&t=?3!2;AgaRj9y658)O=YQo5JQW+QoSb%F$43}U?` z;v~muDZ&(V)+{>%*^U%^#M;jgDl?mX`fs6$mBfE(pP+jI3_Z%Ds8N*8WOJDjT*NZ> zsmj%gorVy*Jk9EHBd`-f6PerMH_tm<{R}D3-P74CvYM#sgzO;R_U^r2`MOR|>%N1? zZ(F274nD-q8Hv}YZJP_aYEExI=T)}Gcx*qcN2}KNYen8AnZSlC)*W$*vAyz-pRm(F ztFVj5kgHHRt?3}a&=$75OfvKpqx=QgHQnnUF{B~h+VcILXq%2XjE&ZAw;v57%()+J zk}q~FXYbTl+vBLb{B2sDXAqnQ5nLUFG~x-~BdnVp^Dm@!SYZdI;_lrTe;4)VLNJrZ z0%^OY?*8gtiN4EuzYc6Hghcd|xj%cGB~0Im{7&VH^hX7k;GK< z6g~lAiQLvZ6O9_}J7=HrD#ewUoZ}?98^59q^=D@SZ+ZS+q83k^TE)lFc35%z-W!SsIi8!| z)p~6FBd=1dI@@{*@4U*Vj;cpibi;UkY9$p<(0lDY%P2Bw6K+D159;)8im^;RuvZ8f zI_*#I?0Ax>+jSC~mA(AB>?xI&sk6mcjr-=hiS8e2!9fA{01JXJGCfUMP{^NETmzO< z?15E_h&Y&zxqmo^z79??n<>G*2F{?29`+CF=YTHYUA~mR$ablYMnWNxb}kZ)JTS&< z@Ns-xmDEgBl-Cq9*s^o-oy+(Tl9o9;U7T`{*cx}z#_+bRF?Y}oVmV`MeWu#t{(vd8 z2s}+>=<1-Y-&!)i216oiL7}7p40wkJx3GOV`TBZ$I_7rs@_zp871qrSJ%~}c6PM%{ zDc(apE#Sw>(%nF-*Di9^*TK)mB{mgBHQ^^dx`~V5?*CnSJ72N`N9IW$&ZY4G+3!a> zzmWUw-iXpaA4pw6+Epx(C5U>Ga!fn9{&{>m7unN;tX3#o-s`;v%R#*ygf>eCcAyTS zKGn6#DR?TcV*T4dgvf-Q%#zu0Rs&_M*~X_gwFdRMx4;f0j^A}9xlt{-L$4Fm#-nEH zo|E2xLeaAUE$qg0JplWCk5K(%^gRi$y8Tqh;9@inp-Jb#p+b_Y85Qi3Hn6cVr0v}y zM=NDwxl=4rI$^h<&TOCEcy$8}*YU62e3GG$I;=&$ZLpzdte|*hQU+WSV7U*_p45hr#`HK!1GE;} zsEc855`qQDO1%l7!@r7^72J-XH^r`a*vbkWPK4c-xA>)M3aCdp;B(lCS&z9ROVJZ+ zm^SeK1KqOa+Q^oM!tSqoeQ7FO5R{9G^YW?3K%Dl4g^BcOad_V}J-*(n2!r!jf#<8c#sf}9`5&jy~O*z zquHElKiH|<^?ah(!j7&p<+pcJckk<5A(^k43gapWv}hu%!L)MRDFzK%#KrRb2ABVc zttvb(@#c+XZbfL+^1_)%FW4Sr1f(SAc+L)_84hkcId5teNuxChdR=ieM&wC{$FJ(e zYp)>jC&2Dyq-jUknda1FEh{`k6`wlle0P%hSdH29Z6|{th;(HCS+OGhmRH$bX<>8Y zGiu8%!_$ln+TU~e)+ndG$H|pCAX$qLc$?ZZdAp-Aw3HP&oT9CHVh#}{3vs*_-WaheES8q#MCkB%7 zqK<@m(Ijbs*}K7m>o$t^S2#jLnPjoFFRcM0pHHg{xrM@8|IC9IoWguK!5g3Y!>mQ7 z$;t%_UX%NKd+J;-Q*U{KaUVz@GEY2gDq+%X zN`BD}m<^E`B+pl{ZB_s%-w5sUT7eH`l>Ovumfz>g1saaImGMc=*w4ewG*_PgxoWpy zfI3?-ctefP3q7{PNF_$N$~L3=VfC(p>p?GnnD=@^KD)(C&Nt^AgAIaR4;0IW{ov5` zal)1Q{#yT^XG7`HtpxckE2S=gdzn_&AKyiETvc_4FZap0e^`yQKw4HjPsTfB@4ZR` z0t9!&Y|^Z#CxRN#5C~?cW$+{XfFjqdSV zKl=}(Ve6`1{+f?lyKLBaXWTNlU69>$YVG0?DFs|@!a}O*gMUsIHw~jnChhDYyG4v@ z{o?;mI5DQ^LHZ$L;7pMU0AQ}_|IuD z{`!6cNca~Yk?6^$M2hlY#fbG`FXwX1sAlox(n~s^B@f78@^l^$!S)R7=*wx37Dz*5 zMo`5V(@ln;kXI1|^H+|c{>FPCnpL=;<3fRh!>Fv;KdJfGPf#_$uo2BjDxe|>42PtT zM4&t{CRZsAS3oo!Q$+-^Bpc$UL2HTu0t^TbZD->H{BzM6WsVmjbbM)#a9TTcA`U>=Lk3amr-N7} z922FVB7IE4IOv=@V3s+lq5Q4Z2JZl&P|!~H;Hi;}lE-3Ruj2%eKrJ?OcO<5>4Lz8d zm>G$IX7KH6_hEl`r}pLV=;iC^?BVD`OtNy|>fq{X?@8AlSkiKEd=7hiKDd}hMH&a0 z8Xr@MeFIKpOq3UssKchD1u?}WR<1V2m^(mtNtvzEryJq%iuZupM5NB3KLmnr1b8Cy zPaK|Vhmo{Ax1Pj?G4*Ig>E*AQq)O|JgE01>475lftyj?Fjp8NAmX64;q@4wsp_`tsl;>rdTZ%rNsbryPqNy;edDK^FH$*)$!6D%> zCk$FK;|0K1RcWw)|3?`uo1feLVao$nc``!eXTo_qpbap6Hsq9V9IYH5^wLI3`LH3?@A)N>jvsx?B-$(^AhLMjl&3)k69vRc1^2w7#NiwdlOCEOtm1~T*+Z(;(TRp2Ub@o$59FhnF_-7Gn-O5-yR`F!b52`m!lTC*p~ zdL_^#AVuSAw9T##*LY@5FkS;^vC>to^5nzS{C0-g08fKRtP6t>nZ+ilGXWe_(u$AdD1wxLq@(5@as=rcj(0#-7K|#|=#U zSanG32LSx=0o#B>=>j+?2?seh&Ivf51@g z1K%Xh{u07(w40{UKLumWR!~C+tkrT^fEu?ti{h1cfi)CKl~|1{9aMgt+|z$ZA<#~mX$k^V9%%t~1EMa;pw3Y?ipS5ea$0c+2J>7A zo0B|gj_q}k!6AQ^y&4JY1Cq~r0N$m@9c)9B@d*+Mum>vXjx{&HCf0C*qc#wza`_z? zu?!~(2yYV6o5U%8jSh=1OH~ZC7>`187x)Cgh(Qdq`fq^Ey;EscBL-{N)}hV%I9_?k zAxbf2csw8`@ruISENMGvMS&x(|BzJla8zh;ZqL{D<)Iv#Cv226Vm9f0q{ol|>`Nzd ziRn4oS(n<_7`S*I0JkTQv{>|{EoR3ic=%_YKRQlQwk`B0-d}@;U(b6Dv-ci|2bgZg zC&htRX(@T~B}=|tbKFH5btMOG&;iDNG)BCr?_VDto-O`XlnSEtAK$FtZq+Q`qBIa~6wB&lEg(>3-z;Llq*Ti+0musHr1upqZnaHP~(zqGUVjpu#+w<=|Ur^lZTSZTetC`=h02LV3V=~{G`Dq3kI{(XW%>ZJ7g^gf@dZgg8#>)sc3{2h zInhAQ59nP>{m^0-_{IH9Vx4+%aPfO{515yR z-XDOQWkRPwNx@zQnWEIZYa}@WH zQ@+7)o(Zlaj!Yxb2C9RHs1GA-3lR_nHYOuzB+#cfoI?7h=mz;f9G|kzfDlPAtZ#)g zOw1Do2-<6(k-4)XZ9dH_cI`%0Q;cC4V3BLxj$Y=mHiqB{0)XDj!6(?FsbHF1E}s`@ zMQN@HMt|y8rf2E&o0cJI5`uMyA>Ng{MrPG|Wx(ZvCSBEzR-zjCN?UckpssI^A`q#fdUo zkmo^`OZ8#T%lXCue`fBxM%8-UCiwWi1YGTT(+R@F5D$i3e116?jIPaRD zR#rud5?SBT^w>s@PjC0Vv}-Aj=x*_w;&z66qx1mnq>TULzT&Qqkn)DX_O@hsA{^c1 zd{CPb3A@|H4Cs@~bFn`^nsfNb)6wni-% zJ?PvQ{u@KLayiuZMNa)td39OXd*@_04u4^6Da-8G8HA8>wMn3WSYV20Bqt`$G#2bttme@x!H8_}MDn`fiK_N8#ls(hkC_pxgVx7?}Cm!V;5E$Ge(dlw|RWK6riPReU80!Eu z+OYu(hh^`(|vY~)WYzPh6;gCg|>S-hfScoiA#WM6t?hc?-n6AltY4(O76vQRh z%nC|JvN+XS8$?CHvQGr?<^D(uC;@z*_cQ}yx6ruMT)u%pW88QIlucXCMmvBBeStAE z@SNQTc4;i6?we9+M4-u$c1AUYx{4Ghe)`+7De;F%UBe5=3@{bi!h5 z*|D5CeNT~(8bQdIAE);>(v!T>+qmPyyO-j1DAp-qFmdyl_>gAwg6dO01I#~paOs76 zS%u@LoR75Et;XCzKbB19iO~XVa;)XH!#q@{fo(@gjt^rIZdFjDzY$UUw!s3fnED(o z`vz0tXi}Jpqgk-!2u_f!%up08hKq6J&|MX8Kim7Ll2!U~0)|{oj+;s8wz1qg*>cbI z%W`XLL+)l)v${Wlt%R?Eo%-jUJ7zjg`<0E2$M0Fsl{tpFxd1Z)Rag^$y zn6YPDJRx7))z3f*%K@E2uh=pwlVaRl&!Q`{^LlUxKNt72$Qv-R?V3$pk3jm~*Rh@_@YcKqTWp;P2hpCaq#yF` z1|Xpe->5a>z6I^>lmU6f#8+fkK%fRP^7aH?28Jid&Kp4dW$sD+bm#5b1=55Z(*Ca~Lb za$KRrVqK$?agx>w_0b_otj%#B6S5Ps2-sWYVZWG2qgp+icAWUiEH>Ok9AMwc#qt@O z*MHhb)E$gXt0PL_hKB>!Qf=*Uhf9ib3V~rO0AZoBPgpCkwMZ|1bOg26beWm2$NL{c zAssh~HZTJxEbCCit7XHm_j78o=<*=NodBXASr&~frz5ba(vg0za%ARE^i#6OL zvvqjxPip%atIC2uEpwTgIeJq2J$d=~I_eoAk~r`zImD}cUYQ65BeI+l5c`7;orD2I zjWTHlnnH)g14|wLEz4kZHGi$vKzdP!f(Ak!`)QwoIQff)!~NHmh@zv$W!;uf(fp>u z=}b_68L)JFuja-rD&3Mb1@sJmLw?QGbE0(_H9zBpx8Z7_uwHw=Map&0)|M^=smlQU z`r27O-6Oq*ol`MRgmq=d0lk-V2v|e8asMgAT5fvvM%ZUZuLolUMN95#qCZ5ytq?p4 zgTj%Ta)5k7aJyaRCLeslmvJ?Zk|N!!N(EJWSt$EuFjFe`byS^9Q|rZ0p+y}21$Hac zt_k;T7fz(3Oim-+%VC)#F7mDxW9NkNuGV0D;N45r{Q#+%Uu`+AWw9OlM%Rys7Eyih zcVeY#2^EM4bHmRTy9LOUIn2;VDt-WADd+Nfh)V=?M~OJ+)Os8{FZyh|SNlHs(nJ|h z8m7eC3x=cO(w4`NG_JyPIlW_So?L~$6#C+0LBEz) z#FTb|R#--qkL}Yk=JPnxPbI{6UnDUuO10>`bhGT=T^p;y9$c^VVzpjhcp=o-iA50n z%~=3#AHl?lE=k$S7G~}}T34v=@RmEzma83S?3+C-?%ry8%w>w(8`y3j7}Q{!A(LfT zr0ERYht`0h6#tzLHqNy$ghE+mCXpR;({XeH`D#6nmq5Gj%s#JvU&`N@<%}Gu%ReE1 zEm~XKio8B?+Dz@K*7)?n2|RE5{ci|6S-M7bfR;YEz8nY7&!~A)>@UhgtVOqLN8HSa zm<4G90Lz%W)5|nsrk2_h8HS`;H=iRGaB}zcOK9FiMaD`kuXG`atwn! zNA)*(6VxiwM>zdgla~XWw9D6=-&RTrCshE$2d!P~^aKQc^mTb*%4+ZS@w1C|I7$W1 zeFx5N&XkhE&2e|!8fWk}D${qEB%(8!^+$72R4C4Ao?t%!Aex{Yw}5Zw{9U9pcqJ*> zp9E+jrvgJk@WO?uj=>ya?eWzF(%%FlgMgPui=|-gmFGY&QW#a8|0FmTN+71%S{sB< zM|w|K#H6NEdXQPHa-d#Iz<5-V@vg6DzQ79t>1_;o!T0C$>(Vb1h#i;-ll0`IabBr- z*6$S5yo#WVlbZTwaEPQJrERVC+cBs%aG zpBE(uou>ha5X1js^*rEAeEGME$Vk8ca@3U zRu*idt`)JIvx!qY>2BB{DK`0Va-qU-C<6N{(vwK)9_d*XPbV8_mIxnnt}1~qYZbOZ z7h`~Ne8!1KiJMn+(3R#JvLxL|Qv{6Q=itudc;Ivd2AXpzQP5R~T_jUy`g zoHl>;pk0KqOFtqI&=Nh_((K2`8^FgF97n1*JTCfDqiEHM9#^{COsBF})4Ma&%p7Jb zig7hR)F=+zpYxibu2J0H29?*jhix~)T&nOk94{1&;fllQqVJLyO$3JbXi^lts1i!V z6O))5Tv^RZ=05xC3N9ySw>OKiLau0cIH$Z?aqX%`;eO6ZYqIRXT|?HV42p%=L!IWR z4EgPtYE&T|`h~+wEL4!r86Ph~vd+bM7}i0YT%lLob4db6&)y1g&lm)4eL^zX3+%yU zXc6e}3?i0;Be;XC#Ir zfn^UPH4H*kr{5sC4C~2Wx}oT5SoWtzaV?+E+P$|TK8!v32&E^l#XhAVAyG&{z=CMd zQALfV%5?;sS}Bz#4V+PPy|+%fj-(T4f*3>jjZyk`N3V5#(O4A2jTh^3Ha1%XWNDnk}_qI8wg}MV4xWy>fob1_Ms-)e8{9L(w;;npE}QgxCNzIh?_B0h`l=TDO+yvKG617P>rP^yWVw@eywgU ze7D+T-n@PzL*Uf@#1w*1&WA9;sSjaFAB>)Z!~QV)dX?5naR-g%YuQgCGpWa^d*>AWyl}QuaWpjiG<`U( zez%N#8T|fwPa@~1$>sS*6RE1pj38zO@U(~78K;%h!S+A?4707r@dGM?m<_yXBD|7^ zFDWL0C_}lA@xJGMC!`J|GoM^HjSTDt`Yzj-WM_JsZB9afa4(?c5bulZYGDR&qY69xl^QGAu7|$O2#ZSjLNxrZ(y_m2Z$r zKxWIv-S9x>TqK^<5(5O}j)FVkzI2eHXJ;kO%6SG|R|U zA+`YDp_nTb;Qa+^ugVqS9x7mqTX}?lgh_VpyHV*Gu0|Xfx*oDNK$^I*`iZ|8e$*r; z58tB*N-po3sEcgyrtaO`>O+^|SZ7gNc2F$gjg&~lB9*KJ{fw>|tzGLaiWqc8;`GpU zyAaYN9;u?1%f9yet{YwegGu*FB)Zu%d-|;Vv-5t7TpMU1qj_Y5CX>Q~f|XgY8+cZE z7jH!UI?Fm|gwriBjqpbzztA{rq8}DIKpNme0~ikTv+GX(7hm&!GRoy~H{o@^EdB88^8{J?fu{aBm8Ca$X$fbec83Zs0=Y zqE^wLxe+qpM6)i}tR;I6rt$H?&=CX^61?Qs8rq*rbefY5mS352_pR8W_g#Ke^FY+x9YFz{tI5?p)tT{xN zGR)UQ5dB&)NL~gZJMq$dzy?QNP&u9#C%_TzD zqARL0(LX@oe?Qk#4Fa1~2jS^Qn>UIXhDMo75o-FnTi| zMTs#3Q=kn&e3h9w1<&viYXi%$05sWrXujPo3GjzOGR1&>dT4gBPc0`0Tt=@Z;A0DR z56K5RP=?3I)H7}xAsaT)=<286w0hIPoyQpcY-HxzG$cHhGA~3o*HnmMh#--Ouc?L_ zqbt>1+9mL#HUc~z$5X72Ob+YGcTE_u(r$q`Jr0EPhL;SmraVqfl#5uEkO{`CFnVP* z#`t+sb6=yWkNc>s0Feh2_b+fJpJf<;p`Q6{%^eJ|Qz3hx3Y>^qcN31Y=cLsYU+@fw zp(34ndI$Ro)DHR}0|2l)ASZ59%fcKD?U^kk>81u_B!Q!4dfW z9c3^o5Ma$;V*X)mRvd)1{>!O=qG(PTjOnUPN_xMk>vl>zWso*BSor@p(kO)D&z zWpp)%S^VEucdE?B>m$8b-n+z}58=MTe}B>;B|Sz52FMFur6vh7Zbkuh z`~UzfP@AMj6knfiA$@|CWO)PFNRb&R37MP^OMq(ERJWNJ^!aS`7Iu}$Fh}1aquSN7 zM60SwW6CG(fKqF%IobWpBmS<@!^3%o7o|O+ueN*m!Bj48CiD&9edM+VrrvT1jGshN z!2$$(fv+H%=~MZFotNtPI4A3Z3`{fKAOvjfl3cpGWF3=eBpbf&*Wpsp@fvlU+Se|38s%BLzkK z2E;zfWf$;1o{}uk`seoLan|ZW*0LAt5TJ+fbCqFq zggngRjj1pcQq+9u)zMeGU(xp+AL7D7x^zwegA|mqkt=OGHUQR6q^5!rqF)|XS^t52 zF5(8v{LWrr!th(AfN|D(K;%`tXpXmhZTZQUl3|2n4b|5jit4MbwJlSnvxPP*IA z)j2X6u?0sPp!>K9r8%&dnLlN^h1tN;^m!u|IWU`PHe(xUUD{e~Pe2?0+idGC+xCLG z?ppkzV${(er@}DoQE&6$+(;Zw$Io-m%^eZ_Na8{IkrY_It37qLONV3vak2{`$Tdeq z!T7z2?5VB6a+m>CcQ~nMq2C&u{WweSh}N?;kdRuHv)GUEnX7NnPact}YEHQJ?n6{l#GCM$TSY@;hJnaYr}* z=aVzSmqSFkQhj-QBw57vFaP z<_0c;J#8nz@+)*TO2-!}lc4*Az*=OBG`z!!k>EaCBE)>tKd%OWBd~tESFuw5r&6Iy z3HL3qm#`pib~rAa)~Q`W`kNY$yaLx-ZPqZ97wC?!HxLFEpA5kBMCrxp1sKl5wMMcD zW)-&sm~^C{9bk4y?d6c}i0y@8aCb@WuI8(&LRYJ5fU7X77*+$lPI`S7z=5pTAXdYd zL=Msk19sl<6lQU^$U~D+p)K*YoXxG?py(@ijqf|S)6RFlpO3zMIp20Sn5c2yLjulV zx=y2vDT{Siyd_Jfgg1nlxcfdFJd2=JU;@fUYmD757q;vR71=IekH`jOZ7Yj1mnY#< zQ10`5snK(Ohpzpuk6>U|LkjPndpO~7cCkoM!^og{R$#(ine92=!|0*7jkw5^0@kf4 zmP@C_crNqpH*wW_o}VX*D_>WQ4MAm(`8JsEG(vg1dJhr3ZMpKN9MyTT<|tbG4y3s` zptY*cPuPQHhd?Xt&wCY1nxN151j(2_3VC<16b4rD+!5B|^>6#FUHpUw(x9!Qx7Tkr zQyLL`s8g5Uo9Pz5!Gc<`NxRl}V6G$)CypVnOgF_rTv8m3vHSWth4q8&$ z_#L<7!^gfZ-`sp6f6SLU7B_K3SI1bzUTyJihlkbzP&A*LBvpZ^m^m5rsdrr5IT_Yd zQ!iMOc{a~{2YcD|VfW9YxOCk^+1!0l(FLOg1y z6qfF@ek5qNXtp~m{NI%MDH)4MFe8CzqNk}Witi22{RkTj0%;c^m@F$qf;}mN7H!(g zJCkLmgs`B+zi6W(k75sGtg+>__nxOt>{{Y7C&Urt;{&oNFgpER3DxcpA06|>z2NXXH9`4l1%6jwQO=V zMEmWFlgtToO4<@PM|4!DbQa-Fl%?5-;DwJ=ciX0h?mKnx`nsG(Him1>(bd{letg$V zttl56lVptllnrkR^1$*#2SDH1d7HSKodJ}uv8c}f5d@i}qnO;stT;-R3v6^ChczBBf5ys#ZRyix?D$UV5Bo4H*BM|fP=)I z30f?M5v2fB3Z>KBwB{}ws#EKUhSpBZ@gzD46;sYZk_p+?kXv4lETWzBu#g@ae0$io zi!>3Ai^cqdetI?A9_u77h4-v9wEuRPa(m`m@G&8$xkppcmoO&HgWw^G8TfP%))rsW zpfiJ@#Y4LW1n1>-RvmPSom;gk(A32OKQ0mH&v(_(Xo#;|5lm-knxeCg>pF6cUg?TX z=YPqwC}#FL3JN!SqZUAyvXtA*BHSznJz#HA(d+hx&THOsVn{B)ZvbhyK==#cFsqbu}RI7 z(c3K3@({)2X22>j;Ro$8VEAacfcx6kcHJ4@51C7#A=^GozyvsfK9H&iAFaR?%`U!D z)o(Q0{T%#E>WxNQctT|E@%EISHZtT%>{t#`5K<3^G}QAamp3Jyq+*F6FbI#;wbr?VhppS5D?bLt+U~!|W~+qe5cMuu^ky zhTj*F#}(##)U!U}%Q4n%salT&RpYz~%sR!r#g+7J3pFC<2MyL1?pR}Zld1D-azkcI zu{m>!i?p&1T8O^?J)1)07|og@1_g^89d2e(1?iT%X0t-z(6*(X3GAp7=Y{+=EQ=WI-}!$H|un5KV;Lwo_n+<&oz-z3-KCC}?yn)H2SO-#y>3<=k-ZO|yuYFqg=serdY4jwIA|UAlzM&S_0QQ+>;=WC)d}RL{*!FLw_~-d{w3y(a?=1e;cMNd1UlDGFX$v17JaO86Sf8aTX_1{){h@o!a`F|^aRA{(R&Ea^Xof~Jumd@ z-ndZgLIZ9HF*;S(p=SHQKfO;rcHU((XwGIteS`c4k`>^C>>qcoO_mYlE-lrz@llMX ziNNpJ!^1Dz*I*0ycUcWrMr=4aY7A(dcTvs@R~v5P(Xk<_rukKommoWknxL&@oU{^2 z1a1lHlsd}EFl;eG+;<#A-cgW=_3+H9XaOoJHwHAtT!$}S8W1nI43z^)D6$Q}wJO!G z2UBT0MWlX%8QjxD9;spLIF;DQt(piQm619e(GJ9>pr4pTJXpiZB`nWgn7rgc!J=Ky zg_x*5*7~;#FKbE69Y)$A@lpz6tx5DJK=)(UGjP-xwv;Udn%$PVzP6-46F!PUc75o4DsGgpMt%b9m z-v7X8Wivc9|Fix#jMgVn&?$=nA>{Te6898==WtKf>%4dgn6%kIsDM?LiI1^Sy`D?G zFp=VI*DOSbJj1yXn&$TWd=nKNq2eZAw5iE{h=!U58$DhE&mDaSv^cPcMO~1ddRgBM z!eESpS9THzPaeB~q#J%G;1;GSi6zUgFgWPKU=EHTC1i&{6au)#9MS%0T>}BBeZ#Fr zKam(_@TSyAmM|DFODT9mQ_NQEC${pV1U@~--)+@(ZR(f-1~5wixFpNOizUM(Q^Y}a zSPg{x@2yGHsQ>70&;0Vq3qUn7+H9-HdLv*m*s3|n4AmHyxj~3En!d6Pcxi$nl|ePy z=1woxvL4gLYe|6E2+8yyP@*B2-XIYrTJn8*wcX<BKLg3 z&c6`gB7&|cWz9PD)Ya3mj8eeqLV%0Jhw z?9$)&O`UGaNggS7Sx)r14=O+J)wz+T++&R^lGkiR06WYX!jFv6Mnt5;^Iam>-!Hbl z7k^dp-lpGxmcmTW=oZxtzph`suA7p*%E4Phltr~oz47Y_GE4Tjy|I&_2G=T@^P8ze zeXZ7ARhUsLjG)G=I7*ZAIdb||u;0EZQ7s%B3wYd*Lmq*1q_3%+RW*G2(;ejhwNS02 zh86Pv6=@(O0D$=aU!mIBn>ZRc+c`S@Un(_4RlzPx0>SrGZ8`ybGf}G2Yq~^bdw5OQ zp&pAHT}8zt#vod4q|4ZibF=be*A5LLAXJi(-aX^J_x)s3;|^21;;US=v&B9J4IK^M z2*L!%-E0SQT10wL9w1p>y^CN%_rMs1l zuY+&z?kzA)2$*EqsYZ(Ntl9nwi>CB${W{aPuBkDMxhGBV3EP@@e|F1oyJ-mKji5!p zGV)lpqvoroI^hx{MoWX&@-R&VgKCtE0k;@0<)QB;q&2GK9XT`vI-hqo&EQBnC*I&O z|KleqkMzwA*#c8GoxU%)+T$6QcIaccn%@w~2N1h~GcTB7Ryz!?ZJCf zm5atgcfC=|PB#R~Ke|=1SvuA0nTgj(JddT>_hZ!}_dYAasqrdU?z9<`?JCJyHDoD3~!{|Z4( zWgbpP4|uSLvf820C$76@+1mxsVl}I(odc)bGT=vd+SpT20ADS{<;jFI3HSZ^y6{y= z?E7z|ol}!wQI~Dgwr$(CZQHi(%u3s~RcYI{ZQJbn?n_5MbliuIIR9WrtQC8oImciO zSC`GDT#GO)^1liy%2)grVrcye9OddG(Y(e`c=?USZ#KIRgePQed^6&@to7NYf9DoX zNFH^%h0s7qH6Dl{-4oUfw6<4WSj$e-`gpmr zd8X?rI8?DNk;nYURPYxn92*M+Z8iSN9I|V`cc|iS8q~Tg$E-Xz4$L^mSo_;_Q^QFg ze16(Lsm$WRZh=8;Do++u>RasJq#?XM=en-%5UPlu=KV4BEQzeOFiAVHYW>7d`;E8r z`rXul;)(Rxl-6U~AKlALZk79N)pNDaqj^`Ye`hLvmy&pM8{~OB^Zr$#D>q;beL2mD`X`SX|bg1)V_;jR`qtTO7z0tAcqtNH|ql;Q&n(cjl?#bl9jCoGu9DAR|V ze|Dly{0GcVrFX%o?Hp5PyZ4EQQAq7E60B_4nKkZ&0G4{g(5(Qo7q~T+e?i#y9LP`X zCPb-*-NG~*5g_4SR-E5h2yoTs%c)N&em&@{;U?t1r` zXmCOHHFNlamDC~@@pl}5TVc+T8t1o}jjx9TquyQdElw=T?oJ75Lg#xsM^^pXlivJB zxvwIbevHz6Z9*12XI}#zaebs7F?i)3b1CMY#Q<-$ggS59>iAAbW-IGg9pFNnkiS~h z(3?;y$C$b7D18c!<6(=a+SOrK&o0dxvIvKZ2QRaiajjR2PbX#_UAj~1S07=zy7cBz zo5RmY{@sR)$L23?f3syV4ASRdcuq3E#K}QJ5!g`bPz5FfYp8IUI7y7Ck$BR!P<_B{ z%J2lFdEh1m%?IX0v0UtfDJsT7j#p}MaH+6hTC_}Q_DAyDP81DArQ|4V*PF#0*QP{_ z;*JvSDaUgDku&stJ=oo$okzi&+?7bBRJ2Uat4M4#S4`2bdXm&?`_aKD@qB2aWCe(6oqHA#a9u3>YMohD=jBWktj$B4X*m zVfY94=X|Zp!cB~{i_dVK;V|l1vE2^K@H>@7XoO_o2CLdJsOCD9iz!36?1th zX*$AVuQhO#TE~ckEnI_C4XsK-Uh+OXTq$h|!=+7a>$*m7=${U>s6-#6e@aIkf(_wD zK~I$%|8oH$(^ZW06O*K$-gE)YUE{9MbWp)-Xfd1~Vzor8{DDPTa`YGqDYNt)LD#v3 z@#}8eT~z+ee}2>lT{m?X(xTFfTHi7_YNtJ{xUozy(3Qnovg&rcU{wGE9&aUBWscX_ zJ<+1pZ8RNcUQ71D&lwAbvoEcdak{N=j3HJ8#d07`PXjc8^A}2q>$)ZmjW09|6%h|& zm^Tu!zDn7IdWQO^d*4?DCiPcrw#z@dkn)_0RT}Fez`EvO>mxnEZ&%YuxGrd^*BsRb zJSQU?oOQm!0GaGpuDn5@K)S2dx>)7H_w#=@_P{fgGT(}|`UDJ6y$2hOWL=T*M~3I& z6&o04Gq;N%_)amM*g9#1SU3?)U-7uN_ptJ*mJ$X-_rBO{TbOBY32NZ&`2!3|pYjB7 z&3m*1hPMgixEPb|+Is93#D;2xvgF}VvKV0HE;R#M>1rIw zup`Jr_0(;UqxI0Fu7h0m88eC)p`^FM$oAVL^h4?t@0SAf zRdU)i3W+G=SET57%R=rahkdMNqJuB-vRp&wP*ifeS}FB9Q5838XEJtc`C$i#)$Ec* z@Yx#K!MLaZJbSf2$gZP++8>sqTx~&DTg?H*C0~CcI*7J6KI=dyTM!5;Txa-0c4iBV z5T~b0W75uix)Zy_t)L8AWLVXgYD>2BAXEIO0XKS7a2R%DPEpGfEKIjuxz2k|nuIVx zc@331tZkEv(PQgvJ#K_&cUoHD6rMCCOJF4cm;eiPLEiOA@yVzW{mG27zSepVt&GCQ zrHGc$$dkVQ>oDZ9N}2kX`%!208V=b9zli$#=Q7TX+&WTuZp|vw@bYZfYsAIDg_b`8 z*DHA|<*Iz-WCXys@_nEe{<%MFhg6*1Uu~xw+Ktty#eA}n7-g4OO{Ph zb^}JrT)_{t_r#4ep&8`Pb+h(Gzo)CSQ<@2grQ)J04VrGL%g5cv!^O$#3Fsd%BX2@< zhox|azb*&<#kAxMH#_ohG52kM@9E72#P-X{=Oqn3Z)S)d9G@RAI~x~2ClVfi zc>hmQ>!|rCzTDnw+`-4M@9dJBoAwwQvkKE%Hpn)Evz%*4J z+M^NFAywVVx+-05SSSmgbhF!W>W)2s%Td86A6aO!xg5B#xm{naHFR# z?J$k?V~fU#P>*7-IM8p-aTMEO(4#A-ZI#u0d16(yU6f5h=Fq`~xAK-VsQ@cb<>lV3 z`Il8DD{P;R`S_`;9)CGfS$HB2M8|ZR=lX|fSsuElr)o0Rj%Bk3{iGQQ-0#Ot_zn(Y zC=gh{M58Dpu?^0$t7=?;Q=}#9mYmNBf4Wld7TD})u(clg#0{!b*@{x+MtZ-jmnj2*w{(y zs}ELIyA40^XXxr!9bPOq<1)o1!#QD9c=UCwj7`g)0MA7h z79{(9LF@1fMPr0KwKlTql~_Z{v>*>#uM~Cfj`R~A?tj_p^eVS`s%;i^yb_G+n0eT zKpAdnNhVoXJQ9C_cLV$ba!6jkiz}*BOakQNbUXOK=A8D zQW~Oc=+T>z?2vJ?N&taW&a7}wMC1I4dN#OWcGOmhiuNO7Uv?k`5W7L8#j?9$qedf% z-MH+~(*Ntx)uj?jPcQf)c~11F2_A-3`{LxK;{Lp&Hc$#Ap(`ySlbpQ}0#KgQ9R` z`D-SNsrBI^RI3fNxV$!jGfIRV0g%wCt70OQQ2M-0s|0`*UnaVJQ-2fF6*=d5Xvp*V z&dgvv^^R>PoU$og&f>r!PB5B-NENvvoT}V9%<4&f`5WA{Oa2X}zSQG~wrMmqYzd@3 z0fX9*-2bxjA6b2;*2HMZXc@=}u$_iX_2#*ocpJR>&HCHUBBEQ}pI=#Sw--U&!hheqNZvZGZhZZD!MIfRX$d64 zzJ*uh>SpVkBS8wT@L&3umr)^Oe#(;E@%=u|&usH{ByeH%as1v~oZO!N-m;1DOxeoT zWcLPiz)tjlD#58Fr1~epY9)K?cEo-SYvO{W8((2St0l|2aba4*gx^ z$^&)^@a-u+S>S57OM|upcklk3oA^1FuGw||+tpsod)ELI#yV*jNQb5VbYRA$D+Nn9 zefwCEoV&%su+jj?3hd<Ot>rCW3#DLMYk!eUWlLNjX#8~0nu5(RAd zr;$R9MiE{lJSk=J6Pl@DY!Ea`cKA)@x%|gqbYmAGbDSNZC@RHgi&4qEDrQGBJ3<6_6R0dWz`deIBc^bvtP@gJ@J|2u zt{3tAr-M_7|If$y;HdnqH<5jrFg!(>NoigKv0fz2 z0u3`$H_{r##Ds7gF5SV0X@}P!WZh&d?_t#y*qa%YA?D&>)Ns=>lafQ474W0>EV(jYjQfVbLnQ0UhxE)(LX0~XlE@5CRo$`#wCyF`WizxYl z;!3o?s=$-G1n&U1~j9N`wrcw&e6KRV;CPX+fBP2q1|8n#?>|&sS&`-l_ zN3c&7?O_Uy&09$E)1d03<3tD!VQ@XU`fV78ki`h@K2c<35LqbA9S;EJ7I2+=Z(1n( z+vK}Pc@WKf9H&Gf;@Hj(j_S#B9wUKek&u8S<{K?0UX)Y~Spe{!OSUmqv8OOCY<$&| zuNJbIJAHxKwA-P8CSR=Eq#~Ur8|CzLZVJT7k>ovJdvYg>=`qB;|#QV|!-6Cd?em(z_V>N631Ftg z{#I)_QyZ%8vftnKINhd3-H&xXtVN%-jw`@$jliVj#2xy;%0CEim=l}h>vF|eb?Y2j? zwd@oaAtHtP@TdPA=dUcpNp~}_w&21Ih1wfi{=p4Ss#D4O+g4@=5DB`LimQ|SxIW@O zc>FAi*5}fiXkiuAsxn@Iywv0>j`-;i{>@&B8x)3iW z`dCdV&=8q-*#&4qk0%!E3ew85)YiaG5Y;}8pQ*|=UBZZt&G)m1=SjVt5N)w=AWfPD zvUkKa&6=}zxFT_f;n^Wk!-Rb?WV>n+o%$^TyHy?F+kEX+o9`PNz{4)`lMVRuSLn3{ zCJwoTM+PbIPaHj2l$BRwFMZssPOYSxjJXutifwZ>*)Wh;eh1F0q4smhY@*#Vj-gg|LvE5c4TI+BVKMa=6GJ$t^ zXhvxukir>2O>ZjroIL)7dyeq>)?<~$toH_{z`_Xe9@FkFM|Cy7Zgt@PGSYc4FvSNM z(NX9Aw#rkCz8ZaB?O@1F%oUqHYXnjn{hjbhOOKxPb=2+frEOF1AT~bZg&7MHk?4l}Pzb<%SIcc!1QB z{F5DZddrW*gajlq96+g!g`nXZdPgJJ97ikx)p^`k+^9APk6FR~ zRpD&+DW+QLFlP9ww&}!3lE}$XMBNB2;5$;h3lx_@?n2#C;L~K?pun;EBi8yjsb%F( zDWzQP*FO8aX_az}#!AN$>s#n2DJNJ33a$G*#+5CuI({TScvS!^*h>F(A!CkE54XST zz3N8j#>TLZi`?Ui#68`KpaBS#O3VF}qW-8!`_-nkaR&vQ;2nfE_u2O+;z6I!<) zD838$Ir|%u#*NG#L(N7~_H0m-Bj!`5O>L$-7E>d{KWm5R#4y~zbM~ShN)lhD_DpK&*3DwB@%SD2-Ua+%xBuu4d2x3B+8KKEKJT*;^$q(@!=K(#ZAikyrILN%GciiwaV&N0jx#w>loxyEiZv zPT3;>E?Zc=XyPzRQSFOZcN@*9p!t?DN+Wxs9(+C3OBdRQj`T8tY0CFHHryoB*b$r4 z==y?~m?>wLQPzMl4uXO@DAYIjxe4X7bcL{wn(gdy+nC6&q z;(3wSwx*IaGRxp1;%@nBWw!0~TI7xT?7!e=0%=OjDCbfYJO5($4<)!+$TYA4q`NhY zd<$eqjbR*bj^~nrc8Y)eOeFgi1S3CKN9*lw^IJT}Cp@9LRuYqsL$jGyN#U9zZ%Q|` z*bJU8)Wn%*;WqKI!Zd>HIdEkJ$&RRDY?-(ed#bBXBvzN3hAwJpCQ%F=B4mEeNbIKn zlGZcziDu@TQ(VEmd`I;37oPfN3#<+F?YheP+veKv1EAUbF5#q618+A!V{sj=en1PRNaNoV$dBj#t@h0%(#k;O* zY@KIOWKzO*@aAiGJ7&j;#-P4EXGXd*X;p)23ihKWj+l6ItLcD)hsjpfi+$?4-nIo@ za3OhL!cj@R)|2MsEy5ONp{!LohhaoVf8P(5x>x?n=%Ed(Gc1?6ATnme&|0z7#$`B+Te}avf2n21Cyx_WkvfblXv=&rDEZ@$SsQL zZaGEeh^nv4&}+4>cj4>H+tbzC(Va{@{I|iHGCBLgB=`z#^zB{yn_idXPI*lWQp5^g z1W3*Xc|SEk|0T4ol5i%?xbU#9>TZ5}5X~l5VmihWGTLT}W%mN{r9!NPS6EC#u(!E5 zLeF;Jz26WoyAvxXCr=6s9yET0@~Q%%t-|zQdjmbt3^K0}$Tw9T{prB^;5L0!=t2!-_{ukCs4Zr&h zE^9l-!RT3i5Yg+IdL0k1Jv_Ov4=6^^5SZ@T$toX!X^|OiAPh&QV5KYy0Di?NpC;&m z@Alox(C$l@HyIPH!Hxa(_z#&2p3aD#6cBfvnTO9<;eZ=J3T}@GJ7~w1E4zV6Sn~_M z99O!jTFrXc>l+}1UF1Mf9@h@Cs|w?5f!na>YCay(n}8)ZGaO>PY)AjEROy5dCZzIG z_rxzd8%Tcmv0-jNM?yvDu}rUcfws71?Bf$eki8EnL0y@p16atN6_QxwpRg-QKhnBJ z^;sX2`^np3Y7|Xi8lBLdkAJMQ$ozPnuQv&q95)0Tt;p1ORGD|$Ns|=D`l6G8RHUYlSO!WrL4EBA&WKv z%ey*)1Nox_Yxgk+lyulKOT%73P=;@1y4ptV4EA8(BBqkj8;%>N?j|@SoC_Fp#Pkf+ zpUZJ}Ie6{xw2D57iyX*xHYt}YQ8y;yvv@GL*`&r^wFH=<{7oQDge*ps4x<{o@#H&v z3v~L@$*%5NGcd0rG(yu%U$imgXJ3+wiy5{vHTQ{8MO4UYq({J&zx7MqS%=|h5iN6e zG|DQlKD0A{tCqr;@7;^T`@4tSbiW9RCRxkbm>{;zIqCaW4{@=aBcgQ&ekNBKQx@HH zu5U-da~TeI!GNTn8UYPKme9y)N3dy|IEv8n9oKDb>l;hQ?en%t98*%resB3cpwEDE zmS2|9qyvs>lqX=Cn?SEF{Ryar9nKu7=VE`(MTvDyr;3i{%HU9SvZ!edTcbHAmU7-f zE-_6`MYo&y*daKwT?2$K7%HE2&KbEi*C!%?0k~?fhoYNM{VSNj=FoaO8rL4_Y7^T8 zLD1BQWqpErCqnHQOvpRncDn#qFh?}AhHOmSo*3L=lONG6izu}4S?L)#w8dK*%)v+_ z3wgkE=C{V#g_Ibve+|>uia34Jd^kA0fHv?1mp6kIxMQ&cc-h-oW=b`ENgd&Lxg36& z)d<$hXpbhMf^k1(ns3XNfu@)JvEN`b8_I5^~KBib)o2T(TMZj$q<|2Uby{n(J<##6D<1W$iuWTD@Yd@fb>tug_Z=Bu=vAkhurd@a&+uM){raVCYY9rKgU!B|Vw5 zbQ#nI42FHiaIROxNh9TsHhxsAWH|w06FUT{Gr|zXK|;YH=DmT%4_uWJk8npWw3(vv z`G#I{<(;4;3TF_oXlGZOj-iF(`gndEt7-#gI=qXuuVG++~wCK6wDOZOL)sG9JO6hvo9OV`M3Kj*|PDdpPrs!xAX*5th z#sf4*T6PhmROzEs7W;M0stsanyiyRX-s*JB)1cXygG^t_Y0ApRA_{dvY9=VsA@0uk z$BN+)h?V9L4v?$s8%;O0pbWAR&P`n5qWIY!HzBHVmY?g%QZVtI5lb@duYiCB++-|10?Fq&$tkI^)5l}l- zB&iwMf|f)nt4N7q2KFehy=TC=CskG~we_UX(HAJ4pTfwrt3BdddXiKfrpLsF*BG(+ z*%mxwRA{`6c0QAK+8f6rn*Mz4M&y9}EJ$Smlpp9sM=TyC8t*glFwPbAe_B+!o;PnL z(DxK2tz|a=#(bR=GLCKnBSA3wokZEWNAfy`3*!1jns*(~rOMF=9FLavpF(5iByOt> z!;m)yh7NQJIw?IJEC3y21akUcLqVhIq^54Axc1f&-z}poqC5o~htd6#zV`70z{%@t z=#Cm};BLa?w^mSV+Y!ZBe9YX!|#8z4u;XDQpbirNlIu=(lo z7K(s&Jm}}Vx^J(+L7(}|BFC|+T&!!iLqHu_J0)p6^ z^*nSt)V<1_C-yAou-JGO+Fcb`(Aj(?nd6!9uMY1ngX-XARM_o5w3a{8K6~W&P430{ zqL<#|6&EOt8jcfq){HwY)Ec!fmF66DeBD~0CgO^s|MbAdsF$9HnkoM-$?IyG(^2^+ zg2Q39ioJa{#@H_eo0^Zur?_AS&l+eFkehuEM2|d|Zfr-KJKI;8=iQ4FFr?65-*uIu z~Bc?*-4`(y4H}00|8SMY$KbQuv%8h;$}giGjo}T73Ze@xQe1i1xq3rt^00tc2&OHO^924 zjTlZqp3eF+qnkx#k?LV52@m-xJ;Eu%x#gkvZYaB@%!_U~^`tUTxcBtK@+bvK_fYUq z^2*C%F0>bJ8su>Jb0FKWNTJcpbmH}qJ=Nd6TMVMuRYyEMLX zoWNZN&sB&e&6ZTS!7*r3Tb%}CQ8KARvgH$2!fMbmg+aMJYHHZrs&H%{W2QaienSva z(aS`aLDn6+dvGHhVOHANS%uyE`FV3$qh-e1>&mu^@3uM@+e$Psx^wLO_VT5d??s*6 zu_*+_h%Q*nJQ~3G*KZLE5l*nHAzL!S$oy{-;UHVn}a{J7UKY|+=8OWupc0% z1zLZiL|1(TTJIvl+Z|Y!Yl%Ouw?RAx83tw2#P+fZF@^ z`j4jZQd}S1B_8LP8{SflX9o#>$2MQLZqAe88p5+38^2n!>!}!_2XQH-#=-H^(S-fBiysy0RJ1Hm1vd3ZJ4i>y*OIRE*nd$ov?= zW4%zqT`h+(%iAXDI+@v;#g+6pb|IJ?G0)>Z5f782K?{E5&X!V*ZFKfZ@JB$lS|HG~J=BFp@YSj?jsK`oXb=U{@ zi3C~Op9C4b6Ak7cI6BgZ$t8Gp<6raxMM7>h^zcCQQE>iB!1*s=Oy@vI3jK+IttZ> z4mvezgDp3ZLVZ8X&{Z_6m2mYLQMy`{fxel}_TsoOlo#6ZYqxREm^#ukIhh=<`bOta z+jcwq2E4K3x_iV^vkAQ4KixqukLl`TAt7C23OvQGJJ5jb@_v&KO# zyi5LHa{>51r@FIy8w&37gy5bWZv2?fyj2gwI+8fu@*?~^n$yKr0WzS4qt=Mnk3mBN zrY*yK?~4;DPgHd5GYCyw7c3dsQyCX8Q)X9^*jjwcy+Us~{KNL|BDN&krd3aDkX04R z71DJ^U+05(r}92!Wc|=^ke9{Q=TkFfF%@>m&<67}AODn*;G7(RIobY(=SmA~yU0>Qh z^mQku$@&!EuTvZ=`-xZ73fI`>6Lhmc5w#HR-Z%%)SdZg_y?WQuZs8=sH%yppzFERYj$Rzj|{g6;mh1jzf zKBn?ZbYRL_=Q$VJ=dyK{LR5Vtf9#W=S297dp@s2M-omJ{=;+ZkI)rz1c=eq$Va`CD z6uUhLzP;qsN+yFvBoy2R6%m%_LcNo}cbVan?Za5fu=->ZHxOxvqlwonsRQ9c7|vRn z=(emlF(uKoOo<~Wp<{L--nKY=KtKeKxz;x4T?{i)Tdx&IRStoHp!?Slxr1Q-t2QeG z{^VPehhnv%nE~S6C^gwiHllWTj{;Zs5Gh*Y+=^@PDQq^WOp=kk*bKtHq`b3m_r{wN%e+2Ec@lngj_xtj>vQ$0 zsj9dYa|f|aag2=%+ahEP?KF+nOHH=Cw;*9l2ASY?4T2| zf4F%U?5gc#pzIf7;pupoE}IY=pF!*u4m!Lf1XUve{xJLBdxu#pqA2*#zKgHZKpnb^*>m2RfVqYuM6 z^^ns7W5g^dAe@ z6~jZWqydfGoBrF~AYOVy4W4ELU2#s z(KgPIDG8^w5q=vraim)7h4C;axx-nv9$_?@zcHXSODQmybxTfr9=EJ}8VQCN;8^+O zDFqWfPGr0~%FMkkrUCj2S;6MCw$9&iNzjo{OG6&)Bsly!?lhH?F-C@gxZQn&5YOuhnd?1PU ztH?~oY;6f88`JwW-h1gSPn&4J1l`K^^RQjlDT|O8%r1<=7nvlY&$gfDR!CR`aVj%X zCtUQfSz9CL6p-4)-+DbNM+_^yi`kc=PG+qHvkFLnU`dC)n7Y7tRUccwrqkX^5T zNSRksl8T7WT!#&y2svS{M#T0SW_RO1=pWdhYvv_AYc@?JoQLd6kB|wdc=t4tJj7ja#$+w z{8)6IJy|<${3`ESROsv~>R6nj6Ok*GL)&+|dY2VVVdHrXJn<31LEu|H=f~JM)NkC3 zrFj{abo&wSP}@x3`su?Nl7oJ$Ugp4Kh>cr3ab;z~UT*lvI_a99hazh73c$e_y31ny zPz(%L%+=f)cf?P3GDX#amN}`ddeIW=V%sK6A;lkhky%+{hAkeBoiUEdY_|QvQ+xev z69sA0YZ+OUxKzwAn0ybe&z|d@z|7G~rVVHdH|2-+P{#69t{*DdoIh!g1_A}9D;NeY z;g4+phCVY)jzuJOfwGk*JA=_v=3gd|q=~PiW0PR?76Bn79ux|(B^eUhD0a`afc$v< z>Fy%KX}~(@+|8q)DY*Vs^(avh?$q9Hnv|0#DWdIXdK09k)op@Ma!pupos_E@49~OV zh``=RH-6M7j=w$?=uph9Svy{N6fK6gad!S(+rlt=-H7nG)Y{=Z7wC1&yVA1tpDZS; z`buM53s%8X6Zx$3J8la8wSQfo@@-RtaIbKtlK~Dg1stm8waULho89*K_@Bn7r?V+C zGCl{^%hhmJ41#Z0{m{uiRkMND)DDT`>VKR06?O7hf4lau8(RJ_$fGW{Q<|rQ(d!L0 z0#umcde>01Q>Q6<62jPKrtbvI1!JS3PPgR-%8^HthTac+aUxWs+dK~3ohckQOmDPQ z%7G6#YRX~Pb(e*766;TSzE9KIAAoQcGr$;PM<|9|?;sPFS0t%(k?2}w|%an%KK z+Yf68!*smvras*b==G%hmgNLi0^2>8v=V9IeuM3JIP-kBIr3FuQ(-;RT1uJK2u|yY zs2>-tsgab-M97P>B8T=^oeTG%1S3V^L^mU8)z7I+)e3ygNjfJ|2s3&*cKDm$(5zV^8+Nw}OteOg#3dz#&w0qPl&qYRDH{liMx_$X{xtY21rau6Qx5`e zZqw=vC4>zcj{d)!zS(G}0@lB4lIKQ!=43G26wo1B6WO9Okv|a(mFgAOIc9IqXq&(% z-4Z<1<;BgLB0wleoMmxl0*?Ikw81V64FsZD{C&S8-ei79jy&ua#(#8gy`XdlsB^`W zcsE{cJ5>Qm&IZzU(Uv<5zeJOq>&haMZ7_PrJT>Qhshv_R0~()Ma?QNiMc&6g^x+z8 z{PtT;fl7^QJHO_SYX&;91nRZLk{PTM*{tTf3D0)$N(7-gqF*z4eSx0&6zBw0e?&}J0*EwQ6`sMlF?X2%44WZ67* z!4<*_dQn+R8iMU#%;vY;gU#(}3Q^r`h3@U``a)DDN~?q2)=v9ZWb%v3q8;^Np2ACo!l{18262VA2W0?d#rFwE=#%hN>R`cI$XW3bCCQTbL z4evow^Zu+Udy)f zl+Ei4AFUW!F1rv1ru*rIZ;j)a$G9SBnU{{pl0y z+ESe~eth~}eGr&*UgGf%K7jJU_ZmId(pI2}A)`i(gT0NowxiU!cGrXrzHOW(&x?LZ z!24kO#;(Dq)Jd10b~4yjCRq@{$x@J!$)N^r4vhZKuggMoT`1nEE@||`dw>t44B}3z zOj1mV2b|**E1W56 zYqp~9ZL(=MPvwEqZwVRYjXvZVqsj!^$l|7rwK47ikgw#&ks6XC50(_SeU- z*NBmSs9QGZhn*D!#wvNeV{5_htP%CDgYskK$jo#^%oU*J?S zgYU73N=pq!%VOiYj?pj4N|Ub=#T3irMVx(E;-9rAzDu?5_~+}twK=DFE!i7W(m=`i zR(f8l-0wOu=yN`Cjr~wx`m0^N!^|GQrFA812QjYLf{ z3IG6uCIEoM|E=n3YiMU^Zfa}#?+hJ0{~tEuf5A3f>s;DziXr{p^cqnU6Qq>OEcz(# z0jq4DW4d0gsBJ*Q1Q%4KOgD-w=uE`Y-F;uPGk@;U;d_zT1yU0c{1v&GnVOlg_U}BR z+njwcP0-m*z9OJ1dSv=wF%Y-Jxw^W!YB(f0sn|nqks$B-hpaR&;)&dk>tHtMeH0?m zobZG3GvH%8tI+QL!OpLrLbJi6)yzz>A>@dc0Lda_o_Q-vKPCMai8g=qNfoi2S{GK@ zLBJLT88W~}iMXna&UXd5c79`R-uzom5@&NlUOBsP9Y@?cG!#^P+6G#EOvTz)-~#fm4lxHA25P1e(0augWt$6W^P`T zbU(k($<1s0Sw;>neps}K{Pvt0Bga>-A1en379E@!0Q;MaPQee6kq`9_^6Wa(PTa?m z1Qd}yI?fVdsB`{I)Nyjf)p;rkHE765e-hCbfkLnYQa+XFsU$|KYY9WOww{5uj$>xT zcKFu@1j$5ne8FOd{Fo^39@q7Fmq_~gRnyHE_Uma@=FFbxy|_0>O_7XHlZ2Fhf+hZ6j$vz<2Nl?@ zb#J(#(G$Vkl4`lo`$3*3>>R(|S}*M0hR{F-I6K2rB`iM~p(FWAd0o~ySJtB>#Qm+R zlTkLv&O3;I#A1FMw~sZ-DI+VP=g+S3TX$oy71ulGKP6G#4`S}^@8jJzg%3Ly=C{w= z&7H~AOL}WZXlvsm+kPWtQ?<scnd0Xm*u=s%j#E8QnIcRrj&H4Q01DDI%T-kV{# z>>xts71Li3)R}c!eW*KUIyVrGPadda6xkwApc{e!IaaUTq?~w7YioFlTbZ}yP!?$? zF`q%BzDiHj?|ryn`MOo^!{b7LvfSx0;Uk&DC2Am`jVpyi6RmOT&Z;U59OQ)J!XW|+ z=O=RnF0kn2m0R_kJ7p0_kJsCIn|}BxB5I#m7Zc}t1~*W{1VX?_qeO{-U=^F1{c5C9 ztCku8wN+S=%AwuHo=q*|J)mseY!k}fVmh{9-qSR0zB1xxnxm!)ivzfYyG1ZWbRH_t z^144jgS4;nc;EsNOkgjtWr{jxd+SM}fn?}L9eYOhIy7y@eBKn2rovC|dFcXQtY4;F zJmhAlrhx8sX(%(O(y6og%h_cmIEx8W5vATNz==tuYa|vpBPjw7vVC%nL55@m#R7bN z#e&^={P@aFdc{mG5m`2Xe$cS!8bI+hjv_6vnBfhO7~e>Zg$qP42j{*@CX`U5K!x0~ zcAPFW9Hk1V5E(84q(apXFsAMdUDgsk6W-~0AUp&S`%+yLPy{eCl9|yB1~5I)Qy=}0 zr&a{UZ7@ZD{PPL&p?wJVQIUtAy$?Euy2`+*N&3eUem^A>ww#@gB!g-SHS;xK9?Qc| zU>3MUn+d<;SR>|n?~>JwTjduThIPVJ=`YC^TU0FlF;Yw!zutEHSR>-tqNvAGSRb() z`0wb$RI0^lQ#nllrr?#yJR%Z%V)}S(sE!KVd;+H8CI)a8__cJ;SNtLiE-B z?Ds+tAK$N7U5{b4QQpozQgKVUaX5~Th1IsgOjFJOi?nkH79`rT=(BCxwr$(CZQHhO z+qP}ntY_3iSI@dffAAx7meY)kb5E|lc0pPx8Vj85)f0ukmO5Xe&3M!`d;Smh`!h-8 zG}8!)=Jx=>@K(E`n=5Y!(6KWLX$}EG=wR>8%&+cJtRSM)_QFFLL zs!yBiry3yfbzpx3+tz*+8wSK~vcdC~cVbtDtPk8k*AErcRK!v4fVG|`Kk?z>QJr;p0a z6KX#J5@Ai5kQT=0H(ewwB#n5x!Zc*q6xwRQtiZEaLSYSQSgVw_wZY!prXvR3i|d_i>X9(cu6iOLlGPgBlI`{X?~seVocxrCd4cTdAs zKs)@y3q-%D%erHuyMx;&_21J!!yom<-@>mS;p2J)#M$-|0ibCENmIm0UHiBk`7}DU zYpT{AmTs5Y9EwSD@8poOy0oZ8C+QdwU{DMt*{uMNXkChi^1G#3>oXk;_f}a$fPL#@px&=vC}Ub-E6*dWsa>VmO=_r25ti@NQm$uCE$jvOmUFv z|Do*;!XV}-+hd41eWKdatwou5`>9lzl_ih?7ZU%8_g(MV#o{N zLI*-{K#+wu3F}h5t*PjfJV1knRw#_>q7Yz&aH1D?2BLZ zF>3`9nnt*pf&_=llvk&D#*qcjKCLEwf*1vT9stH*Qr}VY>@|kjU9#h_vHHm=-q^RC z86DcXHA1OU-W&Z;{v@chQbE3+_v~7QD?>YZKTGC_D!8pFmf_=fAhgUz%qeZ?MEc6X zlUmX=brp5%J`;hraw?jv?jK)!+?}aq(bbUSmP&K~I?ap(>#MlVs4zs;JHi5M>oBFTs^t@s{ROq`9%o7ivVXibIE(G*1B4SO9W(6_xNYEc!fnMgtCp`+b{AUMLtqHN)TNlky4#DIdK8qVy@Yeo16FxZr%-D<2fl~eOq=ySnBlMF z<9Wov#^&qsHc5g9BAr-D8whX|wyjhqKW6xRA?4q%8!a9??k5G&)+gIB~kPW9>KGbX)3UBz=&l-_d!aeo2t!c$LyQ z7JN& zPh*#p&V>nAl$(1dk}+B=p66Rp8y`E-N;l2Q9)^v#PwYrMu)GSWuq;u)z6>!PPEV0I zh$b04;GeZ~sjBwcZWl&R*|Mra(p5#PjtDaqR^hu&IPhq-Ed*Od6G~2=%RvmzpGcKM zTA&Tk9|^AKxU~TQ%CyyUyE~IS-w=61$;t|{Zoa5vFZAc1j>MU?tsO?wHUXTACu--p% z;mf1pxf#k#i%lNb-qBEx*_cwEk^}b&0+UsRvk`4eUL|h@`U^shHQ4H^_y})o8yhzsetyH_d6{w0mPLwXV=Sc@$dpukl_=%buiw`>*rHx; z_kEEISk@#WJap;vZHIv{&AFkTC{p;69>-B>d=+<~Jhy^-dzYEl?Mq!$D(7zy^;%0q zDvPA7K1Zq9cG+;J)|kP(0R^v)upX^-nHjFE@OFhA92avk&_#nu45i_K5Yycp(n%Cj zt}=+*=KJ%3%ftv#`ZyhWGt;E`jslq0-}%k!gK%^=#%MzpNmpWML3V9{d!@c_pnH70g4&N($UF9I&)yL(Z`EnGNqSv7 zyl0@lb4e&sBHdUz+8kA=Z&#l}ar#_DLDG?0`?@1_3Qj>Z&f2$j~mlnS{9Ws>x@((bu-u-9EZ--@!fCXY+C)zYaei>SwK1o!~q zPv#J?8Y|s>q@-1!`}O8x_u^*wj>sV&{F^Fx$13|HPCVmOH{%GQ(kS&!XtmvzjRbUc zqvb@2`E7$hYD8qH@-_tzJAPy(dNWqoqEYtPoR2F;gWea?1QV7KoWEg;FyZ(0>^Fb& zges|e4e}he%Z99i^OAMJsRh#FjxF?`8_)X8bb`YOzb#{Mk?~eyTlep|{}Rjz;rWMB zC-$uICRA;oVG}NxTIXz^HDle3g)1WsHszo{dYBU3s**DBUGf9s=tVxkBL5=E;QaCt zGTC6)`xO}yhiKfrxd4x3Wei4aGc>@hUxLhXbDKYvz!)lol(4dqw9;#75Vf#+6fI*8 z7eO;jjl!9YPr0~`{){pZzx;9)EW#LaH!QkrS9sLF5Rw2#Tnb^c7D*{dbDuRW6@B3D z(H5W-$I5EN@gfg02YR2ka<Q{!JT$fVqpA|Cp#AGp{nNf^^`r5AP)0 zJ1naAFqLko6)jwC8Fgo_r3W1;qTJXkx?q>Kyy#k^Y6&vTgh2B}#`4W{2CiMM?AA6@ z`g08;8Cp?9*l$QxT-2W)HJ0K8T=B}W+X6+!dt=}FE=bCgP|J-U_ zStip^D!Yq8I!d--JcmvBemq={^o@aw!*L-Vtq6;<4~EvHcw>=wKvKF0FmaVVmA-ry zCqY-G-MSLxp#@{GY1rk9x)CWx$GiIPYckGFEwmkw$Cdem+HnrW?2y9WfWqsr`b92S zsE^BK^JP4yyx-c)x4m6F-4?v&<9>>G(in5X@PQ0kvdFx=mWfU^mq)jBJWhB2=LuE9 zSQb|vf|Y9228hKj6_VR=Ol{P)D7I=sCzI*rGTu;iaYcU7nx55MnN72%EnPPI`MI3d zVZlX~YAfck_tm7xvp=qk%b8Fxld7Z}{3jPYJezw&sh(jUDR}E913Xd9+5^0l1*6{_ zm%D=E4OS#t8vx2Bl@(ua?w=MRVw`M^FQNyRyPdXUqNQV5wbX{lIaF1-h$x_W%;qp2 z)jVWNyM|nM8(;nI55b6i1i7`geu{^z$V!d$U-uXzdg`)5oR5*v(SeLj9Q zF`Fw|>YCOJ(7yOl2u6IY99+!DsstvpqG8k4QUfKoVw2FZrD`jZ(T=K1sPD7dLBL&d zq)dYcaV$c$aR>wIoTXP$-(i-v*i|a|=`-3+SFyX>cB-^{v$D89U2cpp{hdP8vNLoV zveB(A*?uc%F&hSYMG*=tG#&FoY!m-iVLh=;-Uz8{ecYA59}A(po(Q_DrrH?EZVnIX zA-NT5gil%k?5$=uEaXn9S_795{P#_#w#N;=_YFAvt98-y8xiKzUFc1G_Y3_-^)vv! z@M!0ir-F?qZ%2$~)SizM`?A8tE)g*XSMmCCfIXeSto;Npu}>{L-q63nP&r-#0pq(M zom^)`5zdW#KRd+?zE+9o&>&3|`Ea>ZRsNTg7+RscGZlGNMyJ5)WmbB3iw$j6A$xjj z))alr-`?&ls14W{dNLu4G~E9&>k)JI_akA z8m1haITr>{0Rk-uPfA`9z4?SchbhB_m-ixRcN#rxQcLL`c`YdusJI4qdo9I$cYPk_ zWqV@jaCUBElTcW>KFvixA!~O$>*{HhRxcZbsC61os&}^c|e8_v)j<6vs;g!|eymP)-RCrxbL(i=qK66u$(h)drA$tH;8uXn&Q#8l z_uew{Qp(-YJCbO>oqb

+>8cZLFP@TicM``r~40&35IrrBJ!g?!1@I(c5st^;r&W z9__Q|s0Pq^C`e_Jto+`^I`xB*oQt%>hdJUU;O{!mJ_Wve{S^>JhXZpPRWIshTKGY` zPaLA?>^jf*`z_`e>pTNesl7P-Pg(r&=3dX8iQw{6{<~ujy|0B0dYaDrMO2fOrvNtq74pOdrndvO|BE*-t#`udZWelIp8@YSCSf3Y^FXfO7*-L}}Pu0CVJVk!!S%(gfx*qSTON`;|2 zv3E2PYy~1`Qb(CYs<4(?@bmUt4m0}PAEQlKw-0#f{2s6(O8@<|peiv5eJ_@R zGH~`Z@BwcT@qt7s^O=7PdT=E?XlD}FdJ|P5+w__;>7#n#3?GtnfOu6hPBr2zKZ zg(aZMYdqY{(VYvw8E^hL^?_pW>+SizJiAN#_ImyK-h8<|9nIP8F+XQtx2OMi)g4~q za(Dm4e*cRmxJ`jXKrTd#>jpjpCqhOkj8N2fBH#g5>mZW26G^E_g7}zTN7aLBpTieT z1mLzIkFq6JNI;&90+7nKNlr>@Su&!+fK?Oi2azqQh*%*jB%qTa1>Q6!5$weFw#~&+H&-FA$}`H9vLjmD8>Ng6rgA)4qBQJ z>pWDgnec7Y;@GeBf_zR-mq*0RG>qWt;ArMCq=pfK%Cs;^a*iobh06&53Mxl|d1-3|klo(NN1IHQ2 z0G++t+miz81AYbC%Rx8T0DvKBI)p!A(@u~17Bd)_9N9Y%paQwD%OrCO{!tq>?cfP{ zJ0c)FfH+uw2w?mHmckmqsxq3MMrer)q8v5m+>?qRVT~@QxS?cjxNu~`W_X7QA9i=a zs|~ab#Bebv$R2Ox6%cYHDO()S3{w0>BHN?_SIX@*ATMs?5(nnWtf7Guh4TyW0AT!d zunU~oD7WXA-<&u5*HEx~9)Gj^mM5-80We37%J&n`XT=Mh$Bxin>dIJ4 zfK(j5LV_g*F(gb(3Ri3#HX3(Ey!NN$P| zS1O^dY-C@Rk=2z$QQZdBil08BT;y}*Y=wq>(q&gTA*~*mNc#QazhpGd{{!FGl1uKd zEj)Ht+}~J9cHbG;RT_8()uH1Ii3?n8LGLQI$W{w>UvB^^#F+a*o_Vj9lFid5H3EWa z1sh+ig2j79Dn2I(=153X3Sk3fZ#NA_o=B~Dii$Xg)8(#rs`1A6TY>uF(nk}XWeYNpHG zGNpizUaUwDC8KqSfC<1CMD0+%aX>8qgc^&%+(rOngKo_&)MHk<0zDARz$q0a>9gjT z#)HT{q+`1X$u`;sI~rS{kZuSM-q^Wc4pCXR?~1@!b)f zujOAlwHzv5-pF=zd=A^%#=!#S_2NMdVR7lfI19h~20j+!46<$FtxSFpX~kB%X3;D$ zv56B@+VjQ%czzI#%YdN)K7bRIfI%yyk5naFSgV~}NSQYL4I!inVftT!B*7w))231Z zwM)%Bqd7oc;W|$@DD*OjfU*$itiUO1uS_{bO2-5|;};ID<{Br;V|C;yLW`Iy`eUh; zspSAYOfZUcH*4>dX9)+$VpCx~wnuWzy@lcucUAPPk7;@#+wF&_KHGT(O~!#n`gD#7~{B&{W2 z*ATF15Yy{TGV4mFTKfufwYXXIUpDu`C@6!3;amHK#2AqxJ?GYAupOrBNojLf#&=%5 zj2SbTKnf!6kS?4Ez;al|D<@0xpeUfYE_HT;_CiKG81z+rA%~xKhG(u)=eDO%jdR6n zZiUdmp7lTkls8h8pdLFH69sV8#;{~D-&~OM9)Mp?)~7U)&EH`N%L3XOjup7Fgfbf^ z`Dvta=4s`r5i=DL#g@%8bqqx(wqIQnvcR@eNJMkTcjQ~P=nH5@sTDnpVJkq-9C!;@ zK;B|NXT@vF`SLHNl9C|VVyUT@hgO?NU+60m8$&Mp@O(^erE7!DvNZs-bm0MD?b}fR zY|!XLv}-9?T*G}w>l>r-p>L#Ii|^dVX%q^R>IXbGT7#GPm=3uoGkfa>D+qcMo8K`D zMd7jIFshc)TXu*_J18q>AncAAjo7Y6F;Z=ErS=Z|zBT_i$ejbn=kx7(A`;#TD^KYX zXb}5N>fnM*hYK02MXseu_mx}|xP^-miDhby@B%j+#+a4JQI3J?6mYh!R#vZOYbNS^gOZtv3PX2Bv z`#Sp1j|b4}H5V$^w?b~$KK3h~w?Y2Y93jHPf?XXByw66tvns;X6t10cVbnm0A~F`GS3NF)v<*;D|D+wt>NnJibGH4Gkbzg#e4M#+n)gX z>p!N1W_a57?MJ@?@t|I}C)Jd_?huyOh_OMgvFyeb#eY0a|jaocou` zBJ(lfY#z$Fd6sco7N-(k8qsNAvccw{uuNkwS`DABBH8ctwKzb_g@hn~TM}{IksrhA z80GA9mb7r$v-X{H8AGd{WPVTFhb{3QP@D%=Fm0D8DH58wUrWl%&tdR)|Rbk0dv z=poR_(+v{O@!pPvl61&)du<$!2!~$Ps)uA|4>d~)xh;6knDRk#Td+}8D zHw$w5=iQ(F_WGzgVRcHlj*8DB{;+<`GK@-xl?l8-Fthu@DQa1s^satVBpw_)EwL+X@C5g3jr71m6p0 zCjkoYm9RC=rRpF98zsf^o|kk$3EC?msb5j?3-te)9+rN$#$93nfPr)X0FnQa>9IF4 zwfRrw7Pq|C&c|X-`!Cc6m%xL(bZdQGD!P>7>yy-+&H z&DO;-L;xcDB2m2v37)x;uGmMWEY)h)MDtS_5qM8;@nH#q0I_Fx$f7PwW{X_GzpP(c z8@EL@q#^fPZoXevtHZ;>Nr**~s{4IZ9g0s^vme*Z?&#+9fSOi6x|$JXkya1KWc7ww z?AsLDz;w+wU<@=(;T2WIH*IKt>S__z!@2CA9^H^wSBrcntN<0xJeSu#X3Jb5N*RJb zwqOHPG%<1WbVSb4!xz!X(a+J510KFVPWSix-7$UrA8*^q@9E~}>5AU=#p?I_xq16} z{D0y6d@*x-{(3*JesnKOk_5;w#CYheNAMG55<`yST)Gm51P>G$g=Dp6oJRYAJyF@S zZ`MZGReLL@Gvr#(Ua9U85MvA2!zo1hq_%5H%X1}5!NC~}RM{05b@hS*T#+vb|E!+G zgGYC1i3*V56h)pufB6p7n`tf&Sh%Lm_|b?!2839>d}gzoKOe-5fGhwgPDlC3QRMzJ z|K5i%R!7ZN*}Ph)rx}GgxU+pb5?|TcSBL0Tx0R<_{A-EWBzEAR%A#n+zkxiyEwgxX zCEVNp0>EI~k8)E^|JSBdcb*IHNxB77kw%$t-DOqit7S%g*uLB1S-90(A9!ixmo>i? z3~b=cqB}J6ii2AyUhV`W4^L@#f`jH1%zbKjL*6N<+~y>+6#t&T9MH!I^jht zqI+66DGh)#S8kMu-dT&as~ehj<>k6+qpr%!H+{aR(EaTjZUr=k@=QO{7FbKeXP$~6 z?U&nEZoZG_<8IdUb9Hl{zK_q-7fi(moTwyl*Vg{ZXRZmP$VYwlI0{5hh?9q><%CE=*tN_D>6$78&8&5)mHxXUa zcYDB}J`MgoofT2k3&SFdg2_G!5p?6soCR8CdeC&FOz3__@)8Ru18Ve-7e zJ1UjJF!-!iRC1raB8FbXn@WMWMi4ioG~PV!s67DpBknCyti|i8clKU%@-vVNAg%XH%Rh92F~T0cqGYYtcP&AKIdQl zw}tJBQVCmzlU#~>Y8$8pMH@(INGSt&&)E`RVEzLKGaa2)gL43}NtxO}sxdXDbzlGw z;>-|nwZgUp5lZ_C3zkG>L-*=T)~fiQr}Q$&emQ$aU!{Xmj-)Qk;tp-pUfGUoVF(Zg z<8j#}ca`=Qff5OlM2(MJsB?pYXy8$35zS&c0T9xe0Ei{`Oo^I;j$G#AeKa~BasT*G z_YGYdgGb<3;ia$}bbXb%b^0304Wq^$BGY*FP`t#ZQ!p>K?9UfZ#`e{oy+Z=A};_c zDpHwW@=3Jl4u9>)&t`5BkXieX?HuA#I(bKKT5V3P<5S5oj@N zogwYP0#a|s?$N(?BQZ{oGz@sdNFvmcS%?IBK|Zejwvzx`@D`0)emOr5ku=d-pz=~#B7@c25prCEREvOIbff%47xTJDZ4dZh+fWv1#Z3q_Om zU!B)8Xo2uXMVL8sBQH(+5rXVp&w1Cif>TNg$P!~5MOytH@fijHuUyH5DomDH-G?<=7s%L*?7HNX+PE(hp;8x&LA6i86wZG zYTh_$#SZ5&6AA+a*SzT=A(ReH@~}YFfj}#G_-DVeN$E@N?&Ap%zhhi=U2rjyi*u=m zs>Ly{pT~y5+CO(@L9mJe$A5obu}T(m%)CY1Wbv(r$MBiSJyw~2C2Lsjn^ zQ`WaY(gqh8HqZJu(q@=47eO)wAzT@bgj!%{f0fGaL0Zc&)rW93AK1pNRiKvi`S^eS zWH2c^XfwhL9njNZDR^IV{(#SGEA<{0m+bl8O`6zc%E+wEJnfYx+Cs{MVJV6$vXU8LxqKWpISZ#qQg-(DG{-mnA{Z+3OIra@))`D0fTcC^j(58Ik{v95Lk&XUp+=k~-=SNg|}McnT1M$0?W! zfrp;&1YBv{DYRYUx=nSM_Gvgo4{%1IycS!uS!$IOM}YvcViOkK)6MOFYrA;)U{nu8+@pQ1tUa(Q~s3VdD;|sw){C#P$BD|@a-a#4NC)a62Y>;e z@Gyb8EX}-N%PMioQ{ zCWjpFEB_sI1e&r`wv#B8$zm&4lr`X&mMZ@n-up)xbZWpC~P#m)&EX?)a@YU+gec;Jk$c=wds%NAxmY{1a4mQpzykrA$X91fwL^&ps&a^ z#e@LD8ietTocoio)@jYU+25AJ2jjs3@`&v9-r5@B$s!@CgII>U0-#Oh*R z>%{c0b%o>7HVk(j0TFz?P;vo4`j4*hp_Z1*byg%p9p7r3pH>J&R^_E_q^tZWYy?of z0-4GL2bnosG@SuXY#<7~vsdi*3Dv?vf!e6%Yx_3B8qG67AU{)0jn+Hm(&84UE*K)* zRC8t#WEPq2FJ^0HX1^j-E;Rhwwv#zlO?-Nbf$D-l}__0J|H$$f5L0I@D&g3Zsp|B6h$W%Ts<>}d1QKt5ofVMsDj z?9xU3SVku?WVH-Do1?1j6>O(FGOj0t0m;Uo;-~<;PiJT*K}*`dsnE%)%PRwNZq=}& zp>d^DFFN?PHH>aI>blzJ>%-3cI&X)j^|nk~DXkf*?W8xC&g_Faok|vfg$~X#Lv30p zYslVpT{cuKI_h$Y8INMOOg+&`nUV(eo@+FdLCM(`XqABrBrY%vox7D>v`OQ8l^)A~ zTt=M2`3#EJZyM`vZ3gecTvKfTz0#BzH4&cq))oWhl0)W*TcGiPlbAE1kMDKt+URa2 z?DW}GFo8H9(Z=J56xpyhKw7F{j`Hc-eAe1kQHb+s&<^gL8{kvfX9{Am{zU6f5<~|@ zf{s$5gf5Vr}C2>0-;Id(=+^vg`v`Q8YRS3 ziEn(pbC)5Jx*-6T7M@VdXI#XdGC(2k`OLebB>hO_ueUKXdKGw;#0)*V1#W!p%{fk zHeTTdy6Yif4NSAPjJ-+dMd>wBR~^%WWsL9G3;>YY)HbQck-;SPd4v$I_vUwW8E-Q6 zVDjoX0II%Q{Fp1!kC1SmHyvDaZ~Y2D2G@mU^)%lq0^0@<6-d@|FO0 zcE!Ax-(nUzQEn4PQs;H2v4si%IYj9GoMM2VY-2W-goc|h?wC=ywsU#9Ywj6XP6+|} zq;97cV+&xm=7v?sI}QMeTJ1`WyQ-yl@Q|vo89H&mNpf$YK;HInsi+A9tyg#~)IKNI z$AJ;MABwq(kIyu&kVA-Sr~7-ul4Ekg{xq*RJO{4lUHH!G1M@cl9aK!osG((BI@%XVJ{?=giHCx<)%Jk8<3K+3S99mq&4_Nw03hm9_})z zg?U7XN-Qg?rJi!q{82Q#_YeF-7^Wo?-J|#t@g4)ybVR!^%59>|Ee{U$hg(jeFQ$g!|qwWE*g<70o_ zcrjkpWS(eSS7xxYm(2m`LFL%v$2TJ*S*z=uuamZVw(A$BSpLbrKMu8IXn5>-$ra8?H`Fnd>Y{jrecw4kU+_(o$8VEsfUF?dP~s-bSVga%k+h zz|?+vTzWWPlB9ClA={seKNh)rRM-2GVW&5}Wyv~MIpSnr1{w^s}*DLeeKv$leHN1J=H3yoj`zOS2$ZJs&hDsSc_2q=Ye;|}NwU5e2eT)qw7YC3;>nseJt6HoHU@&i^1uam%)MaF zsT*AfC1e*J{dbga?!Gc_=@2QnS^x4&2bWy~VJ(^oDFp2&C^ddpIW>0yBCd*p#LA(H z@$DVg4CP;;|H!4WLbkpcU#nIM?Nas zF-UK!OR?3u-JV&)gu%rg#u5xy4j3R#tvq+RF|Nd;&O}1 z#uok_{P7^M5EG}BrqpHLX$;xQ!Xa_d(Kjo%DMZGIm-+*c-G|cR`$o@h4#6JXCiNi2 z&7yF8w?-yRl55br@(g?k3q|1|VLX1zSl`Jpu;)O9>7!PPg~Od{K{h7THIwGlou0Wb zmn$!}y(FwUPmJLwj0%3z(2mou;`xTP1Do%B0ZOpk2N(1yoF4*mPKUS6wG4B{fr%_#dw)U7>p~P3xR41+)fQ1kw!I{OtZd&mjD;DdK*`5PR5VqR5=o}~~|F|QG$DB8g|1;i;$qbG=QS1W&Z5{x2HAHFe1+0>|cw3$f0*?trcy4$h zlpc~;aAiG4aSnnL@!DR9IAh=sbm%g?qNwf@;7y8J^Tnz{no=oNa@W`fl|o1eT|wsI z@@8n{_oTQ3-Zvo2e>#;7Pxe^tbTsUTxYVYG2W@UBZ4cI?RmyXTCJ~AiSB9O`-mH1t zTW-Yn=p&NcN&9=HWj{KU0DaDGQm#ATT_uw&M3jlWL6Jlli_xRrfFE?l>n9r)`ASQ9x!gD*`5p?P9)jjD~V+99Yoqh(vrLnCaa!`MqZnd^9_!}-e``PD<9^p?dDH70XVYmCm_D|!-+%Hvel&~KG+n25ik)Vb99|lfpm*8l@eCDu; zgo6H7Pl`aDY}gObQg5q_F6h?p9EI&C63yq!=mH3TWP2}WTl{3XJAV}g=Mh8n@;dWP z33~xJC-uGyG_xXg(QEtYu^0jws-65|a!W2f|I(KJ>EEv}eePy(DY^!CP{F&J#D>ixgIo;Pa{Hth7()NKkO1lojhFIS%CUJ79k;7 zYvH3H_bURz^T-t{SNk?TqVz()>aM8eUzEK&^ {>#eP$lAXy_9LRUm%fdy^a?19r zEzAP#<1ut=;zQ=8IK;BdpUBxMN9zaQ2Ik~&Y`-Ll zSl;;ln$g}ky!@HQh*t+xXD&qPcS_EBK84hTkNsOq@a2wTNS4Etb^aO528-Tx$vkJL zt7E*YV~}roX;!6%-wa&#${tVCJSaDKZ{N~F-`*h0K@i{73(f-VPhB)ii}NgfEknu! zL51GbAh8svcK6um8qYM)h*_iA59Z4f>Gxxb^I93iaoDRu6>fZ~T{(;ED^6W+p4}6h zW6vZhASZzoqwpP=Kmj3)^?)W$p_~dmaDkYz0b{NXxo15mO6?J_Wn59L3WY;|fnK3oeFMfRS zc-4G;vdijm4P{As@JdBaj*$mNf3`t-qV*sFSl{L{-n%!MB!1-^vD-a8hGg3HIb6UV zmLmjZFb}p+EY5R3r_P&#$eAgKcxjydrq-+B)Vw$KxbQwUwt@@Cc{&IuVV$B92laEJ z_^S$jUDa-?bMEp+e7c%&ud_)tZ#&_*$J)rZN^Cy|IvYzv+>})Mpw|V5N(_27jEW-1 z8|Ab2h(Na7u(Rv)SqqiV-83#L#+Z+_7zHXkZ*2=qGt@~=E6C&aQNoITx({d42*N=ItS-7;9 zyUib(-2V-pp!jX^XT;l5jtj2gTOM`mbsmUDPN;?6S_{?gex%SFT|9tUc*Vtzq>+|z ze&fnYOWZ67P(>Sy-}#Io zuxHEF0I$Iji}P-)*cOqy-uYTW#S%l*6yRSv+^#F&Hz8eeZPd!SO6lrs{4Q~lsnJd) zTT>$3HaL;L4~|S2ckQ}GxdNyn)REcMEhhZG*CH;tM(eUe{ypW}z>lv+y6Ke>1`}rs z+@DaS-EcB(Ev#!+* z8wPscuarQ1+i66HyYcOIgsb35<$OSZw6BUL9>&y$8lxAe$HemEX@J9}oiZf`($Io9 z*Me3FkVIU_P`k4b>FFHOCo|F<@|_2<4kk%3Ix)g$AiR-&zG=mC?j_BK7c3~HIsPoH ztPLdLV3xa1a(V_(oMK-DkTF@nq2NY|E0LN9qL0}7CN+XxQD9P0Ve0ktfhDmsz0U?= z)hs6sWy&=#1Xy~&g5|lX$`+jCmo1vp(KnF%)1UhU*OflJ{}jqKMqA>M#6Qz4TCc&r zEc0GizBTFJ%40$cRfGapbd!1jX2GG`29Ha{y3@@!-y0ITZ)?Vl#Zj=q(hz!v?lA`_ zUpgBE#8Hy6P3Ak@c z_>Ae&_|tU!geb_a$OkuHdF@_W)nShyKy7?~$cWjR4uCW&p(9jngU1(Fqb*<94+vJZ zKWur=YPA1B8ULZwABRgq4&&^@>nh-r5unz1Nksm`vqG%dUl&e#DjhhOnc7^*$A1+8 zu-kgz>)B#G4Sz?!HkL&xjnmp0JqN|lWi5moe$4ylQHYjo!3I2^C-KbWuk~+o1rte= zURK9hYKS}Uz$9^%is;Mvyt&v9Db0*}i?U-XR9+*fU4ZEQ4An8}=uyV@xLo1kjE`lJ z`)Jry#L^hJyr_z=am75%AA$>o2Ee!`Je*kRrZM2>6t-)&u}nD#LUq3C)CQ!{n+c<- zR5l&y!?C}(-pKAOrFC39OLdn%!KzsqS%^GO2}f@CRScXk#!dAV#vXB|o1F{BA)*@x zbgMzoh13ljEvFRH?7NHOyIw0JTUt~*$}Pkfn$cH{_Dpx&k#igu2Zyb9RAJ-k+K(@G zRh7n=cz4mhZ;e$eYNa6$6xL=M(Xsd!Shu|4JtTuoh$o7&Vac=DvoJM^k=+`x?u-)B zQc1Jtjr(hn84f{RJv9?M43hl~J1~yPnCWB7r_+?&>*?6~>C9-orHDLpOXxVVoyl*V z9qnJo@b@KM{R2^Q0FVkT{oN4HVZ$Bux?~me&AQeYnOA++E`$BjU1TbGvq1HDqUWu6 z1f88gVizpo`?!@TsA7o0^6oFNFV;$ZEGlx3L*lSBx`60kuWU=|-)`YJqz3$dO0j?2 z>#Sdp0Z+cAP*~RUOPO>U`RAWR4=N1BpZOPdxj`GWV^$#JVS&h&1DQOUUtup^PX`xZ zxZ?kC!=KF7R=eI5n+$2qwMY55ar7o2UZl7LjQ(sSGm`RR_;~YKJi{Ep3qQUuEO*3{ z3OD2rHu)X2Jno&EO3R=okewh+|qs9>bfJf^`Ucj_@mw<4qTBLbe{J>J4!b= z=BA`kLn>-?S^}e~D3)scan$?usqjd|MSp&>^FO4WV~;2SyJg$9ZQC|a+qP}nwr$(C zb=tOVyL-;u%!j%2yT5# zVAkp-m`E3?MnUTu4A(p-;JNJu`8akmbjzsuu-`eQG{mK~X+?S-x(K>2pLh$aa>mux zWsm)pAeMxDu8M*FUlgKPD8e|=BqEJS9;u^ z(1zi{%%8$Mrj5D)2+FOoVj#y)I-}L0(z14m4Z`fl0Y1~tD42Nl&99Eb&^y!GCF8?n zOiNjA0~=o@Kynd~l|Pc-G(_AG9(`6*?$z6sDlyn)0RMZfZ>!^pt*;rc@bMywq|mvk z{x}Ja|8cP~11Nn{G;_OUR6c{g5eVXksDostl8CjkeneF^B4C!^@W&Pzir9q7O}QYn z1@WOu50)Iw$R;zL)Cj+3Aa3e8-<{CsO>A}hJ`rKxY774+_^YrLB`~S9)|Llu4MYSu8a;zGc6>W)4fj-`r?9fI4fukHI z5HJFekhXkn({q~zW(M|$*O=Vd$+2W73&_a5$@QHLl3TNAP)|L->NKQ3q{36X-ww|? zmS}%8K85tkA>%>!9=z}#lv34ka%~JpeiQG(Z0NgJEEb)~4FRX=2kW|6zPaNE7;G$G z=h1EKE?1}J@c#pvRf=i$MYkUnS;8q5`A#Ht$g_XNR#R(%0SYpw-$#KkqJk=XiLpVy zx4C3Hb`XisTU$9Y|MPQwZG(YaXrM6j!o2?Hs*jJ`=h^hj51xyg&!)O1hxd1LI&h6fnkm#6b(|ML>~{e5(rC%2=W-GkNin74lZo|R^XZgy(7 zxA}T?G);e1XJ)4xivi9y!0=_OK=k8p97%J8e6*VG1Rh{Ph=>H8TfE2u*!iG=JIj%U z%HI7q0H<~Dm2~A;tW~Q0u#F0$@tgw15_z~0>e_rX`7fG#K z@^q^?;FkE&dHAjmIalRA)-oQo33IQgFY-zDpM2c8FreGV>m_j{Q1nfZ7zrBCJ*~l3 zS1Z0LV>V zp1Rqxs;f&GCd`L|Y&AdG2F1^fD+-YV?$&Yv$d%6_ zQ5jy!OCDpO0w8Xiuh-c1vfLGjtqQB1!RgLC1xxeH0DRt=;PEidE+bBN%3fguVF1NB ze-D6|+vpI-!sfY4^bYCTj;Akk_#ga=7*b+>9fCaFy^TbWQN${hU+ov49g(LRRSnMO zptGDABGLnUEP(_ z5y!5uP)|6S8)kfq<#=uYiHlcW^bvxdF?qoQy0u3p;G3X>in!ku0dQ2UA*6y0-@M|} z|8kY&7{LS((ci6_hm#ENj%MnQX;8O+MQ%g^cc6s!2M6emZ*hpUVO`f3`DX&j-6%+X zVm*^AW6oqr`6LOfUg@GK3qUv+?3uU>ky`k|cnKRpLnOr9mviaJj(*J%YA`v~BqXJE zl%QxXX9tGY!Y87m*gBObiwXS77*vT71-~LuHzEkQwO zauD8#iVB|24|6n8n$|+Kcp3=4cn_Uz0zH$U&Vn}uX*!oRkYzb+_>~>n{md|gv}Tf* zH0VU_MGdeDm3Y6>oiANG)6I`i+e!THR|6o6T6crp|52 z^SM`zOAmT)-EhyLGle9*{Dkj&C#sca@89>Eu##3aH}1f8jk<4bRZrZHRUZ=%sPB0z z{pg;Lquy4UmY=hd37eub{s*wDRso&3EOP}><3>hKw&Q_f)_B=%2-hGSm;6E@Ga@`r zo>&(AP1qRMNqFv?O;->)haUh9P~oa)K~4VX*Ggrs8n_u6D8iWE1}G_Eei4>yubn3P z*mWhXGAG*$A{2Lx!Sj=jC)Z)FBu(ylMh>A+F`vK4{Pdnm%4X_inNI!IhQuTMd}xYe zui{iyc+eGgH*t$a(>v_1W=mJ|7|++=4xTJxK3;xe1_z~cmTItyDoj>f<^HMcD!^qs zMrqx?WEjaw`uYGXn8F&c`b3|hS7tB~4Jmq8E$+hiw*q8Apdi(cGk}YfS_%*I72U1~ zxS)hkGrTN;1GnJ6`d+GAw9*7H@4^mKWeSgdjog3*_AQk^rToxHs%fY zDJU48Tba9-FdL-$@lHys+6qLA0bt<>Gxmqnn+QsX8Vxg4`L2JKfU7HLo&lIj7SR}V z#4ihEr+P;GQw`6gIqE&;^hR5>JVXy%eX7dxqPi5Z-a0GA>@-j?jaU&;O}NT_pEFgG z=Ujs|*$9v+t|#TXLZvQX69+bs#9H@W%kF;7e(#!pjM$hub>{=qKvB;uK^qy4|AvRBWD=b_PfjG5d?l|Oj^&=?JW4hbok+-- z1WF_Xp4tL6CWA>PX*9q{Z~dC%2Z*F1yv5h65@GVHue|Ob4AP zw2AjrEx{{oS}OBOO(tsS0IENhH4@aCEEH}wDsWY&4Z%W}d~C(vi{-`x2aVCG0(?xR zw-?D(67tCqJy=t68{24^!IaUVtH@H!xg<5#wESQ#; zlZLf?va~E$*K&Tk8USIKqFG1X8dK+)8DGPDW1&qs#V=Q<@C*3gUVX)#Pj&bME!e;E z=wG(e@>ka*uU>kLx#S}tS7bNE3RQ5p{nP;LXVjyN@;MN%M#`&=nxMTmGd_3kr}(xN zGdtJtvEzUSOJ)OHmRle^(x=ba6X>J9lG)7ag^- zy1Exjdk>0~j^i_o&+Re}^k@qRu0&R6%?UBLe30v@R+j2ON{AdHh=eO%CWaKdi8$hq zSTHO;K?&8nt-|cklXKWR2#`bUdK*$Dba-6hH!~yEt-JL>sF)hlCQK`q0e(UFIs@`U zl~*ukSVTeDv;NEMJUGrYx0=H>V;vTs5*%F*4)EFB?^{>d?d&pmPho}jy3mqjX3frj zHm&B$Os(g(an<*zx`#k5^5%Oj9<8er8kn%vX#eu1qscGMGrv`Rrzk)Cxpr;BNv-uC z#}a+bt}YkSKPdne)kxgg7#er58~Tjw*0ca=E@%iG?R^%sR*?$WXOK#`4B|vg(H?8J z9F=Ma#IyTLm+2XdwgcWfLm=^|g$S~Ep2!}X^o&Ga`|#rAo5GsLUa#ek$Ll}FB4vbC zofH0TPPE+YOy>To`x}NJTr1^kgOjF|Sl7sr`nX1wrs;JuchAWLINGe?5?0Vw_{e@3 z1KfJvYwbd0$L!4LDLtItRS)hZIT!}l`mG`0#D|sks0o7eBr1&eJ)0b3e#7}mO2|Ze z`BX6ee5|QMb02?DbVqOCdiw2D&h#gl&(4_niu0BqpG583D)`sl`u;wk-@tp?EPNR~ zTpqcfufv`9!?7R|zPHQUa#NL$0@-KZDEmF%O~Ok6+Z{;pX7hn&q)+Btg9n1)D|ua> z&o(AXyI54Gt$xc{1oiYdaYvA(23Z-}Hj=l^S+oW=BZD=QdpNqFf^2j48yAI?x{CY( z+~nSQQ(fo~18OiopQ^iaN)DGx&8E%QPgx^lVh550QwVRxtTh{3fQ#(n^e4*odM=tEe-o8fQ?D@!^O5Q^|`Ti7Rq!FZ9Lo*+aE=Cpd#d27QI1|^N5S- z+ujRwEKAJLD>1xpf0lT;*x0qEJ5l$&D=DO_Cnxu>E9l1dsK#=rJ>)LVuR?Y|;H%;* zO6M^rhONtc6FI#vsXNs0+@L$Rb8BE{J)GKA3A^$t=M$Kxw)WwGmEv;;$irlZjxDWZ z9*OhvSgcyR{H*rwwhyV|AKsQelbT=p^dO~<{ec}7?6*y8)bY<8M8bwMjFJ?T=K;qg zZ)7DFJpu+kd+e6h5;xuwTRiNQikclfp-r&m&}kMq$rei)Q=`G@S3X!J{p85^u59k~ zmiL!NxlNRln~#gE(Um1m{}pR{t*$m1MS))zJ&dDg@Y|M+Iah{`vi>u$5)yu?-12W3UQ>yvjkPCjSAQq z0Tha}W!lq@^?55ctfLd7i9^V)K^e`Olpo@%_*xI#m-`Uc)5{eGJPhp)HO^$O&LC%? zq`v}iOU}VADMuqV0D?dF4VF~!L=gXQCl>s@y6p|n)oE!g(20xg>4|Cm*U_^FAg`T3 zw~K`^>v8>p9@UagcA7yp$2&WVGS;6Lv;?`YPRO_^tLBYiHAsV&E3HkbnrF#W%~-9O z7UoIeT>XA8uj`9JLBKDZ--|P$)y5&$tYtjF*f~c2O{e`khB-Ui7J}qeotVo#>=I23 zZ`!S}_fjhKOn5N6xION6b*=6?@@w+FF{!&lj9n28xh=4&4y~xUH&FV005-$ISwKU7 zCPqn*Hi>B!M0Is8>66?$=C5ALOn@PV!DMXa(+w-mutoGtk3hl&4)FDBAEIiQ`E2@Q>V{gDflwAZM6J(B!7^M0&yF#i(ahi=`f;RGDwJ#Hb!SG3* z3KaQQV`>}>0%v%r(*(O19s*IE>(XxqXx=9hxuh>jrI3z29D>S-^9@w}kLioQ3FGu( z65dJB>&Z+S)_|ckA4Ji}QAp>xl9<{Zg@4IQjsI+jNp)#arg*#rA78Km{HbzGNI)(% z7HER`5<@y1#Z>r9T9)%O}4KRN|2_5auK6Z!Anf-(^m^^vVagXp` zYpHwUHJf<{(xN0{zC=`kqvtMuIHbm(llOy=wtj_!UpfSdZ?Xm%oVAjvlT0^Bn2Ve@`tU24t}jIKl@}%l-#Lt&XJ^oX3ie9PYr%wH|F(oF7?TOj z!8IjW3b5QjB*~Eh38_ZY{Ji=b(>8V~w~|3{bfrY&f}{AzH*@$*^?Jx9Yu^LStJK4 zMLa`Lshps`6otOax(&)FaXXM#C3+yhFA*FPrQUvI;oR+G&$2Do*8Xb&w-$& z6=kG(XqZ=2AXB-Drnn9EwXepKJD%}%IUI_#J1%q}p9i*8ISq$~lwY^B@j)F%yQa!-7*%DrC-oEuuChYgi?bwitMN%H9j2u0 zu?c+Tt8OZrc*xQUsSBWyQ}bA7sDmQO=fv36JD;(O*X&TxU1z459G3gwa@31s(ME0i zI+=~7cd!^Z5>hPt5~{PSV2*GhS1-?IVdyQ$3QY&j;Q(0CANSHg{W*$QX$?1X5l7Gv zx(N=`_M|gO*&yoq6cU*Tr^i9is>B4@Gl;@Rh8rltUuhP+KY9y6MR`pUCGEy^shY5uO>1&%n7E`l`WfhoXqu=~?k49Ub5iji zv1RCbzVBaCb3lwID`KL_j)tl8pL$ALMe1m6fwy_N(PQLxx!DC3YeO~JtyAR31=;4ZPmby$^BWexqB_MBbm%Cm5{kVG}HmdL;3(a zU+lT?JPoZ`hwlpO7g@8zZN+vokaIgl_Lnp6tU|vGW<7=B(sC#v9~TPTQ+h!l+%tx*pa-yW1#g`HlThoupLLE=Tp;PcTF-HW~tPnyaH5 z9egi8V98%vN~ZH8LdZUY0zo{p<0-0rEgY6UkojDCfjqZ`{K|eG1WNJ zpr6&nR(cclGA8fYS6SJ%mjIr385n`E=9~jBdC?8Dsce98)1#>d$x>%2BRK_Ekl7}{bVK23_1vxsdFkYcwi$mqN%Z^BaOlxus8)Fv!g?=xdN!?i zktNeJpy4N3Dq@do>^z4swgWmdLTvPLJp`M2qZ6;LSk+G2>glNO4J5ShbXO}o<~hfx zZ`@1Tl+S6kR39Mia4U$I$#01jMh${hq+Mc)p4C?hVAqB8{8lCCBeI6$=`IK1b3ySR zUj|zH4PWZ_>p$978>e*Jxty)n)7-#sZKjzD;w^App@zj$A6aG)bN|sgtp{`5n#r_I z#p;Wxko^RcIX=(RqVQ*SuW)^RIJU0cWEl8sjUMb(Wo}OcX>$2RAjL>&0WAP%@ng>? zrxB^+?06c*HZBOVymk;`kZr&W_{1R|+~kjxU^`td0oFVR^`b2o+o7d*-@sWyRoXj26Hvr&Kk$r!xDd}; z{U=0)_=h8GUcmzDC_XRnxrQAGwfdAleg&}52 z%nF-VXMKX<&)KOfy@uNgPjZp+oHBU_INV-xBdqmpxb-`K`}z7KBepYH8lMf-la{3p zCG+hMjkkv|)Vyhf!0V2DedFg`d(j=8lU5Cs1kr7Nw?b5Tuv^3(M|LH!7r7MKab7Ni zoY6MX)0dm}o)kZh^3T4^^<}hv&*rY{60JX&&9c3DFm5|{?b3ZV?@ zyHV~Ol25TfS~X7$iU`^4A2eIzvf2IqWTf&9Or<@z<=X?+du_t98CIfRldke>KTS9g zc?0d`12M^xxTCLXv$TC5P?9zomixlBypNzhdqFF?{`33kxPJdNi5YWoa8ddVJSo}0 zctWOwK8RbecU68kpC@fiw)&e^-1$W}$}Ilyhw&#;RHje-0=FUMViza%m8Fxtn6=2J z$*26>rPz;GMaQ~Y<8C&kxXNhr<3#iIm0s)El^G+aK_&0=fX+^1x4BF}QMdAvzQgmN z&GvKRB@2jw9#&gXsDwok70ayg*%xqn5uvUu_CkV`9Y(+Y-}G)U4jee$*gd)!-|LS$ z&w2Y({%2iBi5gCy)Rvc5j}G|vQtz%K$H9%m?@uTy@iu+rO+o~CcsU~*lYZxHgSj{U z5k!Fp*a8#Z3vhhRnWdf+u+?=h-rY&Bk7t#K;UECTPlK`8 zGPfO>$J{O~Pu=1XfbQFazW)d7mt`-D2aO5<0O{}_WBr_*4V?eWb@D$2nA`v02>&O* zJl`NPSd2dE?1b-pvqa74t(Q$@LqfGvEin>2k{1F2Vgblh)c(0$<)!}%O&=lKSVg5p zVp>{OR<^qc+(R&S{=pb!dkghIM>7~j#Ckcg*J@EmvWkJVf2(q}@wKEy8S$j4pRZR_SWg>iW7 z!qrPxy4z$QcLq+J9Q|k=pvKBKQ-nosQnc@e2qw+rNkXtyzX{$BK&V4IsfF0!X0(u2 z0#WTI4OQWh>z@Z_4eOz3pk&cxbPeYwN*E!Dzxq9%-~=k^@jLUN8QM?d@Z#u6QH1?( z92r0EuAHR!aWk`VvSms4pq$*~V`gQyWQ!B)l-^!)v9f-IzucTY>?A1TAd7QhE}?J0 ziHM2ubn&|A$yg9HO0n|2rH|&A(*`;d*JRx#M{M8HO(3blD=b=~oVLbrXS4v4I5mYt zG;7Fa3Pn9VCaJaKL^S4zH59UWrXe4EByYcgqoVEPnRfwS+2ZitJl+^J01?_i05cTk zWcjm0JmAKS_D>eVP2%gP-|n7YhgXIVv-ej>Y)+PLxqgurj@YD7RUTM&suDYEw@eWy z=YhWAc!kA{kLDc`C)qTHYjWXRvj*J-%R!0+_UgqMSWm*~i*fZ~Ptwi#ODTC{d29yR zQ@37TUXK|0NC>lmwdRV^8Fv6^z|WK%Tf}!wp}v*;Ll%y5YHci?%kYr+qy>=(l9IOP~MuuT+#$XBTqxbG;le1w{ne$60)5sBRb ztPB+7xQT)=3&1cf8e=STru})*XjwFb-&K4bq94Kw_}`gg6MK>a_}*|BzkXutv<150 zYrma!{`2iv0irDQR43nxoVUDL`v{x3#y4SXd_@qgtoR@}b#>QwlNN5x7vlvo)Bq`m zlf41G-%m*gStzWDp>YxHR=5w8X3V46O&R!JIr}?~^JqcsD4=6KbpfLvs*M=HrI$|t(s^H|7fcVB;mj^4B>BZuF_z{}Hf-;K*+3qF= zpee^pG~{}2Y2x`2jwyMSl%m_guNmfKC3tLb(jCBe{_FYMaf?99#Kku?{oomI^@Geg zN$nAYr%;ekYlChr$iq}L<_<=WC+rrBwaaIa^6ZRb<9~~VSkOEOSE4BMneS@OLt6XF zgY;nNlRR8gerK!3`o-W z)9RO-7N!)ipm2V0s|IMb9O=TJ6~PxQ^%B`-8`)t5OOIa0Jo*-`)S{kp$V535dmK-+ z8PJ*gEcZr`F-}Dh^<+;JBAt6xCEUF!v_H}w9^RBi=vfLtq*XGy1kgrn_wA)eQ$MSO zZ?mB9j(08WKqh@qQ2}EWLo-k<1PrDWegFD{jB^|z)b;wsuIut&rQ;MhyOl9a+zf*0 z64?3YEtb25PZq#;g8rJvz{0{>kL5mL9?f>w)%6(cXRrS4J$#0z{wD2D1wHtG3a!|N z|3QDQBDh6*yWcm`fI%o>H()g=21zom(`DKS%x%^zUlgO6ZE_4_UruUGp{BEyH+eWs zEC{5KPg_~ny2!Lj@E)QO41M5REsYIUk(#9bBTD9o;fa<1j^s^$9Q z?`7m?JZXv%bxeX|30nJPGd3@H)hJxh2SujE>sC`#fWnX0eqfa3=UZo7!M zn`?SoAqBJ)oX@=dThO$55O%!|cBU z(DnhIv)LEY$KK;f2u&Sq{T1&_3_$t^p8~<4q4iBxdgA6xtL(SiF!^T08)3umkF8+E zFMaCnrg&m9xWvx=Z&RPt>TJS$SP#!>z|J1omEZPIW>p#d5v#!T_sGlk$hF!vN9`80 zfLFYu0#*^U2|hR_**QNi%(-umbZ|f+@yaRrGjUA|G$rVY1e<~e)j6^6*1>=hU*51q zwQR%03eO?ujf+w&z~+T2Rs|9;IW#f6ZnsfnYR3IZM6yCN#?u|g)XtG7j>Zpp6gUGV z+&5t`SPmw{Y`9G^bxP_7Jkr9ykzaZ{SBfz3jMpoLkskvl*bun)ooNK6tMSwPG@n$X ztZu4OX*xA6!Bk(NQdcF|^xIx}-)wsn($lcL- zblZl8BuXn-Hp+O+S1>3vH(9!7B|7&VN+Q3MU={^f_pkDUtMe7&d_fikkBm<|VQSe| z|2AGeKugVVc_KmpX01JPScS4!Wdw@3Tho`L7ned=5^e}xiO$ab=-RhV_yVF#L{db9 ze!nxyeNVFfqP36o)Adg?{Ec-hN!9hN*}o97k47pL+`vZLzfPC$UP8{s;!y~)Y%>46 z>oS*1d1=BEkzj0>SajdpL)E9+v`vM0$07w2Jf*m-So(csvS+eShDxrfV#ufoLVOHt z61!b8SCOL_yIEjSnIQ|K^1C6K%)6M|?=aSigDXp*+xC=63fZ3p1iqNvXm^@Lh%R#v z^h@?TIv}uEcJY;$;HhgT<{4)5#ip+*aeIQbNizkseBChv$1w306S&&Wn*#eX&}>Rb zqAo)w=B79UGp*6ZziMHprsL9Wy9zz|hQLjtsHuuC?|`GjzbwY(6E{_WTi8@qv%k z+L#4l-=`$3T82OtQoB5uznF#&L8GKb90q|S`1@Ec2k%-01-a5o*7shi4wl7-!Rr9DP0yXZv| z{#(vcVZA`fj^8&1TN!ZuR2Qj%3 z$D}i<-*I{3T!ZvpHFUOtJ(DTS`*C`)I`omijtP5fv*NF^T|9z|2;hK;R1=?qs*eZ; zFO&*8jgjUrq9H#RC~P3jDk5=+RGbq$ZW+s|uOPn&^&XvN6R53Irck8dqa6G=nDdKJ zn|C&S_%0%4hDV0j+l{kB$Ejt&O)U)*cHP2o+)NrG_WHz#4^o=xYBWu^z3m0o;< zP?qEJSW7zXp97}$8Y7wrMu8r33LyT)pVX%=3y!A>f78I@^tw(8MmWj3VwabVNg-OB z3I(AHm`gvAwLv4zj^5pZuhRq<7iW>-Ukk9cs`B<^_vZQ?gDwF%8#Q=I`Z8=VtNX^hB!!@nn%guXAGC}F_zLUks?RjV z(|hmLSTFgk$B_Bi*&4;ZohCnQPOv2~-L2>dhsdn^vdeEVFo3x>RA|=5}Fi zN6H_qa%V&{IR~0QO(%IvkUiVeHU5lQ+~N>33{zPut@+JMQtp#b#naFfHs33DS*ibi z_Kgh5a~ZO%S2Ktdv}SEox+86GjRZZkb-wM)KE+DjY8iahz}w+GT4KMjxrF{{=$CzP zM++)Qbx=DmLsnt6+f^4jk{>T~OIF#~@1dl%=nq(u@C*=5`#PY2(W()Z!RbOFDm&J(3WGE2;xraRWWmsEQP zbr=5}jjnMbTnQeg;7DrIjV|Ih26T{sxMgelB*$Y{WH>ZrTr_}vtJ9w)4B0@QwKk#d z`>1iL&BP>M;i^2J67O7tTn=%eWS#@>=HObLhP3_;^AgCEQEj)Py|1^WPSwQF|?sgt_L1;nP7`vZ_>z zJmx`Xs(c;~KXuwhHJ9fJ3tpjnjPiVl#M5m;wxjNEZctvg{A&<;zejj^3JL`mn4!WH zvNlwPmqJ{=-z8(vK*7U|xl~k4)hajkVcyWM4qS%~p}o#469hsE`-aNBb04pv#rdbt z0&-2>Wp}ZCTwcg3> zE!*pl(+DTm*JZviX6QBT1(9M-cuo7;kvq}65^n<8{#~SKS&iSui<&W4N*;@p{iESo zcljd2*Iqhb^}PsrM++{!ZW@7ZF;XJONLr`J8A0fVmGs>+{YIB`@>O$trsd4(^m*ER z9){U2h0@Ubw}?R8&O-Ek*|qK&gy6jSF+rJXa=Va>Gs zL{VncfdU`CA5LCeUacR@zA@CFZHWP;DJG^SBfP^8IUB=kPL>sE4gl7A=6qT^`kzCN)$Q)=-CRTeB@ca8~&Uz}_NC$&-<4Urbd9 zEOS(%PB;Y9Oxmf&cAiPYk)rd&c58GOnkX7+g4I&GGAl&(PYbE z!nb7TixcPf$vAmBxw7E-MS+SE>m8*JWV~t$QYD2n$^sG@Wnw=~p5TzeqBo{54?MHN zkP!fSUsL&YaZQY=Ih%*1^9osnYZ?D0x=W@>^}Bg_K4+fCnobUT-k=KkhYLgN1R7B(Ff&D?yNw~k&4kk$% zq0j4&rAt*|nq5r_qfZ~`)3TJ#=@_W$V}QR}aXlI8%cr1Tc8PV}SeX~Tlz_#0#l|kT zH49}tUF+jDVy+c*>q?a4xXk$%pwa5&+E`-nk)8yTJ9Ou0=)Y5&Bj%jIW|m_3_5PXS zS+QcRe~#5hDqWU;&Z1UOCB+$1(AwU?C%}HW;FU;J@~$uz38+qZo^N#m%O}o9-n=Er z$`+Yd%Z!azeJ0VbYFxCR8`ux-h{Qc~3wloLpP(o0IdCxP@Au&x&XD=9>B#Dwu_x1z zdrA95F9)RHl#3$mjlF0TFImSVJ((o_8U%|kF#f!s9XWdMi;=J*f2Xjfh-!ay$-Pt6 z_VD#Jgz2*PuzSeK&XyW~!9&2%d*WgA+x#S?q3^9Eoj-xNb(q~aB7V+Qj4x>dkmHY{ zFBZZ3z4|{Fm-(=B`Z$2n3tg3SjXyuZ4MFFL5e83)~3W|dgIOD0sYbq?SR7kh$34T3+PYXXrm z1FTRIUOVmqqtmet-l40dnZyY^yXu)N-$@&$lADREC7#uXy}bLe>=4$a1NER%x%E66iB7%*5vDeGw}&48w|v)eIBk zfyZ?B6mQG(Mm(=#8PUV`vVPqlo!6kS6dgm>PwlV`owR>JeSI7#D zuKVO*o326Hf1^i(fXu7Zir361a4G-9++wt#~X9DO?oh~ zkbfU4UAEb_oH;b;c9_H%9Ab+Y9Cyw`{_HtZ_mJJY>A}-xPqbATDf-<|#OVy)Py{%K ze~N(0gXH&Q-hXF$ldI(+PrK@;t^E4?-zl~94(OxA5C8x(SO5TG|Jjtp#mU6c*~8xC ze=w{pV{Y0Wvb}8U4V1#Cn&P^YoeA#~33N@+Fs$ul^)B62Hj8j)k0*u}XSWZ0dChVb zizqfG>F=`jOOsq|-G(K6wkm{DGA6x9OpUEuS7q{2x36^5q0Vvkf2wcb|AAoL)tK zuPqsx_F0#dOK#`QD|Ae}cE|MlvmgjwKl02MZqTh|v4iWn2yx5Yf%6-8@Pa*+$qk`* zxZu;;#0ByL3qca5k`U^*B9nFK)qgq z%3O{iEQE1R&|KNL>qLMwp#MmoV5qIl+OW6p>MR{f&@7-)bzMBVp5GEYTkzmQ<=$Q8 zX6^4H)_u&8pLv#zHiI-iXs9tJoKG+Bbkhh%2>lDSztXcDE{WlDaERa)6959Cj5|nO zwiJNzdo>C4UbMy_nJ&A0wTrWLDtxMo!wARPP7Q_0Glp@wAMT7Nz^!Z6FUS*k9YXYX zbzJzKF4nxuDO>=%FbaE-8(aVlg`4u>vJcVo%>j$NOw_)ttujhdkamV^$Uq;0yDa{c zM*WX4_Hnp$1AT68?$7)AVp$Z7b^R$>2K^e(=iE3~NBU6Odm?Yovt9v3banjbGEgQC zS#RS6zL-uXn2>=fx?A^7!~*AIw0AcjS=iqIq_QA-7P`ypI+Ef#5RoZpoLq7?s+^3H zH7bXAnKD{uqzugE+zrf*?AkasPa{6@KobA>=B9T9s5c~i-x+_uJ@ z^;Xd5?MC*PlFQ3Lg|VawL@y3&WNIG0&tT-Gn03G5`y9m>EQLcP$rm?&LiQX)kc9Xn z;$;R^u;qR#hk=1NI7wy`G%-vORtgDNwX!mf#dd#b@#g_W?+|914BZX~W7$f z#Ep8>OHUpzzoI6CXlV1g8yk}T~#?+Sx-3<@z7WvhKvThw!B3zElqUuYn7oDu8R)%2Ln&mz$gSh2ug42VtO z8!Is?H3!0dQ_t_ZJHIY2k3US>BRa<7CpE>Vf3tpp!O7p)z40*JuO{PXjiGRS9G1mW z5p2mV`|rGUAY)7N>A>x=cU*xmYCs83ew(Lo*OtqnWuHe)pwVeH8|D8+{r{f+t|5YH z<9CjFtL$y;At|MHUG+O0V|4ELZd>Pk$uK;Cdki@w7dQ^b}#bcLb@l>12jbvl|V zep~Y33#=pPDz8Ml%4-yLEst`Z_$LSAHLlR{Bs>CmUsETmi%({~Z8DTI5ZG zky@gGSo8ICGV^H6Tu5R~0OJz$xyn%H;gh3=JRG_L*JSbz>W=r6!gHgVWOpDCP|mud zSNutAqk_?WKGa}?ogJ4(hSaGBR!T!VC@k$bspafGD>@1)%Xxd$UL5XyZkKXbec*g< z7_TcdrFk=|Gu1M>DoG!7Y-xjqy8N9duon2Vj!cHvZV{MZ`17jaK+j6O=y4I%F=TeF z!qCSedf*R1oM$uhEx`A>O24h;1_BF`A=HPY9L!NQ^VbT;m@c;~T%83HBoWv#4$mU8 z7KmhY_AzYFML`cg(c+VC)7%m3Cx-T^d`WQSLdzT-Icid9JBwF^tZXMiLWs~8ZLLT* zjT3}`tcu4ZF>cBwiFD!wDFX`$G#q>mS~HGP1%bEgA)PxSI@syvC~RPS_2@PLaOa?# z_^2)lycD|*J~F+v833Uhon*LwX_m_%4o|s`_ym)^DZ)ZhO-iMMt@j2QH-6d0jd+`ju6;K(&h@@GCAIwtnKT5P9T2u`> zKM|H2xL7`-$?VgYF#)3IL!C4*($kpPeYCF1G(Af%~80;%PNMyG=HPZ@s*OJ^k<{ z&c;Q!>r_JEMv{@}7C23E>c~O_^9aXw$zn;xhHZmgAGc!T30KADPr$zr1QFfbyR+YLs;}5ResARYD9Ij_Ocem#G$k1TgNK z!SYU1bD0G`A$lIWaOp1ob~QD!VnxS}|Bz5Ds-BEg-3gdI>DhX+W@+LOD;c}vf!M&A ze$^qPET>lbsD={jIEZN0kdnY!=CMwyh$|&%s^cRF(UaGS7O@IU2GwYf8@7ncL_^-6 zV1H1P7&x*9jM@D2W=)gU5sbPsw0|94oM7!{bo=Ppd*E+>{z`f`WX zpJ>0hGIU`Ah-KPg^i(qMi2n$pJVd``I`o1PsNuy|xKlYM80tDIG=YYNbRKFNa>+aB zKnY`8An1lFKq){?Rb#J9QNaajS0N>m)Y;T{Bq9Km(3X0VQ$**@rXQ;&doyo(v)@oM zgQ+dj$nv9U(*kuD^r5N zhu_z5MgsU@${h;a9HyFz9KKVY_#4b-ReD~=Nk%kadf%#mayNKK59j1h)ZiW9+#(%+ z#oT{=Ktiay0-6Z$sEP6d0osCUg)W2P`}@>&kAa6+8(jnIEoB$u5_~wu@eYyu@o&1U zX+tJ{ZHxe75uiL&m3hhPsPR}oYX|JdfYE+B3nO}n`-cmEA|keWr1)lheVC!@bS5zC zgrJp`yBAay(S5jI%kiJDU9RGK-8iIg9?_Md&lqCtm()y$Kbrfz+LGF!}qL1W&A;jRsD$ z1L70fMvqoeo~=`mNEj}r^>bc=!d}h`|6yU;H6Qxw1#n+q?Q@V~Q0#P7o$}Fsqk#;a z`rm2-`p&gqS3VS1_6)Dd3*y~?Cpd6C#Vgz)Qyf{j{WTr9F4Ea<>^^)Ub$$BSll37@afhi4j|VgAwxRtdOprVfi7cwhiCZub%&-ItU!NO!B>9H>-MHaO&6p z&oDqv5r7E=3;;k21^~eI--H1dR|lKl;?%=LRRt0NST4g;>wnDE0~!Djj&s5Dn;vA@_dHzHv+e?+3zB~IAiSDpzU4sx(>t#xN} zQ`-}Oy>vsfc?V$&3JEiE6#|L&a3`NVpdzUp_e9|mhjtFkPzQ3{|3eC-caF=H#V3+~ za1QwWL&@J|8ireWQ0<=pcF+c)G6i@j72>U`k!U`PL}QHU?rXD2|5l*L&* zv&3IMA6#=uG{#Zkzprea`h9~QeE4P2CQ1D^aO{nbY?>sEDRtKd*?a>pB;Bz9>tuGp zb*WG9n#K2MM%P~x2)hszGSp+L6siRzvn8v|8pPGp&|1@n0-p|6SO^jTn9Y(z*(F}hK)`GuIPwn=pCsDG_Je&cXi#M!15KUv%`$TsH z0?{}D@4V9irXyNHHNONU{odrh4q*%8J|EA&TrP5GL#V7b2{|Fg$f8aErhlGQrdOKt z045*}Tsfcs-atA9$1y`&&@MX19XYRYc>~)SUohPuVCr5Qg!1hYlHuGen46mNep2iV zCtw`}qVK~jX?eMG^T^A`DWqH<1ex=39bisd7S3lp#eB@YnkMZ`k-Ou*qZi#P8+z0` zEd{>9?B3Upd3%iXMfu;5#_y~VIOFM_D_L%p5r5LH-xBg4`3U`~wsb+wy?FUJ{L7+* z;DKEJI8aEr{9g|2m^bId*uc)4M=3nkTXohA4!(D z5r--gt9a0+4gv5c@pMf>Pfj(X*Pt2dqJ7_cv|1no7n;~wegm9{zzqKK~teOwP9BoK%A5%#tgG~9m38`Gjn+AF2Rn!=BD+QVbjK}%C_To zHZSnat68G;+-tZqo=bQJ_71;|2&87O{r$JazZ{&gM4ASWe^2n*kY|Y1k6D1e!=h}R{m617Zzv+k3s(vS zf8x2Go}2u<)>qFTx7j(6T?LC-0$sykl&^B(n17%;F5PiyPlM`Dp|aDRW< z$-l8c=)dts6VubXn zK?xZ=EO zjuYc3=s>Bq+a7~K`dv6@F2vhPRE(b}-!%o)97f5mmZF$I_UNUGW+IJ|AHE)Zwgb;o zYi+HXu5F_;^oG8?I;mVk-Gr_A4qHB9_Mu@@rzBVxVR@ayuFDgFvL|HQsYAMx5U4U2 zY!Ro^>aU<*hOHJpd*sCuy%LB&A*jYLCiR3ZkZ?qZYrj1fRn|;u62t$c*RI&Aug2T> zW5sbkH`4!6FN3sW0Ij2hsZ$_IoR0ES&@K~@tGtFt@t;2vS>Ub=MT{99!vhVs!_1lQD`g>@Gi~0w zYmYo|a(zMuOSlHz^D;dv27O##WEENI`~iXRpb0%Z1p>QGn+Fn#>**Db=`%;1^g(%7 z$?m3xxf-mizB>t0rK(>rR;b*bN( zJofY_q6;QjBNl!3DH!&p!R8%r>P~oXehz-PqgwJarFA%Q1;|oRq4DFlma*?Me*}3Z zl(8cG?`cYYDxa5UPPU+(Y%Q3jvv1LLboc~^744e_ZefrAK5lcj-KH^RVtxarwjXf#?^nkNVEB1@ntOXzP{eS(<{drq(-zQ$)ZQuLJ}2=s@7&W zT68;XbA+*9Pq|MAOSlb?QhW!W2xjQhl&8xSL@C&Atm z5=2sF={HJmNC486w=52DOph`}VHzkyQInyK!8Ql6>=QQ^l0gw<5%dZ1pPt|#m_X2@ zK-JC3sZpz9V-(pGnwb@Xj+)(p)8F2Xzm`j^4(vT8j5SVplpY|lbN&m6M3_dIK`@w5 zEg(VQEU8!;a?+6`!^meZNMmLtN+L=P7@ca8ETRgO$~gG-2M~j5VE@bvWN;p_s|yoW zoVMw1^7d|e@I(Cd_++~?>tl3(8v<{qzbgD=bN}#mAMeKtmi+~Pe{}!L7Mdaqa(m8| zgw>y9+y%li|B$t?NyvbVrhubRnax;v3q6nAM9z>-iHg$>oQNuoW(6FRxEr7*#AX~t z@nm|XbR!mw3q?tAYUNF7+@umKR!qR0K`&|6B}V)~CKBtC;I`d=8Q_;C@?EZm zmx-U%B2FY*)#4KWfooq5A?cWe072grf%?wqr{ZB;fK)Z|NNmf=s|~Q0t`~pP-7QDE zI>8>Uhs8Ixhl75uZ719eZM&Q1e{S?NT zukCSq)Pru!e~{)I754VEnsjf3XAy8CKqlw zrVhfsFgL3=rTB~AoC-6z(w)^5|5Bdm9s5)!(2U2q&PEg~7*PpH=+<@3v4TPW>riR2 zv89q6M+$50&@E5xlPk8^lAs@%ebBY&0bRJOsM1;)-pqbQV>sL~VAkEa%Hz{pOImiV z@yt2o^hQ=6EqTw(mkX1IM00TK${l!C8YuG+^3A!_Yh3>pX1K=fKFh9)-vecEwvq{- zrfAOhK0l2b8(SL~2(vHn-g(&1;&XwTe21jOw@$EwP0`*37)2m8EcedA6eLnyhj7$Y z!H0SFNP-H}#fpbQb}=1U6r)ujBMPOxqi(safUp=Y+L^V)?{M&i)w}-YK6`sYAp9#h znEu@u0EM}CXv@m$(CKu2kpYDHFQG?i!-(d0*(_XV++>H4)Ta+{8U>WhNI= z*?4GXdEk()T$C!y5R(~?+IRo9?Yw=nU^MM23Ky6nt`-CqG;g*3PdGQNvl{Xrcs|6# z5j;~NW;2RlKhh&jK~{flcdlus>kgR3cH^AFVI*Lt$XH<*-PQ0$=CgiC7iiY2SZow?9v67vG`b4Db9&9XCp*>5Mi@_Ii)nIy~L=(WW(2doNxznFe zdHwnD-8MxVHP;Cl_luS<$@3Su^;>t;D4T74ZqnWVesxgff*=n?>_)7R73qb|#m%Bk zKU1ei^~ZVS@yo6pahYJwZ@f4$`i@0QG3Lxs5N%Q99b--ZoI;pM_~hp^Rv{zbkb9@@ zxBW{2ii?Sgq6uO)W>PcmW_4FN)%+Ff&ifQA0Zyr}{r=E?vaf0oI2o`K#RwaBl3A5X za{NQZt2^MYx|I$>fgbBWhim1bB`V%v)>|1784!iD*alXWA2mnVGqE%BTC7i%##C{6 zNh5oi#P2?|ZrcRmKK++>izd*TGDK)MCuNc-RfO=VT8Y(P=9!~^**R6KIOQbs-Rz*W zgIarhaT#?bOO~B@!5-StGNve2s~@Gh0@r6y@@-)dz_5Qu@LwJdQCLa6!`T3R;ovN6iLkksU)#W}G9swLfy#ekAEI zWn?3hNggFjXO8~#qKNhBN`*1%zlUY&wn*_+)gBkrj%JrlEW!2U4nY+}1GH6M*!S)|4=^uYFIXhX6bJR9YKCw`IJVw*1z0VCxJ*)E_?tc@AHrGftX z0eJwzD{?=H$8-L?`YqReVOhpy+Um2vUJH&t2Fn-LVp@;oKWy$lVE-A^DEvij!~g&Q z6n;(czX@u_o-P*lcKWU^mj9#T`hQP0rKu-oRma9TGHMzrDL{Qor3M1A@rSdsLvWB-ApUmkc6G)K#GnY<+>XwM})E(IuLX4M*Fl(BSV`T6<-*QD1v)}8Ac^q^Lds`qZ$%Rc~ULi zBw-FDBx(};G^rT;qb?>ad~Dt1Ui(~GzaBaC zDjNN6^R@Y&Raj4WaenP%kEz1&`Y9sO{*c%+100d72KtXkbFMIU)w2*{Im%Rc zIaFJZ#&UF~+?S2MDkQ9A67fv=+k_)aJm(SPj*pJ@o7;4oR>>A=bxt;I8n$(&?UJk* zd;J6x`PV;YlNdwR2W&1ac6;U$tknqecGy8T|j}EU`uRGI1F< z1-B)?Dcn5Jf0x1;+qgQrm^%HxDcnSuy!QXt|L>r!i|1Fam@R4di&|j@d{P<9j3;ZF zbjQgGRWsSKM!$h2WBOPLMMMb6okD0ZoVBBB(!SRZfIvd=FL_orbp<12{=CIwXV$k6 z%#&9Zd9*1dohjvL$>$Qfx+cnRU-dDxO^zrPl?qK2tx6|FniR|$v!CCrnBGsxGx1EO z3r;T*PmY64c(lwP7(!b&GIc9TCasVPB|rj-QC^93sZ63qI?;q^p~^LLj7G^QIYoM7 z6cFCR96^4iP6XjB-077rIj-H3l{7LC1J9(>R*1;+bcAVmlCp zqf&||Q`H1m0gai1o8>%!_(*tNRVv!Pnltp#yh2R_yS0 zFviZdo}O&&wcY-CEXLM1&9|eg_c}V%7|3S1{3XsEI0-UIK~$oiaS0a06f3bxwQ9;# zGQ>ARhput`xZY}8RY4rLAn}fjEtlH{!SdKAv3H+k!P(s z5zMKo<}nq_T@x35Xf`SrXl2-^kBiA}+J&;xeWgTMMbs;f8Hbv)W{GLGh^hWoUnnr> zzayH;dmU5LKJRwcWYby`pE_5r_u|6JB~SgXb}tP1f&K02uO|zpAnWz9UoqS>{a_nI zK;TI%g;(z09Cb|}v-sC&?b;&viG5`o z=nw0RY7}6)$fz*~vooxf$6?@~L6+<2d19S_&Z*cWd?!|R@I&y7(rX2(h z*dtxj`DZO&sw%v|ZJ{!kfQ^TEq?>Ek)j| z@q*_yMi4}QK?3QTTc5yEmXA4HX`sh8&}MX-IWZ^ zeY}A;#EHuI%Yj%zeSPj-h9q~4Fb`=jDs5nRAWwKPFrPZrdM1yEZ_zXBX0<)QX_Blk zt6m>gwojo_{y3SNU6u2>`^uQ~E+1phS$>2Le(9j@TI&X>nqGZ*a`@9vT4?SrSbSc4 zAr%_J-r$S91cNIa%MaYD$BN%ho}K;zfx~LRyUm;bTSjBBwJ1xF!-u==%TQ%kU2kR$c$9q?P7{>?3$8>|9;@C?Nw= zny{BopSR~+BS|B$u?r`P!~^lA-ov*g=3CPWW~FcjU%A$@M!mhOqF&Iqi2E&Ov2y

lOp>kjICU#TmeWX=L54nAYNFXS*;D*Q?iu}Wg?G=9*D1p4D66r zjx|;%1biKq$B2?ezsv3Yr`C>KO_=JqHO^si6k&Im9-|0Isb-*s=bR?5x*qY7@;rFt zRFYY3n13kY0E)FyJp`x#?%*$UXZZ3NynyH#JaCgx^{}UV#uqqifM1>O6|ApjESNp8 zqv}7LP`iR64C@6TT(o_mRn5q8*sy{zGf5Uk;eHC3nZ`NzxVVAJlmo>Os9f|FRD-EA zeC4#+aB2O|TG$G+xYav6y}36^x5?eGbvv?l5B>QELI- zpt-pKD=0Bp3}em$gPO&$+YyIt~5N>&CyWJ(-@hi2! z5AW^Rz|>vyo*-hVUzAhID#xGLg4Yycd}MlH*QxPbk%8tW*+9B$U?Xd~M&8BQdjg%W~@#zq>Z%)n&AOjT&2^*ev0a}=XMf_|iO zy5FCl^a=LZdYo@S7{y@$`%BvD3CS4rJYjn)yU>%o5yQ$w=)~>8&PKL$w!`3y@ONWJ zk{M~Tbe$-v#-k_;0R01uPI2@a z^a2bsaK5(>-N&sA3j%ILy2>`lw)!R|Q}&ZwJsT?kNcF>=1GGh=1%yPJorcVx z(jyQa8Joz>&wK|q1hzy}ip59@s?>raO@KjL6B)3Gg9;I+Ge!aox*?Jmqn2(Wi5b={ zRN{Q=R3kqt&RtWoU4X>XFjUjG-00WgWzTVAKxrAmntF_WQ=X@g0X z%yyB(#RSX>$WjN3W9ePCvNcfk4np7~SS?`VJ(?{jun)dgm@*7LMYRDN_tA#Xl6?vN zI=u11hV_7^3>T1}e=?Wln6R&hw)bu(K&pT;r2J~fj=s{7*{l2x)#8AjS|Q89PU27a z0NUnFujjM2XCDyGO@fno2KT3efig4`sJU)Wk2MCu(v^rzA+>I{P+g@_Rq5H_!48GX z0E8)S5QBs`MgYp3>3wdq{lzP%fhBG_Q)7R7`Q>u^mxDwdBK{gmKHg_GA|kwl{6NJL z5lEjCXx>m07SZ{{D)78)YsL-P`y{me)y~4hhJ8LLCYdP7^OKt}bD3M=3Z!+)3-=jQ zt_7T=R6#JI??}mLxkEdS!~inc(`aEnT4mNGN(5;X4BgB#MGiP`0>P~6{dy{^F9+S> zbj!)0Vr!E(j5psAY|y2p#WDO;Ix7Olm9JyIJlH+^)Qx3l6FKHZPTtpVoqJNVk}G|h z70uYoW)jBjtTjY{38KG$D?H^lFWrD-(8LVCu%Oks-zh~6vJQCc1|8CnnU?}2&`C%E zjS8Sd1(|xhfojn+(5R;nJceXKZWxEEe+t+&w^X8WTw_l3r6NA+7@&ehbu%ahwasR` z&njJ<&8O4gZe!{qR~BLXGj7?1^MLRJSJEXi(v+*k$zMPhvv%fbg_9sOl&PRSAr3+H zJ2%RZ|MvFS>gBd)%rQKHXs1)Eoe~Xq-}aRmflDH!V%{@|qP!~*>Bh!mpni4jwv;cg z3u`r<=fz@SBOBU8_qWMLvEW{$)DYc{yOpK1-JgW1Fnr4E%P(js61h?_fa>qzu;ttd zvC&%n(Gh+7cz;#t%b0RV&VmRewU;N-C={k#cF}%qmE{OQrKPoUcLNkuETq~OQOj$$ z*ve7UF+;v#u@L1gv{&TbVZ%qH5=LS>FlM%#ccL7lT)=9*s-sj7afl{^iMT>v-&HUJ zwp=OWqhysT*%Vc^#KE6M%3AwbQXE6H@S8K$1?)c1s-kox5E>G@*v5Ww(I8%<<*|#< z20cSwtwPN*DJoq!bbj8OqY@7B@DHDji8N4HDp|u^#i50{;AgScKaAd32I72|1R9w? zI(vS4{|HpFUs*VEMjg`jsCkiX$gVwxOF;D}t#a&GkjPhrW(b;US*lCtQ zm70i~JBn!NpR4Y^^Uy`)EV^TRGbq|OSNbG#!sEvnQA=C6hWPU@R1-p^1Nc+y_E zp+2>K*-4vma>@A{nFK$1H8D~byH@L(0 zX&@{BcZ01Z&-059-hS_2PcJ`FuwZr}Z$OZCS*L`^n!qxgWhLf`3^@qnDcVjMrcjR% z;=x{77#iKWwnBixLDiJmu68~e6B`(F#sm0Crl8X2HeK4D6o{OD)s1*!OLMfg1CRH7 z<0z=}Z|(5L&{_YjdBOtiPns4Pk>Tg3eE}iO>~O?)-8QSSwQ?yeiQ)^XEh(mj@`pgX z;_bQ;lvTO3gL@ZlF?$7RZ=fup`ChWM4Bw7r41X-Uj>i6Hmql%(_2KN!=IxR7gBwtz zs%K|i(l6FKZMI@PjBgbdqHtR>vv9R5z{?b`6sB*mGJo7uZisg2bJVG!R~FGYqf2Uf zO4suD0{<q46@sidN?@pP3mKNW6P4 zja@y=ibPy@C+^PnmX0i0k#s&>_{nM!KGxWq&vgaBhRl1v-u7P?`OsrB-pveUvhlH| z=muFl>>+T>@_)aQ@H$jG5_^z}(nys7Qo#`FI(C!kq~D+^zDLh%d|tL| z8ncoSwwGG(wdc73h|$w3U#Y+?czfLZkTw94P-yyGljJeS$ZHp~Fik^v4r;{GskMQg z?U3M+3h6_57Ij?!Co%g!Jbs!NX(}BkOQwAeQB<>x^Kn@!nEAS-7#y56Tan;6#*kfC z=s0(~^a0SMTC*lQJ)ze}&<)b+_a*Hc@a>duBFPepx(UVq#vrq?ug^>a7p8nMv7|RO zi=y-|@(H0KDrigb7w2@`)DJGL;<}7`PwSBd`NH}2v`4ZW-|}Z^qVVUYP!fSc(NZ{f zg))d;VM7F7d8Nj@CruiLOqd6nH#zXp`O^X=*{0zRXFA3zm#C+Ri+C@r`09OEcPO=a z_xk1aUDw9s$1Ano$jKlJ(l`5Ao)^j7Q!8C1tm7%qt5Rp?b)2BG-o4GzRT;9@NbXX{ zKPnbh`WdAaPgF%>ON%wi7jq-$iUFujMd^&m{)qaML2Gl5EeWMXp8xWv2{I~vEI}nH zpwB_ECT7kuc4yq78ewAEquOWGwvZ_wmX5i7R3F&X6DmtWW$)cWuCZXAgSk`4YK&?R za1_t`6Ne6kZLU8GsaSKl+E|>`;zty+@5`hz{o4bfAIz|%YAtfI?y8e!1H0-P;64gU zM|n`8D@TNyG1TILax+xB53Yd5{&Oq^ifQK35rQU9Qh5f6#|?yxCidr{$h zTmG6=0MkWHMYy*;yM%)oqxv?Y3%CKhC{b@*r2eRjjy!n${x7kgnb!FeZ!AyTO}$Pn zUyAr)84bszNae(t4DYY&P2WmIX4*`ZB>Yo6n5^vLj3}OCIO}#rgMOWmJp!$ABjnji*O<>5`8N zm$e;YS-X}-#<}M=PB(5?U0eWf2Z9K|HtF|gr^ijQc`NxWyN4~ALDPgb1I3nUJ;>(M z86gbMPDW?b`Cg;b%(YZ8I5wgg-C1A zKM(uPSTeX9K<>m242N=97u;rBSJ{6dwATCi{1vrS=XRsxEGG50cTWa(D4&G%59;Fr z#!(qQJw0%}b-k_3T~N*YrnPXVlNGJQ+gcoE;jL*NERCxTe^~;i^TKIMe6iqK6gaM& z>Y%!A3||!B=~kBo!j@}`7!B=kC7Xcm&I+s+uR^E4Rwiv2m9Idj&kY2s7LcW;gjQ^L zTFH0`1p`m|XA{f@9vxqxDEmfeoSC9sw4{!Z;{1r$<#E?kdR5!Oy)H6q#~Z^Ff4 zck>_DyM#^KSyo7g6NG{3zp_QR1Q_VGMptXI!~9rhCpx#hXq1kiPv<5<@o%1^Egd`z zf$gD&X&u)kef*QXm-uexf1$7M%T;@-Rht**U;}K`mdulTo|f4$_xU(OmR1il7_~sC z%98%Rz|Go$$;PXBq>T5~<|%8Oxh4i?(JiOebp^}r^_%APtsM&*UJ9|?8UA*wC6c|# z?-G|Q!NRoZC)yCbh!5jrGP9JHY{q;QS{)Jw96i?MH5$)K_4W%kKmL^^$?)GG(|^Nv zJG2aQE3}DT_8V;t%C%)W_+#xR8tWHzxB7X8Tr&n7iN!F*VV|A1Hg^C(cG6yXt2z>` zTc7w8MVhqd~+u_!b;I0ZYfv)!;_DwDs<~T5anN)y{sW%V<}XHm;Wt;Vp}nS+X;3;Kq09^zuFHXY=eNzL zU0k5&d|&ra#fJxSlXZ_&ZBOGW0TcM~7EQtAw@4`fbE{pO(A#sJBL^!~Di)+8S8tK(HDZg;`5;yzax1yYHi4iCqV7rHc=!I=) zgzy7}_Cxr0rTk7oW;hrKMrzk_0>}4jftnHqJjkt}SJiDoKh;1LmS!48!PK-T5i?cU z<|zi^1W8VrgnSW!=IF@o)&Hv7+Z}r}mHxe#UlhK%34#KFW_(-*&iFNkXuieHL{WI* z?$xR(fNYM08)zN^3t{-xgYTo}t8(MXDj?;zjZZg|tpU<;*>1h+_x(ISd3QdU{1Q)z zwhyILGoDY)ykqaG07SJszwT6dfIdhsqUsaKQ&0AQdel-b%iKMm3+M5sZHn3qyj%84 z%CUg})9N2UlO$&46&60{VYpJ95N;tl4ce?ZdkK2AbGJGhMc(WG>8-=?dZ>@j|F%7M z$kq(1V{MydU=!K~+<|mUMW9UwP=n~W$oI$DB{1dT@_wXc5VM%WH$BR7j^VSn#SE?f zeHJcWp7G&z_-_AUJVZPgdM}J*mBG?yj}2tUxuxQC?`D59CDoMUtkH=TSOvfV2|WuG zK=dgW=EwH67tG8!7lZb#$*|H4TnCg}y;VIuL%ViLO~XBA9ZROm+)8kwUYk%n&cvZ; zqBb@crVM%O6n_1WU1$V7VQ(kz>*vGq^xqZ8o!ghwTd1yixIl$3zn;t(VLzj@GR<0N zm{})eenjFy^`QtWQe2nJ%?Yj~v}$j-b0HgcgwH$V_ci;HLQ>$9v4J6%*qwDrtBZ2# zveM!k{;}o>MPA~!+wI6S0PVp+jW_Z?U!O+e0b5GG039(XH}A2dQ=p2tF#!_jI4p^w zhX-Y&>{3kqmV=8`1X>)>x5$T>(HUw~U{Zn>>jF@TQUv`P45YKDrvmW7uU$M97ZKWk zHEXFS4MEm%5TZO{rjV3^M{hp86|W0V*xK3lH}K#;&A zA`VIZM3C_nE6F_oVw@7_t>TAAFvTD$@Y(2DSxv}eNS={9m+xjsyk*KDLGwG99|*`e z9jWBTu%N`Y$aTZOzg0frb3%kphP=irnc6o@FUsf12*EywAF^7gB-Pr1B)gR(``gYz z=Fj>DA+w6&3*_Stinaol)8F3@HGp*9G1q_`3DBY%OiAR#qKh?_OrU;WVIm0!HTn;% zZ9Jh14D&#}WS&t|{;*~F4D2icMGzc$XYBMcF5jVgm&!ydENsFthKQGL zt@7_>gN02YIrt71U7s<$T50ZdY!G#Nl(7p6CgvK=B>Hz{*Pi~TXagDqGsz|tTV-$> z1riK#Z-mgU>heOg-GD>c({bgxjt&;CA?#{iLHNs)iV#C3yfnf_VX38qb<$KlIJS_5 z%Rq2VlReSunbMEi)CURAB~Stti?+r$q;=*;TYq8?EIV%Bw%4{f5CH=T8}^QW`)4 z$&4IRn7J)oKo;65l+#j^Z-S5O2e75k3o;R+&H=E{PCh)IIx=+@3QR}t+bEOMR_jfv zTN&};Uk4_T0c7v2J>Y*HPQpUey5dg*Dz&oHpn2l<>b#x-TEdn$Txlu%a9BmD>w9}d z08Bxc%GDZuRftY0z|zI0m;H@dnXmC|4aBsvut)45YI2~Z{f)%DV<;g`<})nOSQOTQ zoH^pJ(l`JRI@OzyyYfu_0Rn$5#FQI$@{-^pFoD(E7?@&ILXr7tA}B0SvUvT04CrSL zGwFx#2t<8aR%H#AVB|)=xe|31B=mXZ%2O`jT#n*qncH_-WYWWKD9rUn+O%A~?Tsi$ z?V&o7QL&UK`h5I+lTOW#hn*bS&)u85S_xZjl%=0K(r2GE+tJxNk~1JEu;&fJ;0t@6 z15Ll*5zJ;k)9uU^=FT0G{2X>i0`HmRyGWO9iF0LOQbKDEe(54OF3na4Fl@Nb7cFp+9)P5M?PYe&kIF?yqv*8R#WZHyEWS~|r&)^b;Q_Q$4MA)p zrPQ2YfX*kCQgJA^JQ<2;8%<8zU28+zJRKrQcQ{D&YjB$nxh)|WrK$EK%bh8{_W zV^0b#0Ew`6Sk1>@r;yZAjDn5!DGM24+WHl6?AHnqEE7e=tSyoqjnuFr@rbZE<^7lg z?dEC@C&KdZ`56Vr(-S{{_iFE0cIgavGQy$eqZvcLgq^Ok|ITXup7Vx^>}{4M1!GR5 z46hjKz#Q-ZR&0xIK9Wm#L%>D?l)YM|R&oh@`x)V+29%ziSOG0F$H3hx!WX?SgF`!u z6hEjA<;rs3srmNizBz0jV*r+wH>B!a8OV*YKmj!Ke5OLZhIu1onS!%ttRp_qWJ{@g*<5nF;&7U?1T#Q;BC4=G9DrmB#@A5 zv5FJJPrIv58eY6=Jw72Y3(f@byQp~P2soA%22O*Qo82{pNDvRunhFrEqyYLE6Y9A) zBj|qzxdm88FJDa@_ZD&_L1RW?s|MZE5~*1c_qiNbt_|-FjuTs%Pjk8^KbE2<#lGy?R;~@p#Tczhr~29fw*L zJ@%|i5@9rhQAM;Dm{233+C&^Nsd#@|Lf)SMOMAXdKgcG0Q*bNAO+w=V zRM}g6sSbJ_uB`JzpTmKFeWl@8k;h#_O*=&gj1)TgO!WHgOH8MQpjXH&X(v{eBs0CU zlm#z_N$%W)acx1sNu?dPJ}z@7HF4rRF^}gEL5AH+B|fj&L2?K-VrT(NA9wYSbq;?0 zzoVU$_3iK6p#=Z7ZvedP(cn|To@&NMjY$+53%EeSHDmqSqd>CNw62$ z4#xk2#;0O?0U=e5s(hD2w|JV zYDW+1t4BM)#@&9RmTjWEUE-wXMNqhsWCotUuJ*`gPBogRhKRD=R+0*osbk8cV= ze?^GHi&?5S#xWzNUyOInAWjCpYY9FhCr~w8w0j(m`1f-QQ&Jyc9T^q$$84O`UQM=E zY$E#)wy44JI4@f8*AdueqDwBX7UVM1s?UaJdMQ!bO72y=uyKhylhUaP5*`(h#efIO z(k7d_IfAbDOsXJB7#@^H;7C+DlSOw>eeh)J{1ZE^(Nb$tt~GZRABkbxIFIxbMYZsx zm8x8uZ%2KoP90l0U^E=qvk4QX*BUB9zSg!5dsPOy0@?s$aVW#YA>kujB|25_7O?~+ zI(V$vfheh3NBdsa0iQSlu_TPUP3v!C!5Gj@V8YV_f(wTk98EiB>S0Ge0D~W#+9p5- zsKMBt{~=cvjyl%W3cU#Uh6Ic&RGtSa6z_4xn+J@2yF{D(vfOjI21CK?I}4abRf@2i z1nnwmDHH6X-(hw_3(PUn6?>^(9}z~q-l84w zLMG!A0#hzx<2q0l%NN9!nY;$v84WRq4hMly1mW(Afr8mjLqYSFFNhqlVI+QlT)JY= z={Hl`lzVi`lykFE#=>nY+J-l?RmF|BYNQ3{90Sa`=bB}3T7&`U^?TTCwa4pb)jG%; zwTIknY~lzOJgkr;2!=VR8($?5f6 zcK1tJw!e7uZeMh6nU5HIT#%ugvOE2+7Fe7!@EpkxwRfq4)H7oA)q_1mX0)!K$OD4; zI)1N$0h2edsBiLDR6|Z5k*O4cc7#1(%6q2~p_#WOSl%95YXv^nMrC;$6*R zmGhrXn=)ijq^*UToC6(4Uw?(|*=kn@MHxh*o1XDg^TcMYO44;)R;q{VOO4di$2dDx zPGI)yiguF8-=jF|Bro?DQOW1eadpyUn9SMAW6Kj|zD~4J;GHqHMBN+#nlTsxkrN5o zPHJP;kE~XJ$l^KNaUwrw_dTmevtfh}TWN$v5sQ$@Qp zF2)E>vOf{-Lu^gD>BG#9bDERUw1qg$WtuG}Irk2Jmrl)RSKY;!lo&4T?C9mt`D%Z1rg+KfaN&h^^RqqfR?Oz@vUWnA$B-3Z}{wlL#!7 z{UM!AB4kljR2>+ggjs~u+#67HHMn53j5$fxm_G#?NDyl)hOPvf?p>SFFrxgjY5edV zdYnG8Bia!XO(#)l{ywqv>XXl>exs8g!cmK7vS~$g^_DxhB^KrQMjs{B=DKDP8XhUL z{7DGtvR*?!G$bIQdsDsB{?{^yC)6XY5I{X;YsIZylHXU&8f&a@IX@d*7#Rq?s;0c`+2tgMOWR zeg>pqx1YV>K2@ep!uP5dqYMCKw>W0F{dPA_Usyp2mr~fYb+K1^YD)S6zosxKq8E*| zZ(tO&;Wo?s13)KO&>5#~sjSa#&kx~6n8(SnV4`S?nl9@N2AP~%=2zQ$&_G)^Vi{ib z?V1GduaE))M#xTA7gts)_(aX$tIzX3!d9&FdMXTv3&h#+BbD01${<`o_5G-Y2R#Fn z-7F14_-}Hr_sTFSxs*-@Js`B2+%b7P|G3L)TofBv4F8t{RG>+m-0WPKdwWC@jd#|M zmDza$So%+rV4wFqcdTMBT7HQDQU&0mK%?%I&Dz7pBK11Byqq2mFIOf%Uk{#AsG{9p zs=ys5)4{MaOq&Yt27W&*vvIy@79`j!CRkKR&!`66x-2ulG5u$QxCBmF;ABwyYldR!7Q`U~57*>{W~;D2qC;M_U~6=)Ccj<+scVm1kpV9LXfYRjNo#?+%~ zoPRV9q#A0mzl?f7HeJcE&eOH6hNbAwDwD`JjLJDMq*k6#kTvp;!%^y44uhZ zQ}hN~IJJ~(&t+4mh}-g0vB^Tqgz4gRGU8UYOP~o=@@KIj0jEeRP=_K3l_p(mnPc{u z6J_%x0BLDZCmpr#i=>S{zuzbY`?8rSvh605Vr^pp-(0fhfEK`O&{vx@=|}|;-#|q? zD;N(ly#cv_Xn@-+>C*pU&5g9b(*otMvoPB<*9znwI1Z&ql%rBD#B>oW(FO%PKVne+ zL~kChV21TiDNXOC2pp`WP0N@Ca_ar_NbK^PU<;@TB57cjAU*mgxQSF&8O{-PyFcO6 zL5A;h0OhUdbA(>7Xzs1b`-F2u9TTeL^`XvwfAjTtXo%Reh_8>oQHjml_>4+|tcJf5 zd|>~CB)LIh`8CFzYI${zuaD;x`&SPZrGrinzxVa}{4QSM+J3K(pna0!&jlWRhnW?E z*KA_M!QZW5UN|+ZTr-8O{MW{B5~$9_>Z=gsYo1Kz9fURkVn6bqe3~5zE!l-(21nd8 zUL2GU9>WLuT=}6>V;YoWp`~7Hj^6=5J;b`?TaXawsxTk1-eqT$G zl|uJjhmjAn(+*vZZgIDv?(R|f9*Umey^dtowbh@im!IdOXg>@ExA+Wt?KFzkwj{AF z*}ZO(x|jpV9Go5&CB3`+*Z0>m?o1_t5$dJB_DJ^lPm0N7sPa`FrV!JBp9v+HP=hY| zBH!oAg%hbfA5EYAzPkfUP*I^*Ot%~z{d$#gxhvFpq}1J#25 zYx__e!A33~iIV-}XoNLSzGb)i0tcqp#*t0|BEL$-TwH?kuq3vSZnP2O7)l_>-Mer% zMZfH8dXk(K!lu5&@x8vry&&?;dR9BOL_V#5{+{I~?ZV?;h*-BwHzW&q;U;CH~kVdF#D9BOagQ-T~%JD;JWrUAlQ1fGYt+p}ouaW>n95!SF zvj!H`+jTRM*D)|GXJcL_Bdy0gIRbVbBHC(*HyRx=R+bV ztf(?}GFp>p_;~SS><3Bh?RSmwu1Rt9uHCczyERT^zq#`Oz;PsV6s# zDh?@ZVyPm|(QWu~c?J>3sK})84(&_AtSO+(ky_IRiU#5!u$9#g>Q*WyIQxxip@hvG z79Q86D8daQ_fcP77^vDXe_~)37A7d0T^rDUjaAwCd$dekO zfH3Bkdi>vn^w|%tJA6IGua5-D0s>NJN1JtT?`tDo68jR{02E~LaI9DmV%omnS}-e& zK(q4mMYZnoP%&Y_a)X}|@6AlnvaV*{cn!RD-M6>zeo*KaG7|g=77D?n(-z#8l947r z%JMoOlX?&%XO%e!1ih#)K#T8Z3kkZ?Jd6vuY+7?FdWZsnQlEzs*8V{;f=WdD|CRt3 zScj~83(6_lw_jH)NXl!e4{x^#*`1*5>qVsYw#8462Ty*^id>mWq@z$DC#w8@VO4EQ zpz%GsrzhC8(S1_i!O#e7{rntIC}Wj*PnDC5hWZ#HGoJ)vlJ2zlvUr5Ba{9ZkipNj> z6~6s1-JAc&XuF#rsD}fdlV8bO@tmDEB;Yx>>we&E!Kdaq=W@A#ADItbiG4aN!s<5AGM2WU|LfDmD3@{EPnc&7hhNo;3 z$)~x*VCX#*$U3a`fhGH9s?40LJ{TSwTW}6OU%)}QtS@Y-op8ZDUZ35;PELQTeIFFz z<6XS||9VPz9XeJXV_Q$qcr~xDYWBnUNHFe7RQt@l20?@g!4-h;yo++Et2o#?_ib3c z5kLsm`k$f3FkwQ4A%(CKB{-eEq zEPsEp+mNfh{d#%e?|0&XPyQ#}hPe&gs`l)D7Y0vny$XNzWbg}Kh5!G(@hiSJvSk0u zTpPDKpsciLzHg7L3i=gL8ak*P0yQWIkm&)H+{GA3j=kW~|KKAA)J_Ei~-X0<=)Y*F@FyV!E=!oc@w{0K6J|8?*tRP8*Gf0S!a zp_lF3)wW-4e_v7>3NG3mvd)(q6|S+uL{eY|@{h}8R zf1nSfxnES4u8iZ)^bLRvj^|K`a%^w(s5#$Ds+#*UY7X+TPQHKr`sn!d@cAwalMosx zQpfIA96Cz=L9zNcA&q}OkY(u{oVP|^clQ-N$SEKHb9i+gt?e-l@RU zfw(r{Qc!15aA3JIR9sInHzwHdTwH!%V(DZkZuVz4RD#@wLuSSU5oQozR<27U{(Kl- z7=dGRHmsY^DctNKS$94!m-Xj1KkIX8l|s#Is)2l+6Swmbvx<(?QH-&gLZCRi zNlOIW%&`E?3R8>GBj1Uq{0s%NmJ40LIpvy2N(W*kYQhP=Jz=Hy8(9$5{Uwgv%&vN_)EAEGk}-*(3S+e|(WO}P8s4BzzyKO2n; zan>jVCXiul2=fqPrwKE7?OLd32`` zF0aciabcgJF~J}Dz}RVa*ZbUJYgF0zU10P3GxveLGQ8;fybT&>cuy&j*dey zCmI@s-g)Iz!jI1agOc;Wr7ZjAv~ zd^a2<1_~cil!W=EpTS#a$Adk+tfudj)QV8S>t@y*m}?t@hE>`?+@`$+)7UQ^M=37+ z<8sKqm2dNaN?pC?FED(IeCYNXf&dr8JM%9|j!mk|5S~SUa^P;NmXeR#@6to*K@igT zJDWS?X%VqZ<}bJxzbPT0W4>`^4dj7@fvWZ8!}P+1wcPk?s$SLFQJ|_k#hCUP?i|gDUsJvWPmajQvhxN@dg|h%!0Oc5H$Z@So6wA#Ds__0|f-`QH_jN~Q48Lrdf2geG!9Tevhok?* zuw6IH2r|b)d}t=@KavmAy4In|r%8%!l7rV@Gxy zGg?Uj;e+1`8}OyMq~gOvkXtrrVMsrR8eJ$)7noBv-P*c&CboU>B(gVz{XQE1J-lQk zp$6+U+XXL5uBwf?QFMmPNPuGAoa0~T@NdB)@B#YI2X31Ew`r?PMS-i|FQCpU<|4!4 z03m=()ljb}2d%+NE0 zW{og`Fq^aL+}4t#tv7MnSiOKm^FaCswtxeOd4nC>m;)H+H+F%?uoh!g>(#z_4@2-N zD<6bd`2Qda;)-$xD*9!Bf11U)cJM|Ly#Xr;Io`{UXVd9iHpgsfp^|rw7FO>P2e4I! z4XII`n;(o)#`*)a*J}JzP-yoa zIR-xQ0Wfds35S36#)q#0b^Lb--S% zA}60!SO8e=xq~!0fri3sv&gGUJl`}^++FJaWduz95;-sInx+b}00F3Njyv8qFgv(T zf9qZ3W6Ur#x&?AaGLbrkfK5icN7`f**%P!uAUq_Au&OP$*-kFEP+Ag8r??{$^$&+= z#N+3Dymmv2%?j%C7+#6{1$iG9Zk?-7I~zIiQ;=97Y)U6o-IFk0YvHxi|2nqovz$zx7b&2}-xz}%lH6Fm@>Z3>n`6#`5L%Sz;J zaCCf0YYaPctk(8h7Oo<(b>Q_H(*>JF$-D3VFfvtq*sU3RVE1{jo6zSs1_Z!Aj7S=^ zKbs97-$kfY?|9CxpuL#8@;!ZI2o3LF-T8U5SxAhT7BQ17HRvh9G*eoE-(6k?&@JP^ zmxlj+wONeo#aPVvT9LJe`W#aLTM^tR^zFJCpH=u$XOPz4MfL4s#zvnbM(Xsy2muw7 zkjOVqQ8s9VyROKjhkU~tAjHxTv_Un@T2<@Ta0U`!;YDO9WlF5XJv^(X2vF8Ga}P0C zuLhOA*euq#=!7!f?h+4iJ8gr9hQ(>prRMJ>ufYdVeK6KU1Fnkim43-wlW!#~`d|Br6{%|2}X7#Ds2ZShs{C?z%8m(%TMEd;=w5)l@r#U9*dOt)+T z%pARb!CU|c+ldj5=^3diRbDh;I~%RH&O=O6f0#>hF|RT<9)Tr6`9g|K6}vv zwr}pla&BYX$Qw4aQA164cXD;XiaSuNJREGEUs%H};4aJH8LifW@*0B31k=N714}jl zw0VZR?WGJVl*PvfyGC~d4=Zoe7^4ROf@t0RNx4a>)FEMZnC(Qr3YoM=^6F>9fm*-g?dGfnNGec0 z3#rd3sdz9RXXQDzX?mz-%9T&0vdZFoUczW*kkbwWGrePkK)oiYZ_{4wB6bl*MFKQG z%HTwMs&cSg+^1Y*FmBo>7j7%dh1RI%+470LygsJPSlW+C9l7Z-*)&D*}%YIdd9+v&AnF086Z-?{8@jecu0 z+Ytu~g9m3+^AV>bY4THGGYH@$^*nLJ2ppV*R`8s*vPmqjGv3D%} z_4J!>jt{?idU|M|uI8vbQ#ZRc^7Z$xu#&Ogh75YQHlPNi|Nc5gG0MISB_I8_)6|-4 z;7^A?p1gkg%(cM}x+GqeR~S|YwlYn&NG{2%8A%2?Wh{c~7Sz+DtOlk2?a8E2Ge|ua z>PrN`q0h`a28BA#HQ4m+ILjBF)Fpn<*_W_wng6n&UFDY@q)lsBB;Uan1hOP z;d==JlCohM=hui{6jYVzr9fK>3~+_D-EJ{H)y+vo8QuAAz8n>g7`)_W3GyH>u6>M= zQ<$&QkE*nqt|_-r=4rSXfaM};C%-1GfQsGP49-t4 zR-qn@ry-4(a^e%7EiJ|4sAfa6;t)npg)L|1IWyDYUOzr4!oR|!;_ik|cxdc(ZI29r ztd`5=yry6ydE_C&IRoSaR%Ig9eR>J*hE)ZiMy1DqfH-EGRe4s=;f`;K-r8aD0#IZs zZ+VF0x0F|C{!^`|Loz3DrD?AswMWIS-h!q749A)9%D2;1O_T^Vy(jt8GO;B9KLG`* zdw-O@gcPKa-R($T+3oX-Q~aMo%J~?Y$NkR}JPSjgknXX?4x%rmPqZuy)Gd%&1uIJ--ECI^HC|vS}78x36E&Sey#L+>h1W)5bG7x?;t>^+4y~3 zUFQiGe*L6$tNX^*Q?kzgCG>`i`)eh)KkL2u7;014ojEE~g`PhlsX?nS61;-3IQBm= zsTmzKO=%54aMM4Ul&xhfPs{d^Uyhh<7ins?KX;S!Q+e#4nx3t3 zGD*;OsYY(we$^!XG}`))p=orzG6mys#_Q%?1+K%hO+BA==2K*}A*dMzZd7#bfy#p| z+R-XHO>{WHB2ogsg|>eDo^&B(e{8whzA3U#?t7-~iPX`YW>oYPM z2E0LxYL$h_68DCRB zYu48KNa5)-pt@yaR#V#`pJQ*foC;)Mx7GOuYuX^hxKXL>hO}V1Li6ZmzOI*MkkiX@ zaRKA74SrXtVM?7xUJW)Y+X`yvAfGl@RZ(68p*Q3zWuP-fep0mz%}Awl|%(+DBre%EGt)-S= zDd{DjH(XWbHow4d^f#|5ZQ(omst2TgGirzU-*LMJ0^Iptvx91f&INn!jn{c+y3v~v zkxk~|r}f_}PLG}+9Z>ozo+I(2h<6`PbZaR6vbp}>fd{T#Hj5UguGp^jM zEVAkEhK~lw;vwl9qqkF8133E4bc&%>(jH@DQQNWmS8fA(Egk5tB00Y?&u(1Uxa|aE zUR*}WHdx&~%n(B_iZ}xA1LFB=&n5Oy%Q`3{mFJR+RdtZ;{MKx2`$WcKA6Qk=F6QFy z9Mt;S0|t4wfZ$2#t5AKF7f{uBosE64~%b)fLh031yWqL!p2t*b)CKA8l4mxpN2(5UpemZi=L*OJ3 zi|)>tIJ3%DzB@JhF+18|z8hamaqy?j*1@L>xEl={V@w{|Y>ZV<+*n@$t-S)ck7z(W zmk4Cl3nQ8simTR?+SE@bfqBED0!6nQ+DfNA-4Ryi*8N7?l7tfUqFAzN>2OXFyIRLO zLXftQ2VbWog4MG@5nl-DkjwbFY|Llkj(H;;LW#oo_s>52j6!k+W0=a(@JK`xcxq-j zpG%bVd6@p$2IW54t3K5b;4e7jwyVygNN1pucuo$Uw&F9Da{rD5k6m%0v)jY@7+e=rYg_2NevYufBIRT|CxJ}%pv z#ne27?U8CAYTn_867si9FGd`X1RF0*f60#m)yqS5|LE0{cegqd#v3=bZS%X(Zqb&;@Cxz~s zw|{IlNM7X}T1?DyJcUx~ zwZgyv7m`M^OOuI3?rd01c9-U%a^6nuVu`&S8bl-|GvZwvUKB_4#&cW4FQf`DQe9D! zK+(8w;An{%B}H~dnbM?3E!se9IG6M4qMY80Nr~<4s|NP*Wj+;5e-gH5PR8R$!$-MD z{ov`Y6Y$hl2D24Jud1@eV0p~DGa~~8Dd6A&RuFPY+n8`)#Q>oav;VDSdqXy<$p8qm z(>BI&hO=rRWh1I$B4LYw?PmxNSP$MUHKtqPAu@Mtz)ayIQHNwYXl(6c;7AiXy97yK z)E@3cvP(ewIkPN42Tab*N>!L?Dswz$-T7;sJ3rC3RjIJnPR*vl{osTdDoiZ*ACE%y zHylbWGE8haI4GVBfeNY$~4C13i~4SRx#A?9U@`NPUL z^4&}LifVX|k6s;aqyDJ1Z8w1sqOHOg_Dvh9s^w)*0`8z(s7X54^nGTd6um4mYXCnH zL1h@tX;NTX$*g`~&!A4avL191&g0<|VITejWdDUnz8F5zkMQc^4EQ0cQfYPq4vO2B zoyjqpOhA~??>Q1p@3hg`=69mgsJVS?Dj?Pt!=aKi>*CP!)R}wz6A>vjUo0Th|hN<_5Cj8U+Rb%qJHiI2N(k8rhay{jW;sV%z7nr%xmh<5#y>SH zZ~<$HU=I+2K}-A@9an-0VjT^)21m`By3rULSNXOuJoX{&k7i}@s=4WlK>&FhSP_bK z`3`s@Nnz|&3Cc1*LD)Jpkh33mMpTT^FWg(MQn4ona_?Oto!vXr&6w(QqX=b~@s~jbd9R?pGAOSD-vY7NYt2 z1|umhAg$M8L26!bcEjmtMrEswBFvRVhW)}^m@pnhU~z-8yNfM>4p3tuxBqy_M-iz? z(vh6djIVdB@8fWN?Mq`GeB&83yyfLhPHhQBPZ|Dgi&(W(iH?%I&~Aojdc%lCgy4_L zY1!J;Fd98BZP%K3>w(SfuqJ)~gd@-e9TZ!4`nzUf_IOcGPvxy|WKD2>Vu6$L(P-6^ z*O~I2sf>5w>C$HDkg1IvN~SsVa8adK^Qw_19f3>4msyLir8BaMAtbWBpT)Y8g*SVm`@zKpC2_N;&2p2Ayk5YXkTuY46m=-} zu@->_VAeHWHRk^&6Y>rna3kv;GE?}UUcY!yJg)w=4@SrHg7H`ZuW>bNadQErbqbz- zCk%jM&eu&j1)DeWTDiN_EQZB1T52Zc1ock}F}z}XV0blz?9x)ErL946B^Fw-w=59o z;-Ug@rKKTnxFaE)rG^C|>4G>zjg=K4G6i8<&?;PCQRH(yMW*uos~7)SRLf?1nLW;5 z41ceWGm(i|mRXfN!%rQlJ3|sbKRl%WP89wGP=VZh7-R1SbPY2B)`^L3$f+fiU;rKv zG&(YM68#h<^IcdByx5TR*G(`Ol#beR4Y^5oB z!c1(6z8z>^)=L`X>)-s}RWl<_XzmpVcWs%Af-*v@aE@Zvcif8^{4RUX?dq=*6|j5fsYEUEC1?{w!`vJO|H?51*bM7MMpH))A7q5?BeE0PD$Kp?iKSI3IoNUKmnx70Iw$r-)s-GIAH2vb$$>H&7@#59#k!l%R z>KA{0`ttk3lcER6iTB(7-k^B%Xi)SXKlx%%?3ZWL{r=njEd&F;;jJLYFzjY+Bm#p0 z={+8E!gn;B11mAft1(Bx%a3F&!n3029>Y|?kJ77$M#5e+a!U||5TAE>s?TYJK{Tut z0}#v-pookhT@oxNC(x48NI_5V7-rj;!}R&gMjUFR!xB&n`8d}=oWO@{?s@|4ho?Ub z)=PwM6sZ8J5UYcunfDpCKG48@AnHT3D=biPUQ$^C=S^uThV@`oogYl-XQSYoNlg6R z6bM&)DG;2B|BVlL=ly2b)vBKQ&+`so-Dr!!*m@=)ssxJp673Z z=d!D+qDSmVd=t>fTeL_A3+C@61?K;Lp$@Kg5xKJs8NzKppI7r>Jem68Q3*3!1fiyA zgCP7*;gXtGKc+&EMJ3IZ=gN%O10Vg&9~Vp~D4*WZ(tt(7F^Ep1uU{OWoQ|IzefRq1 z;pyRXRj?1?Z>4u58zHW#g^>p&p>vBx93k=!Xa33Q7;M?xc6B;|+yoe`DHZ8-P9j`}f=Zz2d(Ay7xBa4ZLaq zlrosLfqPX}adBeET^ z1~^`pH-P-j?@-GKP)Mw0boif#&&IEhkG_8K@^Jk8@aw1FzdRkVDTxACy3(Z%X{0DM zgZR8)=^f)iyc88@@gzMR{d&qAji6&}xigm^u#(M!(AkKVdFUJ^!i!R>oDxU-5)6+C zCOSC_-2Q&oTC`N(wvFY{A+-t4H zWC+_7_o-)W-N!sEQaElfSENDc&WEIoil5!HiI?a4_BERfBGjTdfN;0TFbdY26gK!Q z*kJ}HR_0FlTyk!Xg>I%bplIm;JctFdRwE=je+$dZf!}pP?V;@@Fwly)RlA$m`N4;G zc62L$l5%)&cXXRkia&r%YI?xRrmsw1=%-(-IaI*Bi$;3Nz|bi?74aN9NEVr!Ge&%I zVO=)3M-COt*-Y}JT8qs{la|lDB$ECD$1nkvNQIOQz*&TFt732Zu9>_%sd~ z8neF?{0qz$)MX2R#o`_U^oV&MZZCpEk+&R*qwV#5T?#>p(H&%^#8?7W*+aVozv_qO zsn^ESIY6tEw{+qXVyHL(D(47qQt6hwYhUFp*cF+ z?#lE%VA0LXS8RB>4qUWPG;9AZ?ORbNy6w5oEqDIb8$Ol_{+@Tf4HI)FaG$lwSr7QW z(@Ny#WJn(LBm%i9X~K#Rwvkyi2Mzv);{%}$RCCc`VBMQZCM#}E_;4(d=1|_yA$P+Q zYR+sqZHaI!-OLbe2#nCltnMIsm3X~#*J83csv+8EpN(UM&#tko@@!PC!6(FycvamB zLfJMKQu?BoE`c!|4((Zo&_PP&f(O&j!N(ymAM>LWL~h>i7zb4Zfw1<2=ox91tAh< z_2&HCEOXpbl%yq_;i~ZjHEg1%yAuecy&#bw+6GN#9`vuL#&i7#n{LR;;V;3ogAoi< zCQP=IBrTkQRRht%@s5t_k#Gl?J0s6TwFBrOWe&@%MCK~F!+PB9#_(WMeO@ZjE8_a{ zCye1sYiDYNPn=BQnFZnFmk;83xAli1DR^UYf%r5@SJ$XGLHnzf2h}7_!<8F9r*WO9)9!m*^g1uOtfd>6H9H~ zF5t<}$_465tkcEqlUhy^KE+rmA{&NxMSX6yEi*8YPU3;MkZfF$WUQ@xrgOWtaonKk zF#D6s2&7zUud6tGP4_8bW3KCL=N`}$Gf9AjHm&JDxsF!_+bs)ou?(Qqwz3B+*@X-toDF8~@%^lLBr=A-E| zYH=ur_FtA*<9`2<6dRA`6OwIeiFcye#EZ8i;JAf5QE_5<%tH{`X%rSsrqr794d}>1 zS{?V5zBtr_W^=wiJbm`<_~ga^dpQ37l_6kP_#}4jKzwgt$ox_0fIp8#CBk-k{SNk{8`?n%3B-$(n7}|G zcDMDHK9$+TROOMRcQj(iuBZmlqV^Nm@jNkizsHO*m;?wrV|>2tc|a(hB&ywpzB(Zm zw-M%;-UzlzXSd)A9yop%W3D(N9Jn1E*?+|^I5d`M(YFeRe}qEtxh+qMv$TncwBhu^ zupuV9G;u>imk&*Sk!cdwE#c6Vb=2#Yyb!~6EZTSV5jD&PZ3-r^E8xEc@ zqaJN<$2_%tV@fld+{Wa2&6gp!#6hCm{QUxAeVf&~nptf&-VgZf`JF4%fW%GPmdIET zg&wC7Y4&t1b4@mV(PkHaL z^yq!r-|^jTZ(%&+_zaN9wgrfEJQ_8ttp|XXp#D^{ffK2W;04_UGS(2IILvRofR04# zaUe6K(>o!zZVk7nu9nSOieN+nT(*jeZ_e;3i7TSeG_3(fB?M!T&d3Ym8Em;760C3F z$n(w=AE4fMo%@boY0@2T3EeIs*#I~P4Fw=@ z2*rU@30Bp4wE`FEOtM}?=Z9s(M1>j0Y+hBRnbS+Cb^k|O>;AQ`s$u+dEk#L8i}eC) zesM)Pi0_{QUAtWXS1LM`sCk?-!Vj$9t{bj4a*wh!61N1v3UEAI}$t>Y?MU z#Lopa#gjv#mITX44(3#s$VDt#>{{jR=pYLhhg>h-H&6nzY>UbHq8FIu;~`Ec6o!(2 zNPkWAkPctw=u3=TrgSPpkzDzg^QN2^mLe$Bb}%|3`zPGA z|LXx0?j)F|4Oqx}Q(Z6^#Vfc*Rjw8idF_UV*r&*@1oa4)Ykj!_^2a&4h!boVxycQC zg$06d%5txhGuW?N|Bd6X&+#27?(VQ-sTaczo&x5pT`UZFm!OkisVq@Qa(L)$1`Or3 zxrv)4H;>~bAZJJhG_Aj{%?YQx69%}&)Qy|zoROC>Ap*X$o<&Gdx-oh{RncOqa><#H zBhRrJB9Tv!Epd=W@#W=uy=)IY|NO$7n$6j8+FX4Ox|iWaA3mpW_Jan zME*Iy-hQ4u!o%gwSCcU_V7D#JW(B-%RdMq3LPn8hU|j*vf@RkOHbF;_51@z5ja7j^ z4VQ7XW-a;Rc3D1yAuCdfHytNn!_TtJFB8_%xfEQawsR@8pOVpc{39M9=7w##Fy8ba z0{h?&jM9ybrD5O2nT&_YB?IWCVmDqJM1)ExGajf^wiBrlkrU7bl-T=OqFfwDhPt|l zPL=ff-!}V5-V=wTm)Aq-Ts^S*K(r^|?pH3k=YJ0h_`m*M)R@hYbHN9)?tyzW7`9C> zk`@M#=WiwfL%@H~IPjk^7^Mdvtt_(!6QTsqF?WsA_KNb71s~8$QfqnpX`F%O&L~<} zYlY%R?%Qqi*tR0BX+3Aa_}i=?rYn_~2y4+p^%TQw`vmk2RP<@43x^jgT+|Dqm2}t! z4*jDRL|RHNijJDm*lq*3izD#*7!AN&P``6FFxys-ix;=b4))vOqu=xg9tLl$+W@+m z(?`YN!?G3L>ZGy&wc4aa_LhqX;*47UwI-3a=BnJR^-HDqN>l)ZnZN`T5~rEIuQ(!l zUafIB7K)S7<5N(nsv}fjl%~LCzEQ|p)fWWPsTQVLcnF;lIT_V_ezUHpt+m*7kYtiR zBJMRttqICc0-+f`ymsGcKGxqGBpbAsz+N=Cy*{KSEgBAJoYm*3YpxSHUJo9;xB??* zxmZ7Va1aN7v?@H`P8n^w4TkqhI}UFf)iOg^Ob}Nj(nF*xUK+Tk^}7myRVc`NS*7a^8eu$;duTu8(oz~AY?zu-KcFVKhMmE;i{=d_P^gHGl zSkT?x`QHb5k5d&bZvluqp@;LKUkbt>sskjl8Mz|3#3D{NE8ti7r0=2*Y2J=E)kn%l z=lfPEOEMy<0PMY!q<1B2xMLu6n`W;=$=EQIf>wr~m2TkA&Qlaj_w$X>*~c`cTc`+8 z(7PU}Gz`<2OuSAsjM|F@k#`f{ z_?MBt2;>%o3JY}`m5y)&$Z_V|vHRQA_#!r+X75o$_loNZ>_hutF5Iu9gMpvx5sh^qp~Rj$z369FSshsRss zWPy~hDS@c758f+Y|9JZC(W{65`fpzhF%?ywgtBz(yfCYnnG}Z;O0!# z>x?SwyB2*=5Lb^Yh6c$nsIPYy=w>xrhF!rbKe9m{2ySk}W8ABw0uUyi^WVc4-@H0H zK75{xfAC{e(bKV6jN@7NcepIN!*SCWwscU?m|fu;zR+Y+B(w3!$oqb$oh8ZR*L(eIle$S}OANzqY;rycu)yVAP?Zu-Ww^x$8E}U|2!=cHNB6sxi!P6qi|xHr(fSk`-_H4PI+jEcG7llydL^ zO#PCRH+1Hue;I8<)v+svO&f_5!7zGhvDhocMxOFfaR~p-yg;|d*v1^8zMOm#SfIEf zdoYUuzuZw1Eogm6C~!S~FZ)_Niy^^Rb@H>}*n1BuYl1lgXoZCu7uhHi*>x4MSZB{8 zToH^S)2aU0a+FbnAgETDy15FMDZid6#v&2NmZYp)Mk1_p*l^VZF7V6ammF??2wi)l z*1()r6dy8|ezp<2WeEu=*&)$>IYrM@$HQ3$eyi5hfqw%t0yIh)vQL9b8-9vGEtaD* zH>dKIC>th~{QBH-nt0pgmq?Tirt7OY8lI!woLw2hO^PmY5Jw=!>F`pL7c?haHcQ@j zzpySslQ#3%G|%gF%oUeY*PyMl*H2AT&&Z>muDqj2oP+sFST|izG@&j<`UkO80Txhm zUI2IoS8u0NMZQt6bW3zF*Yv?(;r_L4U4fHciu{S3MG=!%HOTJv9f^3N!rq_;PRpz6 zbAZ~nt(|Q+&=PbH46LMK^~hE#Qd|vKR@lz01~)>v@ST`KX-4s_QP<#d;tVdbfJjr0 z05i-Xj7TLQ-J(L3simP>BQI_&gH_Q^v!TIgaLNFWfnyZk$d*-Zh@J+G8O@Zc?^|dO zyWk)uT`NOp5fARUyDB8O-0^Wr3l`RjoJuZ(n+029hTm;RQI_5c#y7tqKyWZ$7l@7& zkpRwnNV%&JWxF5)IKnZd#Lb31{2M|Y=F%2geKudKt;IdRfYvdtOS%P?*#J$2iG0)z zoblMktM$BpN$Yum%^vo=2dtAhSo+%C4GJ@Xe2KYKdPulMctOgysZAoEI==;$TE4bB zC%ltlo1T$o{TZ8gR+EfpWLlMfhSZva0ZjpZ%5NTihcZ5h+>yMiJPRn9G$aVV#6SfO z4$;A85U@-f4`bb}V&v!ytN+wFPgEDz*}l@5)lmpK5W#B$0ef86`TT&hcvH8T4ot+j zM2ug|Eg1UPa57Ouq?Ylt{aF|Fq-GN(rgdv4T*X_!oIQaRt_!K}cU9tjM-@;pkXAxE z{cFc&&5_Nt<@z^*U)T}lDRET$-=7>BA#ESBNWeA&@;n4&x#HCWFTgXYU^wYvP$V>r zOgz1hi)9*a{QxNTlhWnD5{UW2lgP-JjBFg9rTme4r8(}Pn8-+Mu^!uo#Oq@$Kf(e6 zTHFe&fjV0w@#Go%EwMC;;o9F$fxiZbSd`k#C5N+I>s0v2idBI zD_%fMxGR6{TC)$CFCmWk<1ZL7$QROUt195Y$17S<;siHvpxjLKeq4f|4GjJeaKm< zD5(q8go54Ht zrwZ9%ZXs|&niz$i^~tKM`3x^EU}$z;L-oC8ae(mk`o|zQ|DF-ViSjm!dWsrnBS6nK zT6b3AFU&hhXul$FQP3DaiG;68)j(AmEV992>b|_=P2X@LZ`S7zf4|RY=a_V; zo_*`HT7Jj0A2CrBCij1WHUAmbb-x%C`@_F9^`f_LzB`cRj%8HaTPk(kA#&&%CBUMP zl6n`Mzh{Ot&`pur;WKZZqkSlttJ-w0Un!n~tO^)9{h-7(GMns}kdqb%tueF9Wv^{D z!bV8kYF1*rN%ilB3JH|R%Q5yR;|>O4MPfVfj)1r5g??YfcH}P{PPObOdX8*CUhz)< zzEaX*M-fe0cc22^dP?6s@Q&rKfOd>ccd{*|S^(*ccqjt&{nkUFd&Zc?Bt^fr*}ErF z9}O$KS$sUonH^%%wVR8gV98Xa>F$sBi{HeP_9-|$;RJx+XPsV%r= zBQG$>gH#K`e@@=8Aiw(jc*jb9@L%$f^Dtos>s2#9uADf|FdsSg~YZTWtpciG_yZ^=i9Xu8q;YkWUG%H z&z?u(GbGn*M2E4;L38$(F_X)HEyv6r(p2k8=h>W40g;u2Xg+B!sDSm*W{#ymNy>me z9WX*W^cpvVGE}UF#x`QJ^zc&@TB(B4*wXP9! zmHQ7S|4e1pr)TT}k0^wf(pSzQ8L9r8)F7ZVcj(^S1Jr*jUWScPCRXzsz83su{l)Sm zX&AnDkiZrQL4q@0LKd2jW#t3W+2fhfw0hM4PP}nU$j;-}hlj@}<3B%!tWF>7xxF!0 z&DC&O+2S_-JFkw$r$>J}d=E4p@`zKp=+F@V{foF-uO1QX<_R@!$0Ptv3wE=MC`W*|&$^Jxx&@jm{&G z+tGvHbmzo7XSvg)5*>&1cy}7`Lvr1HGWuW{6gi&qL%|Bu#y7_7mi}JwDd&cx_^T^cOzbip~;Qz zzi*!Hn~ms6y;Ew!4LjO$NIHKH$JBX@I(|*hEjUJrZ=<;+PG9`xt;$bwujR$6 zA`n7;nja3H){2L!a*)_y@5x{U!K1WdFp$}1UJ*vGa#>(VURsf|A;((`HDQlAyu$G` zgCh_wTC6?=vsRj0=vqs>Na|Y_E{2f3P)}DeZ7}X0)i}eVVh~r-r!+Clh^g=5jcVpI zw-~k`?~%j-hd(_3l4&NZfW3+JthboNgXti1FJ|gK3TiO_S~Am7BQuZjnuVhu;^aMx zRbo>*qM%OEJL{K=LV2jm~o>PYCpw2`GnaRj@4T>~ZvRm2O*Nltb=N;qH-Hdk-*qNP$FD0IZfK zt{F=GBtc7qC^lSbOv1yd%yQXE!99jVOWLBEr2@Sx^&Vrh5GT{*{HNNBX$k5KFt%fa zO?QU31G8K9v3r3Hv@LXWI+kSA?%op_dO%ucQbPzfr|e9^FWn{e!n53h=}Guy9P7f0 zY9ciOqWzPB{_IM{{b8y=^@bevxm;1mjZwRLFlil-*zLZCMe?;Q3Sm z-&E$quX-)E-+V|HXf1z^K0}v+k{*nlokMshQJ05ft7F@?ZL4G3>e&9rwr$%+$LiR& z?PT)JVxDIM>o73V)52Ums3yY$?VN2I^)rK`w~JYf59tCDb~ zVYB&kWa%TG!bmgzc?DGBDE>HFm_v|=f!7e$Wi)(1Z-&}(1hBSeHU5Nhy?8U<6$QQ+8 z@LuAYiEhw=)&;DivN$W%aWzZzZs^|Brn5Z7-?#K0PyhK~OmytTXzE>t z2|`!o@jV(uN9Tn_s)XuP^z|);xst1F{BC|vkB6z)ee-R8pU0`)tpJ>u-Z)Tu;+<;p z>djR#$~#X1;NAu|$G3-v_r%WUe)RYn{82cspA@Gj5uZEW$Hi4>JtSlba0F8?c&iR+ z!b2g(C^fE5q>1WPTa~J6Yn1ljd6?`9$lfVh50NY$&jT=7-N~C)q4`y2>do-Y$zs01 zudJb#r`!C_W(A~uW#r^baM>%casO3_V8R7*>ZP_<#0LY=wAPQF0@DyUM~80`aeB3j zoocGovpnK7v@(0UQ4&9hUsz>N;l%cfSuXmVB)C=MCqEQ1Hd1dd1XBJF=&$w zM6hnnP32*EN(YgSh6YPoYhwHlX)`R2Ly+an)eDyxQx`SRgip7YhyUm4yJ2e^&C!;wNuI_c1-~K82hr71F6% z`~-%po6zN9aB<%|&^qVqzm~^3GkD8w3XQQD19zK54}7-Iq69rr!&zYCI}_IjjZ1aM z5$HG~Rv}`Sab+#1+M7wng)dn_ayzPCvpR1U^c6T_9Sv15VkE3vQh%?Y5ggd7o2@;1 z|NZKYJp^>7!!zqJVJhu)(n_bow*<|6GvY_AfYeFbBttMnY8LCql)IvgAn3TdLClL; zz8Q$)ZQ9Ajw$_lFv#t7KPV~S}(_u}u1+P=$FJ(&D2L%M^R`0)#P>Zo9be1>`@I=uD zcyM`R5FBNz$-tCxzR5`Iwz52p%XkFY|8>tyQ4+|3`{GZ03wAv@{rIhwJ-Ec$jwo#O z)w!A(Bm$|Vy2SM^_|4f=V)9%wArG~WLu(8hhdW(wgK7v5$Wu6~I#tCaZ;!VE%C}q? z=5r=Y`T*xtc`xVFVOzFkUDvC4-Z($9T~--Qch(E_y_w4SGQGClQws-6%A@-<_m4Wg z47o)4RrRzG+s`zVZMDyF^4{IoF~-KjYz!ELRoL)U$HXBy;+~~Fj2R+P2`7VI0!b&o z<+-uVKumvSG(hYM{0rRn1@68a6Xd9|M?U*0UkpcfNOHCH&Buh~1`()kF!#i)q0wPY ztCxG8iYrLC(T$c9GgplLzs+oRopip6lpU+qy_0PrB@J2`{ z`28JXcPk>(3SOzNLMQ@-+=iaYP)3;tY;Ig#T#+(=L;bgLaN_9>cK#I@g(w@;uB(BL zc%CT1D<9m@rDjB+rtYk}!El-Z7lOeC1Gk$r-t} z&>v4YUX|sdUJ(a*pbLY;H1c2$!+NSoR2I62$!|o9zPp1soY1@+S=#r9dww{Gzoa2R z7jiiIxkLX1ua}7Aoo1!`Q}D(K)`o zrm`S9fE{T`RHSt2+c*PHzSbxouq_AXKK3|!+DD6X{KO5zO>>QU`9qMsix5C2^JFOT zo(0$yclFNnX_`>$WWkZs$orzT5XX*g8jnu0x16hPoU%2Q_NZB=m0+hq{X#e8I%sQ; zB|A6e-?>1b2mG2K?U0uUc3LntXK9D~Di&Mn0&IS~tieXdJ__UUndTcmakqG8;yuZ& zd0^d;&g0LlZooW;!`Y_0;%##0zG-Hgg zSRJ`7H=)cYobZ54P{C7ILu`shZ7&Bp8{_;|F7TUGJM1>At#z7U+J(Mf2WL%2V*yji z*lsFmib~{R8{t`f-S?|CgKrs3rxEv7d}36Ph3dP!R~}lHlI-n|?bVxb`V^^U8sjVf z=;~vlm~mj%G__qB^KwE740@OPWu@#6KsHc#$L?2WrBvLeCAS69S5ocLf3ep^avbBE zWD$98JUDq{gn(O^_7S_wL3A)f$PE_wV6exQ@0^euD&OoywN(J%yV}qwt<^-76$KT;J6@jVLurm3R$78K<@F#1Nmh;7geypckWH&m zhFvueT9jyJLeOhV797Ffx3;D?qp2jz>T#KdDGDJwA@`Ykb2st1K*2ab99cfQM68>J zT&Wxuhn}B%0j-g*hH1dxA^7lRbj{rM8{=JzgRo;UYcf1qYkz#~zUm<|$o`3NvEV^J z`p8rrj5AKcau%|qg-Qbx`3e}G7*8=!FTB|8_ABMGnpMy?HAQ_)NSS~ySkSM*C-vmm z=LJty4ifyWjmaU97P1&{G3LVvx%;#(xpaP*j~NeS&~B6f-5$2AZhTa(-?fdQ@kuWt zd3-Eaw^k6?gAm~-{&+J}@F|}xWCV4?h1+a)xDf-oy6rSd@_av<(^kbvKUg`zI>cp` z*@HJzxfolz=*I>K__{qD4<0`_v9DLd4(?Ve-BKE!9Tt7ew`qOJYI$4^`@i0o*S-Kef$|t8xyY~byz}{e8UgR&RRj-kqpx^@5<}}nW zBPKX;n+Phc;O_r(52!t0^74HctTY(83OLcyOi#Mu7%#{+w#`)C>`C04r4{?-2$Dh# zx^}R0Cx_DnBZj3T#8ct@im#480jATLdDoED-lsqrS3Cd+6{_?DOHZ?sVL1Bf>>Ze3 z0xe1r-BvNFKbm=l=ctvnz}=E4A9`*>{fV2-1FZ_8SL3|B!(PU+ojN}Bf}kfACF%)g z5&-G>TuP>FA85!0D%P(FDxiq4+qJta(v9@4*OUZnDGsM=n%CWvxzLV@9hN1hLLw^+ zdxc*YZ^T|Z(&z}oDw%egV3hUJbrquU!P-g|Qn{3`8kOn5vAJd|Mc9l{QiMbj6lZa( zOo~;2+xO@6X2x(lEr4uS``M#+Xr?*=v`lkFC1T$P1#||Dnm$9GxrL#kR&bYd+Y+;1!q5-X^wIxYLhXc1L5iGi_{N& zkAl5e_TRFEv_g+`3em(n3t)!)n62+I$}H_yNR8stO*@6oA49mZQ9;&qvV{cfkDdEVS&W;@!)~;lL&U5((JNaImqd*ZwSf zTKD)ZAbUANnu>X4V1hxMR622UfxDh5;h5g?l zon)>|C{HQXAQOLvB$O-97I-;%0xP0z~PdoC%8CXREDq*qL7awRG66Ql|5^Qwvlnr=lD@?EFZlo#_ zc3mT)=^1hMZ(+RIp8xDsT8%9z!=wwQL}U?h^R4p5uoG%+*m{s7pwq5-B%F6X<$g8v zXuqLw=;b%~i8j?VOUD*15TO6?0ekm$pOAF$K+j_du>^4<^#26s#TsTu$i>z7uHGGK zN7Rlnm;xEMvH4H9DHja?f{nop6xGU7cvVa{1cuQf>BgRLX9v zK7zY*5{h<(TT43+swzsfrUOfb)b+BvqAw2$z)WVV#0C)412VDIWAS`3Gc4L$oN|nQ z+*Qw!0+kq4FK-(IDa16$iY@<>wW?vHWxq3wNjJl@s`h906C$rHzaN3l*13gsz+YYj z7WrC07s1Pb7kr&ML%hsQWnUNU1_)qby`xim7707V*m$&($92ysTHy~4elhkhi;b#u z1dlLR^!40K&kVpGX(^nry(ogrHl85#;u2W@Z=sHaepw#8b0*TWdt0lKGewRR16AS> z`R}a}1VjZjME!0batNllw)-slE>wf2mmlE8rG7SZs@-3~;Ayg8$qZ0F$vq`Q^`lvu zto9aW?lRX53atYjNqX)w(B>Bsxd)l5bgE)RP9S<=LdRj`eGActd^|`^Yq$jihN4uR zQbPQDSa;bt>xkj#AxC%@9BcQ*M4C}tP-m1lqW?`p;zj{1oAB1JR(F5GgN9(IHCf*3 zW#OA<$fu|g0W5V-WHhgLvRJ_Phc=i@kJhV|e%&qPpOTcgN+O#sMDT;0I|cq|7{jcK zEsZ22_tR7?7Q(pxg7059Legtgp7`kP>ny|D%Ob0{+V{c*KpQ;zf{zmW3NNovAsKc(|d}eh3<>>KC%hL_#-RG*~;(xjy_A84Qa%Udj}+S%L2B`?bu@5IU@V zplwnzZ*3x^sffQd)lL-Y*#l5I#QK>pFeQW@jo)jo_QUaR%i&N6AQN{>sJqTYKFabj z25oq~?pGQxA5ANg5)x-WB82XIH6CUW_62r&2bU{5n6O9Tzmls0_7GHa?*DpA=n@|oNGHq7M9$>Vv0Dlu z2q0^OG?a`jP#%1#L#ojw1C}e2Gf;4Ht2aowzz`SEB#90S!jAm-XRYFus^?lm1&0bf^bYhI>rkgot{Sg(RuEvIP|b*G^gRznL(`up<&@|ArV056a(V2 zhtQ56+A9BIFbM@(0gZcvLlP-jPE{WqlHX_tA80Bp64eW(8!VZv1X##1!u7#_*G)}G zsU-Zc;+Z)(OD7Vou!eBbYwFdesg}Y}ufgymiK<{midd?IYKdl1tl}D$&62mY8OxTa z#TO5p;0GdT2Cc1QWp=n5Vy`P4x*OZnC2tRBjQ2fNvh=O9R0hwKYL||${Nkh+{s5H^ z@gREd@_~u=BbwA+c>62+8*p~5jMm*rC#wkdXDN9Zjb;*DbUNp^;f@K-rf->TXj=?| z?aP=NDPj9Ql5!x*mm`ht*oe~^?^4xlW>KErZU^NZW416iJ6&q!%7reJ5+Baxb9hMG zb5iXUI0Vw`1EQuLC%(3tRU=`v`_Z@v9%+^cIs}Xibt2fsh#M8Inxb+y^*Sg{9BWDW z85TOT2tEfhe0ShK1nBAIYBnlQ@QQ#UiZsqf2LQp~26=_>t4V z@OJoZC$y+N>0i3@yekt5g#cWl!qpq-rxGN{<5NT9;?VZBkUGl^OK<6?%(m__tdWn8 zD|H5SI-Rp>_J4VT5+s!*(5wn zoX_Tt(kQH-;1!alCK+2=8~OWcRja8GfG#LI%lFjEXnI)^q?I2Qevb9M2Wa?h{_|iU}Z-yvePlR&HgiE@ZOrCexLXl36n9}>8WVMsNn%v9Y5khA4W223>1Tl5BzpJII&!N3%ER{ZDOaz2{7uM`u z)>tdkxvY=a@f(Nv+kd~{;KK=5k9192>I%^tSRS9WP7&TFixfs=_(~I+O>u!rVDceP zo&|@;DnXdhtrxGpa&fnbzAv&qX8>PcYfN7&lIJ=(wxcQL zqrD2JLpOD);crJ9pL^Ks=bgDJD{!SoqG*$dH^rZLKKdrsxg1%Hs;* zkJr}L)9;(x;S&05ubr}MFy$izBR~QOj3EVfK-v`_$rWLb9N-T8H~S^BWvTAx|4jL7WRiglQ8(ud zw@qbl%7JQtyQ2R(DT<96VYm0THawy6%c?Kde(P`Tp>L%9=!tEs4UHkgx82KfgBDW_ z>LGsmc7LSmQdv8A3VwY!lpGmeQu&G8R}F^kv3(Z=QHY{_8BQv`kD20RZldu3-s5~M zawhN4h1Yww#>zr8n(GYzt2q9{9l<5&u?03 zkEg$CTKepPA99X45JQ~cWCJ%U!gGRSxD~CLu9)He2QH5gr2uxt1yDROR5;P}Lga4H z5GoSCP<2@0&IOz}YTG>5#!wJ!S<@5VEew^SOA zt^|^C@8mj2KBV8lR5Nv-G-SJIz1rC3uk7-iW)WxpsX)5BjJ|*(-JuTE$CU1058(L} zF!zP__w26KiGQ6u)w#Axm*+!D+YIRF3JTA?{O02J#8(|z8qbg8)Fuvk2r~CB;LSVu zD4%kaRPxH=KepQ? z|FFY~qMvGkM{Q9*#Vmes-GX&vFXL(IbkZT?2b!-l|HKqww$p2ctVx_gdMM{hBv`OW zp^69``i4H^DO;*JeKoAt0rte_!s8I39*WvmIeFn5Cov@l%i zH~d{9jz3l}S`|r}4dO|9NHI{Zmt{6U8HDZZd-?P1LL#jpK5PFH0ht{3V{!3I-`Suk zw6;~G?=J2{dya8hY^-|l95RlZLeegsl00|@G&TUpY7!Mu>kZgeUYR4V``SOUS!4O$ zqKDC92)fMdj6N?Vs!cW>zsDE(%AIid3#0)=ka|;XW$fT|OYh%UH>$be`@O2;@N@xY z?UWI@BRJmfZ9#ss9JmAdBbZy8NHzuM;a?8gvjG2GqVy$9|!!iz+8#KEe= zd1%K4*_6W#^;->8w#pS;Mca%njCApakvC38U5iq0=%s;S0Pj%9vV#ebfUiweO_CHj zI+9oVaDZSHXq)&I&sE9vU@H8lSrH~IWv{dzpmnCcwcg(Ek~43@O~5J7rz}YYU5>-u zri_fgbX|W24g3N^2tq>7?%1q7D%yAVx91rm=V5i3ztiLi_p%IOz!W?X-g~}rHqQKr zCZwt;ZvSp>AlGdrHiIV;k4xa0cyXNpfFFps$*GBL51b9W@QNm zd{vpPXUfb8TW_=tu%}xMSw8C|G5(HckyW&Lmz%VLP>AoM-a{2i9~~?H`047U%=Rgw zOScha&@3c#QSQjUcL^7^O76~rTmd@jd9 z(seVz$ITmrdiQzE7h!WlYq?fBk*0wbcRJTN`WGdj3uP-&4YnBV>w%LoO#h}+iqdI`fc0JX}--LKb!| zO$PXRr-zEm>FRP$k509F$Uk83LVWa^E zXb_P9@CCMj-FRqh7XyM84PgC12Y~|6Ik!TwRDl4kKtNTMjjpyS{t3m${+hN^5KqyA zVz5HVZDOeU;j5jrY!T%%YQg0Dz*IO}83U!E=z)|Q7XD+?lF;mBP(VZ&*Ek|1KIR^r z4S!QI9GQqDkF}ym*}))AAX`!`!|ICWM?3fMzJRR?3bkZ#8z7u^Y*+w{zgWXr+zD-r zfHG1zq&!M7qxoj~ReF@~GT_P5Qed%}Y3i=C@fKq_0+WRT2-w{jkOwVE|JlT}eeQ(N zBZrpZdUnJqhQ*RSHD>N-U3vGeTWPydz|gb>#d2VbQ=%oyBGW z3S@)(8zMc>i|1!c#_4{ zq2%2|?n-kHRZJCtq8wQbG=qoX+k|c_RsFt%b-{vx(_KYS5XKKBVlt-?yBe=CSaxGMJ6J6IeU#7 zbRS}C{G{mcMs0>nW^jtN7*tWB{;gHuOfZ86&2{#z90ubrX$eapWs9#}1!}SU?7bQA4919ew)qm0m*`FUSY* z#S(xL$V@ER%(6|P*6b{S3Tc$j^!U7;Wln}WaH3J?1{TX(d4<#)nnS@vmp%G-`!6I} z#z{dUaDJrD6BQ)Su#AW{K)ot8*n?pyt(soT+D@zsyTQer+7V7h!?p@E_qXdvjaf-I z7<|A{lxNgNf$rP~P~Ypy(Brt);rmgC@IuHI@TH1>g4 zKI5CT{(VFznNdDsNU&ykOXgKSl^CqxjR-uUCsh&b_XXR_v6O~N_}zD@3KC5jHx{CeMi*m zxyC4VxBT+0Z2gnH<#_p!`Ms0Ofqi^;lp{?Zcg+2AM#pXn2}m8iT|%u zgL~n~hmk?fuas^FHqBGZo~{aXKQ0SjoC9~I3sXGTugQ9x#shId)l@DquVSPlDrHbD zyu-|Cf0+Z0H?jAT63tfed~N!vtnYT$_vNP9+Ewa6t%pjp=)=xd+CR8+ary?_@6!i~ zP+4&j4Yo;ieV1h=Ntj@`CUHPw3%T9)Kve(i5LV6fc}rhdh9a9oT)^72^;2MH)t_6M zVO481c>FP*Bi_XrFw^1Q=wAq|F1yr>i6Hq`8xU@!N|wmLU;lhqyo-w`q~P$(vpqZ=hv@c^mqBqBC6 zru5Qf*rR(u@^Ak1v_FnqNfpe@9eDyJ%`M2i;!l{Yd&1{vZ7FRUfY;-#i24<{*A0-9 zNY4clos6K4Fil(1QZc#@jDmB9zR=MF+=1rmpi5bFV*Id%W9*v5hrE3|VZd__u6t7yWC>^$0VHp-ZSTS<8pE8BL+s_qAy=S8uKiI)_4C ze?MdHvtc$<1)ZsZ5n}ary{|{)k(V$h#4LF{{8T3A8vI>Zx1cO$mX3&&}S~? z-+6Mc!T*L5uh^-)tsMO3#q;xW zcFfT14gRb9VWz@6JqeygPr`>(=Ma*8RWjs0KYf~RQ|xwcH+a><$N!dZZABo4E{U9$ z@-bCexIc({=?f#4;F!U}ZSq$mqeYRTP?hEP*4$*qD6FIxrhPkdYjohD9uj1WSC(=} zRpLc5;n>MCUg#^v>;TWj+UA6}gxPDk{16__b}pepD1(VTr4rl&yUpvA!U; zcNozT%VsScDMtiCpUf0x?}>rY!h>9A1h=TN;Kz}h?G<`7e^%e?lr;dAQbs#K%#jWI z!=Nju&cxz*=vEU)F7d{!(q?2y0d!&)>taZY#S|<7R8C2usozT+B=9H$nM4j}ai@^|0n^?l1kvh$dAnr`W-EI<8X z47BP8@C|CxM1P;r71do9l4V-WiqQW1Et`-~Hg@toi;jbWylhZ(cBz~VHVIPV@F%Qn zm~}X>L`k^#1S(AyMtL2CbaO>foy;43M;weiFkEl6Nu@+Rb@~(_ezrY7&?mvH`4T+f zZ)&rQ)N&QYIK7e3!g?tsmH8gSAcIF}R)LTCg$IIuMZWA)Bgh&64Gox@I&n&gv3 zP1(*fpl}q=G;jk+ANAb38Hd;T1G>n-7Yr8y3q^k5AJu>+kj@2?tJgCc&u2B)|L-qq zZ>sJk>*l*-I)!AsXhIw>_wdoCs+2weHAFnl^3W#a>0FOvEieI@)hzXIxa=d=&@~0e zXU^_XAJNQAb@v7l{gTF4Ro{*x6ihRxf1mbkDK~A%%S}G$YI;gFkll-Mf1j!>NjW;V zS(9DR65~iDyI{nV6p^8AFm+H7MnB0gWp^gmf2 zD`xSlf@Qbwd2YN5Mj(S z$304&qL^~)=<#L3(2ZrB5>C*2IU?5o@d&W&on}yEPMqCXQtk%|Yx}MC$k@_RF#{8w zbZDike;wQSa007#Yn(d1%0*f4t}UQdRuj={5rU%TlL$zhLUcjY-(o=5S6>W*mlyA_{Xbz8xo|>|cYH(t+j- zp{4)>*Nq&IoPQh#W#)7_(euAh&s}B*h!c}7tUD^L29oD&%0`2%fsT`+WN5NWZ)8h{ zyfY3=WNFywOz*ge1iO?mxj_*r5$xev-YVwXKpl~afFl8MGPHmE2RXR<`pX~Yq!${rV zM~Evpo&m0KAVYHc1OnSd23cbauJ(BFYtfBk2|^|;HlG?;Le?=cvF+RhmaklNIWqH9 z**Tnz)T2S)M*Ywcg!+y?jMcpeDO^PvV=B_S3K-nK((aEhb=d%tl?qYZX4Q!yEs88b z;$=Ho-PhwhyA{=5BqD+Q|q zTw^K2h(s*{%fy`vV3aPa^dKe7Hkng&nnUD8oZ@!|mTca$RGnIjk*zVBWYKZwFwls2l`x=NJ934*aV#yQ@-T+#`FIf{l zifKpGJ9V9SB68@;gUGb8kPja5jJ|^VINT+Xt3x|r5XGO=O+pZT9hx0Pu7fv9enxm3 zl=CDswEGDP7$l-ffa644NXdGYI{6f9up#9wS-vgV5C&Nq)>ZL_$SbNdA|WYn#l7^-k_%`aw}30f3f@*;9OpsT*>hl`fZkdGd+*7qeS1^2 z`zg{H$p`_O3$We`$aIIHLzL;yl%N5CJnt%nhBms{296^H#j@NG$H#Uvnd8*9{8LN? z$NzWr?Ha`L5Bl@np5&ezCC)n3|%F$)pVy55W*#8EX%_<(D=E z)chrD6w|Ln7lx2mYSfqoZ5~g~w5|JmOrG<#5_uPAx`w_Y%4SFlDJi@aM zXV5LvvWUr-7fT<+E1`E2yzl41cu*p^p}>tP!N5 zbrxS;*S16Y%^Yq|F?`V+6C0fBaetsmZ#{vdwzt{Qnt;o(^?OAe7*OQ6Aw~M<#cOX%U)5J(6cwyeQ@R0-beZ5 zM<9|pkXN0wo3?e{voiI4UC-kbD_iq}zKeE_ivC9lSKh#$h7~QH`gh^W=-aoG;mG7H zbdFnx(6uX2Nb1o<;qG3tT1R$J;BsH*88N~q&f>`rpA^TK8+$b!0R*O0=(+P^7k{e5 z_@$4>!lA{hpHPVO`}EjXz)pTtJI3)=FcxP0ZYJAy{0?de4!$jTt#e&e*Tkx;1M-8r zcDe)DGG6CQ)5Vq~=gBit{HvpU>9ZT}A|UNAmH^7#L^neaFK|K6FXiG$Exb((1(=VC zJL4A;uo2d6pJX75wiz;`^tNH>PjyiM>MydvhuiZyf`SmHBjuP8|8wcemsRuJ{7_oa z0(cP;0B@|4Pyvq?XK63_AT8z}zk&*CXxJyk06e&QE(w7G$0kA`AJ`WhXY`$Kqr@sl z5eX32==6%bAQt^=kcZ^*z!Q_y5s$=-#$L=NeAncnAqD@R#=%sp1uA-HJSTM@HvH)P|bR=Bt6SkmjdUC4C6F2XG*9fU1MQWS$B ze?yZM2Wb-&7J6JWVZ%S9NVR2rLenri$S!^=F`TjK6Vi*?+I-d=EoapDn-m$}1V*4A z*nf;O6j@IWQA!W6y_UJEcoJNt6z68IMpO9Vl(EAF>@L4Y(OGXM*Iupn zFXfS8a6Jr=9^^Pe6Vk34_I49WF=7sdF07+0O$e6kj0XaovqG=O0l##P2#e3gk&v_Z zmIF8qG0^pb)jvuQD)LCEZ4)uUm~2k$8{~D;8F~4z&>5;!XUzr;`X#zB>jof4@kvQ6 ztb5zWHSk|Wge+P@k^{s-shbz(*EW0{0+xi;H|2NNXVvZiT_3o(uuE(|5~g9K8CE^x6~CrHCUG!i@Fj$joZ3pw{nR(sTC!o^x)& zEi2o_>Ym5dgYqq^@Sn|3amK{x0P}bW2~#hkvozGy^!ScaL1DqW7Gwt@j84ws5)N?( z{kB{w=ay6b&MTu)1O+(?XvWTUdcZp79_*1$v2{A697t&qWa}7iO(;x(4PNYFHU)!I zW5=T|x`5rgJm3X4X!%w;Za7Ej7Jnu0eNuEN8cJ&7fSJ5SaVjJ(*lgrg$)iJhHrh40d7$a`3b$q5RY%?P1UOi`Y_Or|ar+cYOI zpjk*D;7vL&S(Ez;iCh=lAVH&|S1TS@tUIhtczgvVdUcvspIs{PFl=qvM_I>RidcYDP6C7X_`rCisT9X=@Ab#KW<=cPahXx8?hO4}(L2SSdXS=I2H;3Ca87q@8`Wwl4^fY1ifsfdK-MGoYErL5 z{SsF*pM}*^T-_f^wErkWol{4XUSvb(JQd~k`>OquL$=ladHYB{nm%aG#YP^gOjEZk zg_6p>gK-@$wKc-JwZBgIl@i=N`6{X`%mr$UckS&`Bz?iuG9%9bq-GQVK7se-=E0@?CQy1b4?#-Pq^(j3k@28T_!;=n|7f7`huf*&eDw{g zC<6+H3IqfM1teo3E_)rC=bH}#1e6O21jG-73uJ8QPW{z&%{j6 zNbg|pv8I%4}xB@ISXPID0sl>Kogem^#rrc&MsC1A)q?duaXVa&de7vllpf1$X za#_ti)x+`5N-{NX!00cb*^D~|%c89;oez&Dp4hBnBHdh4pq!MZ8>4912;mP-QO*k1zTd&Q2niI4^?J_JN}+ow7SARudt{8MJx| zK`T$QE2C9#N^0b=N<*zjh0$)9dQevEdfuT?TOnjqbBuhlt~QaIL@&%y zmsM;giB1qtNkG{&7+b1(Sl_rry7aS|zC}6R*VnJLQ=t5PBD;B9 z!Ny+t3dh}G4OgLxP+{6dc_qGCO=;)v3y*>tOAeXasu`WzNT`Q;GYjU7$i@ z4R`U!NYonvHSR}qL~*4fu&DRajsDJCoc!{xvGdIZZ#!aUKwywYk1>2aLNMIQ;CAr) zImO=(;v;iW2#Z&;K0F?GWAhvnTg3bklQ(F3_xITPn?HBx^cIslu$TJC3Yk00$Sj%$ zV(dL;M~Ep?Z$vzF@G0Ub9c}7!BWAj=`7~x0m&?Wuush^5?a22`zovt->XY8y{HVBt zMW4Sp5-&y^?v^cj9ut^3_yL>|3-|+WmLD-GK>*$n=4juv3fw{60%UfygksZFe|#*< z3;h2s3}35zv46HTpuN zf^?56Y^U@HkvhM#GBK*^H;W|y(Vz<%%p%*~QzvRgMC{{V^=`8ik^YWpTmfCuw~1sK z^zb;sQVzY4RmUO0keQfnKWt4AD7GoMqvei5SfxmZRgDZ}t3V1_EjHMXIDWBGSAJqJ zv?@pQO2w;B14CFV4mu#HtDA$@zJ>q2)=+}ZNMy~PT5PaKg^_HkT(47Gq3oCp+mhmX z+F}{q2mhpPEj&XT${Oko@Vi%x`E!n|r=^|U<sI`Hnm@SMJA2xL z&vMz*SD`w~z`FG!fgi}S#jM?tnKr+vJFqT+--jisGh=)0VE-rGEU8(2>^c?gpDetz zG^%L~8Ggnq(v=USJdDPJx!o{!3H;94lf6+=ckAN#eE*or^W@^~#M0A|sk6~=j^pa; zS2g4#G%-P$5@vk48VMBgXpKijojBDgorJ#G z;zhU);EO6wgsTSKn81eC5aI=1mE=f9*LpS9nAbqb&B+;dRhCAh%POg6`cOy|P}^jkU3_s)wec;=8)uM%4KC*^IXA%iQJKSZmyi=BiOSYS+h!5e zQf#Gy-L-r%b3l@`M74H4Y+06##$O&`kVcd&4m;Vtp?Ae++>TULj~$#_`t0v`!*siKk) z4I9@*XnL@TErCG9rh8OxRd$*~{Jy`@dhfYB=o+bo+cKTyqTQXqhmDExk4;HSDMk@S zfgy43y*t+O!?kgbAg{;@=!-zb@_%8*5LOV(Sa|h66JYpao}eSi2NE zG-)koip3=y)GQO2<*6uy8a9AK4TO4WtbuaE$^aJPFFAlj7dsPEwlg5BTn;iU*X-#i zy=8uj=|KM2H$K5IIlI*rOzO&K?v0qKj91j&4Uqf{Ow?2Eiw=(#6qNC%5OdRVD*Msu z2FJOR5<9FCj$lXY*c)@N8u!mBTkJpA%k2}JR6ucP4=k?S&$Rf>SJBE^PUy(3DMnMB zXQ^Flk}(OFp4a^o#Mu-x7Xn3LO7sh-*_NWnV)N*`zbi}eYfCQ#da~#q8WptYu5QkQ zA9F8j2o4kqN%5$&PD2EDd8Vwt>IXzu3AW%{q0oH2OB^~y{kp4euF?lg%U>s8?vHu0 z&LQa7W}(n@;J%}Y7Ywy7&^?^_q}Rx;)8|P*h0dN!7JJ?mD!U?V*%(?6Fb7CV?mog3b& zElP#9DY4B%(!aGgI{O|57{5+02_}l_gQH)?;rRF~~mHfc*y!M@I!$6l{dh&RSCc*ug9U_u{p` z?Kj@igQt&PIQ?btZ5}dhe_D(gW>gP<@#)6CX4~E!>|VBziEv$C-eI)@e5!(*r3+iW zGNHoOz#>WT(y8?TpQ!-`kxBbmGAGaZT-^%;;H6vH#k#@h+_i_QVy=;{TU*^nZ`hdN z!*`kD9fB-cxf(qpg_@rzv2y3w3Ny zDMhJcNmyd4XV1?n6h;heK{qXoeQcokZGHNzK7eeSpLbiMgr5Zjm}^7Ieaj&2dgu;s ze6Z>edWb@R$Ommu?m&q@T<<`>afqFd%4OPZd#b~Fw0AAc2d#5Kl(~H*u}0w_AuS%7 zuECM9eK%3vSHf(^I{u=_Pcsz;<~i*agDgz?{_}=wR3C5irN_)8HG}v)MEi z=|G6hdF36T5+k$=sj7Q(`1X;M-Nu5;_0M-4AwP8g9Xze8_Ze-^CHyQUH(ab<*p;*@ zLIQ$$bK3kh=?1gZ>#o#6=~T5&1a70SAXnPPo>1i;@tUVd;25wO50t0FEuw=&K%Y?T zN{pkstdwU^M9*ZZ@@{(7RG$}=;?C)xe^Q}ThD6VmZ* zm4*8=Ggj8j@u95s2xne{w58Q{_FA`7ZaX_i0A!kdA1^ej@);-#igl~%L)+8=1&;~O zTtN`AOoydB#Z5?X7ulkq*;DRKly-=KYokKuqQvb!Pw18ggH&LYMp=E1$&V^ZVPq1F zv4wUGA#9N25BUE)6`5gKib}u$0GyNlhvkB?y`8bElar~P%m1TXn9~#eXUFX zgD+!eS;H=>zYJ0Fmm@}D4bH|MO|U*+>{!A`Heq^tKJg)bZ})Z@<~I47fI`8rA(fN) z%(vHhcK&u(Wn1ZlZDgM{ULUTBw;)`Tr*=th^Yths&OCDw>`!G zo1$3iA=-zv!vN2g!&SQMXHVcDKFwwdo@R=bIt3s?q(LAM?PQBc)&mW) zRr+bP-cf`A>~kVvJT23hee4IQuV(WTV@+&7W_9FgC~U`${XlG9l5P!OsY}wQJwso2 zk54C0SflA%ITflT9i{J|hic{&@DZp<#2XS)srQ)E#G4jW^+mSH&V&g02A2Vq{snFt zkjkNmWd@D(4xy8u8__<%^h2Ho4ZweHPPpzN1=Yngo$9BXsSq2GF=m9PH2YVl98-={NT!`+?A2T#`OGY2Z6 zMVzk)fm9>*n~RyRj2A2ofr!MXg>D)x*yzzvK(yLG%h#8jGsN3KAxX)jw|SDGLGW>X zhdbyrtLU&)M--)^$YpthUyZ}jG6NbfCTnn1jfQH;kIrEyG*GT-$)P7vW8)g<3=V?0<^Xx1msq1gWRXwbmn&s;k+h zA@J#QMidh?G*b1&7tMp8Vsa|lg3xak?jsSxWKhyHT2caf3KQYBN`w*ME_Q^#@0?Ln z)9;;{nyC<$qs*PhERu^BM7t3gY(A7$4B3nuUpM<07TZR>eB4|4% z541Wsyb|C-3(>6)u1;ibFLtg#pmI~92VLp9DEpUp^1`q}@3~SoeRMB1FAkaT zp>VoE*@dvya5-dC<^=}Y4dsQ@oQ@9?59cIsRGHRHVpLc>j82U5Q7-oNG$oF<01`lO zo6v<^G@)+k;3+&f>5d2E$VP0d1wXV1`6o-O3_L!@riO_WJ4TVPSGYMi1+W4k+{Zd> z3_2neWPMo~qge(hfJLA@<4pi?iy9gELphDMA`xF3vQeAQ6eXa8tqWC>98q?C93U4# z^$_~*rr!0FYA2r|4Hz;_1jh)a5D`8o<(>U&#Eab2Vol@W4vRmldfMmS({VHKZa7yZ zpHnTZ;lL>qVY81lH z-g3NDPM;W|I>?6CY6+)l(r_^WKXAhe7-f+n97gHv;?eG=h>F?+)D*6TqDcgHO940M zFu(cBTedAg9DQTWW&0oud099J24yyl>Oqhrpo_eFG4wq1yh7|X*W1s_p=DaVJQ&|iP`s2S^5n2u?)`JhR4v?WSESIbql0A_xK^z zu`BRSt5`Lh74yuYt4Ghub17kO?3JDryZN2}G65hLTl+IH8BJ^F=N`j{eJ3AcdZIeK z&MjUI2R5vMtBs3Q&SfNlS`xD!V&W|Rd=^Dt6iho$;B4BftTwf!6J?naaPAN?(<2&F zU}dNPw|pSCRk2QnZki6ozWJ#Hr5<+YBkhQGW&%0XkLsq0{;5hi!%zsP8Qhq2Qc_7R z1i(lv=d7Q)hJEJV0w$-<%zi;qvlSKDG=wTjU=@#OO0LS86Ezi1g8&Q{BO+&t(AE%(h|9`h{=(17o$6={96u=KK9Nw2H?YlKK^jIO$>n&4E8ZtvKj3Q}Q# zw^1I(YR*#8-rwg2|4WGjSH<4^$p^`J4lENL6W=Si{{^>6PP;&zKu%}elbSKQ6kkdN z+(?5Xb$BxM3qa-_7OLIb7AEvGSJI=Eu9#<&W{CwgL?Z2ifz)3esO~?I*DC17>ez+t ze@zz#MC-aNaq_TqY{&|+VIk~wkL{9@S`033Rd~&ZwhyUOvyP= zyRortL`hUH9s_+bMI8O8Kyi#_HLbSiGFI!+n$~i!Ah*==-tSptI#{pyU%FJJRpqWz zI>=4KOL)b$$^A_7l|q@T7WT#rK52jfW&RrO%|E`>X51u}+EG{BW(d`R>98Mje`KeG z)B4heKMgn>i9s$(3kvxlj`a?1vR>xK8;H3n2w>(2r)D9|fiB}N&u(IG0g2~zl#yV& zMPT>mW$1w7>aWEOsI$5Mz}U1=v97dgfKEi~^|CxJm~(LYZdma**2XpX3~?zd24Lu!ycq{|qewn{2HzeAzHPOIqhXd8b5 ztk}fjqh&|pMg17Rj+MQU?Bm5d-dlTFkU+&REa%lp-UKWq*BR}q?K$dRQ0qtDecGJZ9Y1E`l>$wx(YbxJuF%$JUb{OmgCM6Ttk^>x zkHxY_H1Dt3++_>d_;wfSCMmQY?4%IbMFv55E7JKb5GbnU;n$S*HN6J zfDZ-nCR5{TkdzXAlvP20nTi{qR(R@jWfEi@)Lw7fJ)yPc(z~t^=TfXnZ8bgLXs~wG z%C-SudGYl}U#SP^UQDZ~Kjl^Z#?bp$gVVkMdp7B4szLzr1v^|foRrNn(Nxixs5ALQ6E&JsNP3(|} zSb2ntJD;_Tb?~NzbNODLlI)cJ+~XOJN>7|hf5(3iz@Tu=R~eckBtQ9mIcF!g^O05- zaH%`DWcAJXazuFxWox8Ug6fzk&XhE};aix}%=FcNcUwezc@%3^aYYV^mg&Q|kkTK`m-Tm(J-QOJt z_&LgNtFta4$*#Q1cNatadfMaP!S&(c=y<@cuI4xHe9b@k_BUkjUi&Q6^=ppTR~BmS z%Q1Sbv*PZ1c25^(=;*lbf7UNFk|%#k|Ir4Zi#?Ak|>0Rz6&FNY(L1rQ@%sDQ^-Dc0X3f-PVfZ)R>y+ALzBx<1N8j+6*EM-J5l7dRhy zms&VO3MUdC*haX%bx?1+B_mRh;kZ)I7x;C@hc9nF@`M`^b0@A_^tyj%UWC)q!RKFJ zEE>s5SuF}Z8c>d-M2E4u$%UF+t8{0EtaT8tA$1a@{Flrnnv1%~mmbh0WquwTH6ER} z9UIj%0clLR_lwC-XK=})0b?fhz1jaAX!LP|ME~>=Q#8+z#VfS#-6jzzQC0=)b+y44 zsbXD-W}^B>P0tspm2$?O~%UX+_>Ma=%-T3i*X>(PqMgp7WE!t$z=MV3E-_dqd7k- zMwS_@o4tpNhZxXx$E$W)yuFxl@@m(@0~8&)4o%V_O5@x{^#Cs z%QON{{m0f3|6@J)|C7DpWbgjJ4Gnxt;Jpba&Nz5;}g9`Fg5b>H82g?SDEd<}v2q?5SBwI{1NQne-8px^4ZOSjS)dW*aE*7Dr% zOglDc)VrQcq5s12)Tqy0(COBlUlY@)>mS6qJmUZ>Wl+HMQeM9yOCCstcT}yRISS>+ z^AZ-tM+rGhR#9rJ5hXvMx(Z+T3+X~1VOfUj$x)<~hnTMB>5!&?2o{*aW@8GUW|BrG zz(9l}BUaFUiQc|NhmIB3SJq(r<)7I({mPeq!NSCLsj0Rn?AU(dTqx8_eUWVB=vch~ zH%-VQgomoz%2}Z_wiGf&aP`s#5{wnHuZ`n)N*sPfjF69W9n-fMxb zlYtSH>8#IhZcf~8T=OL`B6rW(C>pFbqYB++u3Bi!CbI=_n=@*_==*#l#^8{q7@!iQ{p2^3Z zz6x&UgJ!?C(})S(?!C9a?fabA{|3v<_wI9XWC2Ul|D7{^A|!Qc#3v`<8Ho{Ze+xAm zS~5k6Q&6ANF+JP0GO?4umq*%od##t)7RAVct>NR70rDs~^SQ;Jn|vjX>6!PsTQ74H z!(w}j`gQ07B%#bUT8d<~Tj|Q7vT=P%%!+N-u5Kn!6{Yr#TkY&AWlib0K{fx#{B%MF zq;qaFha#&?<>HMH(+M+u;DAOFQx?vXZ?un+(KPqrYd0+`uA1)hq@mNs`r73z^2C{B zfjt?ACdZJ~MjuMjesWkjPViY|d)j&)O7T^DkFwiRV@J@F-6g1lp3!Y)?_`YvzV!EK zfeZWv_^)sP^-mc906p~ue-wQFJxyQV($3OFU;qDsESe?=Mr|=5gxtQP#vLyP{!PZ)O3Er$ad9M= zb=x=^#^VN#EtYc1`TC6Qp}cA&XUrU1dd*c^=kSAG%t3BVE|+qdF|f&@SBpZ$D>lRK z7%;$wlQ0)Ms8~Jfow&;qMlru9{f&lCGVs)=<@Z$%%ffajz^<0YAgT%xXw|&;zp?LMqkA7 z+kmaBanna>UGvAnD;6=}7Y2#t^aAW*QfWyN(hKVVs2sYw>zHhnQ37epG`v=p7*s7y z%m~T}YcZ?vC&_Ym4X7rPCRG$vnfAV?TqdPrEQ#}Hn;-991qV*llcfvl6PWGO$vW0d z|MCe~^gUX?blc3JMl$c93Ld#nXi|lcU0EOX5!-*Y2S9YLrCG_EZVPsqk?S=rXocer zhYqPME;@3pT$&zTGWj&iz2ITw5dVLEU|6gw29STqv5YzZfY5*VJK0zonYy_<89F%p zqtuQ6Z-n}_j;;Nn1k%rJ-8rce$oRyx1-}_HH5noEwiy6mQpd`L38!1q8xWgymdSe0 z*K5@!y8D)0kZgL6gm^K@{Lb^@%F4>K3U=&Qe@-2blrLsY(kBB)kKZ1L)Ydt1uRY0M z(&zK&VaaAEWTsd9izDS@nlzTix*-S2A1j16%$l&a-vT=$ze`to%_xHWxEGu-*SlNI zqmJ=|>V!@UVF8>>9XmaB`~ zBD}vAdEN4C zVd<_VZ#&%?QM*a{@>m`%uiA0F6iwtUm?Vk#+kw5FdOO&@7EKSs0z@pbuNxVx^;$J!}r8}2ki9p;`z3e<&D;-kUQmxRtaqBO(F(B z;Z{lj;~wokO?eo=hX(D6xvC3_n7gik=5XX#E}4^B2l6Y2yX=yJu#kBU?2i6^uwS9f zzSNO1>n#c4ceQJQ(s6lcXRHwAc`eEJcXI!51hD295KG=rT7XcG`}}MmO{YhSnN|Oz zeWBUJSR5=d#kaBMj%CAS@~7a>vt4#AT5U~;R0oRJ)8t&6dmV{rZMMDz{WgUF+q-Qf zmRt|lA-l&ppNEulb|@RIwd}bN@`20+1)?{_qHedtuf@67{fL*e3Ns^!atJqvpwHp`W^Gj)G`fT` z&SL6oJ8{Sy+I5}!wPnYBe~!X#io-nPM<&dj&*;#KuVjb_qktTj2ylg9?TY4W3~h@g zh|UrQ12mJSbCk&|C6V86de$*nDf7x78D5V_vS$nkt&oIyDyt|HL7ud!B>97*aX2zC z6qVNl56Nn!LG^U@P!Q(-X6`=jbSe?-cOg=>BkO)^7DQ^!Sr}fnMhms&kfo;vAZ%Gr z8kMPx(h`Kxk)E)mn?^19AEykiZgS_$4;0H@-Ds(;3%zd4}y^QG0R#S#PTovBRLP4=iFjnkt z{5pDckdA|oj3p{qPK-hq!HAse0H~bmX*pXM9}8#i!-3?1Mkk^XSk+GwJZpH?CnBUL zpe@b~(5jq%yYIQ=0Kh9C3A2bVHt(ptt*hc-y_OHnbuB~ zsZ!&3&NpA|&1|~0GAdjKtZpjKk&an7H68SeL}JF~0`5Bpn!XEX7WU%+5#>0|8icL{ z(|wG+&(UJLTH(aNikebK(ogjSZiCp2JJY;MjW%i&&g4JJuPUBISQ@SuCTFO>t-L-j zHj+YaJ9w3tEbs@A1AOjlgi3Clmb4bZh{93AeoA)Pfc~Ud6ZzyzBZE@){2x;#suB7w z-1*&kpTb|3<^l8}?Jq^!+pTrge}69UEQMJGU@n)4V9jR&#uT0gMc724gFq+A9&3g% z^fWnRRvpZ6#)sCmn}!(S$+uVGYjL57>mHeMgUhI+lxJiB=G&cKJmH;HgFik=AOb9$ zP(p)a!3rHe2bnTAl{o2WN0oJnfyp~~VrK=gv)g=UycK})EAh9&C|o^+6qTA$%MW6x z!w0n16*o|xNR|o(td0GmwKyOw_Uo03yacU9*_C3x&IwiOkFF=I6bV)(KuDF1Nsx8U z8BoFMI7v%WfbwRV`C6Q-Sup$u>yXO&*tDvPoguZ@cP4dw1yn{-ReZt&l%`+%*oOu_ zys}aW-zt8qYT>WXr7_iWtCL9B$9e&uTX~#qdE5YDLjl*nWeW0b0beQH7}AHZj|{n| z=rbc=+GBL%bAxS4m0dU7Hq;1pSd6h>RZ4v>tdr9U)v6hthGWa+aiTuCc}+k3ow8#kRJ8?bw#~?JuS{N zD|Cgr``%pcB)|{5J2F@|E-X*Vr)Ac?SiTe`e-OQVWTo5E#SFwX#}X=G?wNYmd0Y$K zLC8N3(O`y$bF18Zn)QI}j1gYJa?$!rl_0t68WDND@E()dAQ@*EaQhbkxp4cIz#fN) z?Ku3n{mPj4c>T(!AWRJ${^U8wr~%|DjoPAvmHU<;4VXD@+}>J|35UOWc|aJFxtI>5 zS~GW2R6oRcP~(U6hVfAFU=qk`eq031Q&w>h#R3uTSa~Q26jqHWZKgPS_vuvXw}~s!ajJGBBM)#k{qPO)ri5rj zT>xId4g3IJu>E_YIZWq&0IsIOUjeS1!Id>NPXMm;aqHI#B2&n#p{~k)fOIUltL4`@ zUfk>AeJ4510C=tc$pC9_;=muL5m57a*tJ~=>SpR7j`VS3&GFIvToH1+XJvT1=h;Fz~L;86q&QEGt>>(i8< z4Uy&6P0&P32;Aa$H>zJ|o56KL9;_F^fmLh_zONz?1G41yqVnIWSO(XHIiI}=a^{)` zU7}ze7amW+pstdvSUN`(R7zF$_q$f==*@pgsQ}q^9tpVy$d@bG3T_lPHyWJ21a<-$ zw~ti<9jUjJ_JjHDg@H0f%pVvAz|0hsI+Rn|lvCbmID2U9hb^NpnJ@giVM|h1UYcLO zlo|#qLpe_Lq~?5tD*leiucaSK=fA>Oj88iN;8wC?(KwVEzLW}43IT{E9zPuG+s5zy z%ZRhZ-g95fj}!lTc5Gge?0=MxZCvg$L=`TDx~P{zb(dXSf&V1CTlzQjR({uXv!tns zE!Ez}d*sv?4LHdb}>%k!cFy`PfGu5kq;tZ zcTvre%hX^lx?V1Ey`#EAWuA72Y}Q`-CfMm$`cYX@vd#!QLF#4#wur^%bGL82PB7qb z4gn(1-6yoY&DRZ=f>5_rl>A86`s%@lD~w4ttyc$qZ*=gZDLwmrN<~I_Ua|;alQ*7z zvc*r2eTKaOO%xS36-l5ltLyrh%~VQ4fSI`JlY*iwzU!23tqK{yb$JF9(aKZe+;We#mUsr*v0<8G`;_~ z3ZzE;pQM)!;pawQupNTTwKDO#C=B3n(S>=HOXIqhB2=)Tq)AqUNHsAs@%v*ZA+ZL% zi@N?Yi1s3O>@3&o-~zNu@uUf@tH-pix)eYrSWylrfNDy=*jiQzLd0=ULi&)9o3f4eoAt)5*o9=jpzVIeaxlIiR3QS>p*83{aJ?#CJS8Ejc0!@P zllnbwea;^c+67#-K)T=p0j%J(f)pK;L8rgF(bOw3u9{OPR8?;Ge;_t zkd&M&mpeGSA~~Ncy^R@__d8gx^D}$gbP24y88QeEU3Le<-1YYNXfE+2iRkepgqOvf zc2mBtdN{P;V_Obvh90tI%CG$H=}r_1d-rmbe9mfSulJ4{ykLRGjG75ie)9*PUUSRe zK0J=0t13po$`-{`3K?gf(tiq$fq9w|w_M~Y{i8VS?jRVoHU1PPwir-Y>T0IyXb(IN zMZN*-?T}!3q%M;Tn{=2-lMXo8 z0XeW7;k#U2OI%BAjfFiIukKM>$*x~~uG5aVr~N+cJ^Pkjkp3IPEl<39g=~0#{2p2c zk9)Qyo2@;d;mqVDmz8d=X}tFhmkhVQptw~a5meLs#O(|C8=oWCy!}vo2Pm+Qs{@Ry zkF^G@0ABJ(o_tM9eIC9om^{mW+QHPmPH`P;&9UG-e4 zHbp4b=R1tL`_CyC-!LmW8^UC%h&NBB{MaQ(q>nF4*_wDOEwx$Oun! z@Q8BOK>0fG0@g|)?l#+RAOzB~w1Yv}Z2&02vpb=!Q++hlux%%XGcRV~>G(cq{Lw&o zSFi6Ny@CAd`Osm~K>om*=EH{5bW+*~Kl=_xR-#z_D&9Gt1d3a8JsZ##h2btmM*bQ` zi#o@^UB+U57!oHD1r565N9f0iJLMSIdk`r3Hd_i)$l@QCgJ%-e&(hJOq)XjKI%dS zn9cKJyNOO04NQzxQLC`|tA|KGyfoLvXQQP$HQ{-d{g;1?ghQhCh`ALo4Y?T1+Ui{37v*ZGrg%4vpKhe?(RJqHaC zE$spvedBS`F^Ap4?`{SmWdcfy$tTb^O3V0q>A1*9Cip8Ta2v~CH|t;y4X(ghg-n3p6o#aV2)Wt{W1 zJ8Q!w#@+aSc-bbtKd*LOJEPekF;iCWbm}6{T~UuoO;cG-_5vD2zZ%DdwVI^98{<%^ zoO_JDV46yN;E>L}rluorjYxO3l!~jiq}ZA?Wz}@(GpXu8?Iw+ORGfP(v4znl^?F$V zDkIQre7%Ne?8YzG_cQM^C?4B6dq2;YN3>nsy}f+B-Msu=hU#lq$CnRJUoT&mNWMMh zV`=;iw{!IJWitMm23YsqwFGgl949AEkcsJernbE+rXLiUFKe}GUzuT2cQB@F_H_PslXe&P$vM0c zg}BS(DGYmUf&6MlhL6fH*6%oV0)#dGsO6H;VzMpVSd4E7DR_1kH~VL>5@k8>*}1u@ zs+yX{N_uPg%aLPQVyC+bqb~-|UxVT`Qh#+5Qd4-F5?W@#$phVOM(L)62!^Jr`f&f6 zx+ayGDpGu6gpyw;RXl&68ZMcfpylf7>H2y(x&raN;##03Yh3X2-vNDGUF z5?~^0jF(LmX&FaXekgWYk`b)&p%oWm$j{{;p zEk$T=8(>xyvRKNDNQ>TW0*7$pBtw$EV8PFtd76Jay(H=gXm@jqnwT9wKp-(+TZ{L< zXB9#xJy0B%hU4-{IH8*YD|Dt)+Xp8q@;4F9I@XBt+7ufZuO_%5I)Oe_A=3H-1@l^j zFuaq29!8#;YZ^c|{8l)?qR;fBA_X9fqgGVxd)P4z0n;7}qD`*&AfiR{$~znhMgSgy zY)!o6R;jX%>%bMK1wCH`b^t=c#FD^WKn>2jLz)g$T~iC*#}Y4wGaS|<7=Wp5Xc>=8 zDYQyNg)F(r0*Jw>2^^nkcn66ErzF`ynYm%uU%^195sT{3DmeCF3^G`50w@vbThJmR z+JrFh4NuW-ugY6BuEFuha{oG_NoU5axeLqUZ!n)rbO%U@^!39J@$>CM0)GYBMs6P2=WLB0hq09&Yahkwy5k*e6yOv}z0?5evw!??kw0Z4ft**T!(GD*?bU!OdF*6AN8 z@Z`>lFcQ=p9FNzFajLO9_!BG@y}pxL9$i!fH{emOliHw2{6?NEks^3z*z~Uv*BemE zzGD;K*lC>2hGbIO3>Zc{RyrRQsVNmR>SaQxeBqmto5z3dI0jIJ9Bh?Lm0WRn>^A9f zs%gtI%}KsjUaqf;?`L3+FcW}O`LjRO(I10`3w#29z^gEbK2n7(Ya**q7GC&B&8@gX zBFjLFtty+-H?%Cyi9{jPssGN5KQL_=6cYyZ4iN6}-H38_6WxicI@`Q8XzP-a9S zN&iT#wQW7?>(rKQzV_f^ZN zk`#zO6bRYr!2>k8tZW!edsTaF*gjwFQO6Z2Gd1cF@mem0F8tJW`z*nK=GL9L0&T~7 z;eu70Smnx)PDq>LKz^KI!ulLvASv)hlWP19F*<$yUrbt9(<;u^F$Eh6GYi zu#*=rHRfQg*~93HJDzj9ZanPjP=MQTegOxGzto*mWrob4D>(|=?edS>7~Um|c+Lt> zf8ug^(O6n>a9a@gJG*JpLxZQ0BP{$~wXGdYJvCpw)0nS;klo#XQKY#P+ArSKu@WEv z;(}nRVkc24D~K$EIoh<*E_x)XHbEXfv^-3lcyAK#neEt6D8UP0&ywK;)irw(N0e?A z^mEBd-??t3cHrDVGzu#h!`)<^sK~vcjcIEve_N}Bs?_2wHn!UP{zvZf;cydl*ca@| zJ(6}t$tBj+NLt1S+9gdN&K)oX_my?zvQ($;!U6se3nic)_W>7##Q+Y{P7$dPVu~9a z-2>(=G_O-=6*>Y)LDR%|3KoEj+Et^x`tm(lO+&nTywmbkB|@gm1*O%RD-Hm%Lhq#B z9NlGf@jQwBZ3y-0ytt{I^#ccP@<~)%0$4akizP0T9t2E{;-tYO%x8ej+UT-jNZH6n zeA28B%-IG2`3|gDseeP)rmg2{#|X~1d!})Eiqn7+9^%8IKM-kF&)tf!rBoq~!$Nmj ztDRMmH#H!}P5Q-|GKwR1%$IM;5LCY{{gr|59G@9ON|QXkKaU{zCFwgAStk0=EH~fZ ziDE3mWjJ(_oSUKMO-Ar`~ctg8QlfitOjrqErsRE*mf~Av_RH z_NyJ?JavK);6X(b#cB7;?VAs1^aw9UNN^W``E8+nylx?Dlomt0;Q=M`EBb}?1PR<} z`1LU&giS64Fp;tgv&^|qRQWf}oQ?1WL*#_VY@LaGbe(JjxDp1I-Xha+d}FwEa~OH$ zh!s?2b9u10b9t%7I6_j`^~pZ;h8CzfVW6h+g61*v8@QuA-s%D4GOpOq zl;S>!Uh4z#=SP@-xIC=hS@-SZFaFlyx4q3Em25TnQm^74nVNR$1h3%yJR6hnK4pUI znJft|;23aO3a2pBbr#r?i=5^mQa!JO=Id7O2PW$m^zKZN^^9VaJ_J?#1==KGR*vs= z+2+2!20mN5b&&U2yI#hiDxxK?M`weiQ`ZetX#Hei+gzQ(*?YY*9FYY#xyY=E%G>FQ zzF~cdPf8^4&8Y@ic9u3g>N9Yzt@8RnjRRB!$W=l6JH`fk;|=RB{HIZC;SPj?VOg`S zv9Qi+kbikH;h&Az7PP#9cN7S+tb(y#OvBg>!BN_XFEHV)s}}%Oyt9>dNtKglyv2F zqonpoLgwq6>Caqlj%W+fmE_HFU8hJ9kxw>zeAwz+-UOS)tOwnS) zHXzzWOLpe_M$L=6J3*SeA#}T$ z)g_~3c)VW8yiPG=SzpHL+`va_R&vCA!pg^qh`sbD9iHH;i`>67LD4`y$pX3vRc=J%!pIwVt)$_Nz}vEZO|wHE1$k7~cE&eK?7su`E5JUl#dA+0g+Hy} zyseTKNr!><0Ox>ngr)s;gWkJbFCXPnkBQ#g))x%RIDC9NJq>Ji@84ut+=X?S0CNb( zE4dLMb014Fvs zsK3=~R)^k`DAvfogiuG|vyKuse!A52n7v700r0_(2yv^$r)-4m05Bpw22%J<5=ZjM zy0F)`9%WI|ulSnNqX#de#t;Y)j7)7-pq!od7naIBu;y63ma=oY%W$-u9?qH;9x5&r zx9qtlRqT^D*4Y}tzvLI_e*y&7Tokm3|E8*l{~NaQp8^E`jW^qv{x2ct77hLX0tozG z%M5xTsO)>3Ugmq-A?9`1Y-4Un&$-q{3Mf#tu%<*Pl*E-Bx8?UR6B9|Nc7+#m@r)!N z{Z84L|INnX*2RyeaF6wI|Gs~<@hNY2ia+vFwjRovw|b}@dye4lb^EfNRilz8zm0#> zJkF;BGwz7125198eZHMn`BK{1 z6MU$Ihmkl!4c5lmX~ci06tI-nJBGtb7DoEOq>mX@Lb>p4mSi-8zIy)!+Cy!q2%2Hk zv@X%3Nnr3B?<-Tsj_-wnQqin+yshQ3a2GNU?E!j1!U4?VzFGWi$TXm0aik%g=~1>> zmK<53aooV=hw1pWEdWeAws%3u;2~*ZA?X(}_i8`bB344jNN$SY!0l3%|0H(Fn_;1u zQ*MBSQR5*^T^(YJxd)S4UgGv&9f^V2R*Ou#uJ)%rpU<_9A-hLi|5|GWenz($B+@8%2hwB_%F`{_++i5=T*}3^HJyM!Pn9lJ{PsNSNtP=N1Bd-_ zxp=0@TPj*-CJQ}caS)31GoGHCv%SpAL)0j~j3^zBU{YC98ISei#}U zj&)&ijH=Y?^d#au7Em(#*i>>l3Q5Nu(F{GXXQI+zBQGNI1?x2r1r&gVFGm@GqL7@> zCxJmk8-oBHt)_R_$V5e~fx*onTjz2#+gW0OF9JsQJz?Gh)V37jr`V5HmX<%yc&2HS zMv}(%&p&>)EPiGBj3UF;rQ!KPq=2Q$!adEIMz20=Ar9mG07?{^6|VB_0hN&-dE_l zEh4C0oh)W8v!|P4xLzo6AANTYhlEL@e;OpFX+)Gq1_}6V-mw)K>M5q6A!Ag0wA zf=omi`8zvOf$Ing&HJ`u<);B^sSA0gtfs#i_Km4fNNcZS*4@2;pUG6oII`YVqKi64 ziJM@AJf@g2AvDr7LtLrOi6pUtlO<0$Of zI3>m(6B>LOD2eJ>f%L$)a0P_l;nay6p5{N)qIxFP6Gj0{x}U&Gp^>(`j{d+e zPoff>I?`v6Ud^oDzf+@NU+6WU<+(}l$t(m0ZWU-6HT~;V0%@=BHtEuvUA7g7x-pY; znqbDw6Am`!PohC(cv;a5EPSMe+~N`*`VNzu0Q-U4mqi72P<`IbAGCOX0{RXEG=0ir z=!U={edUMk!LNw?i10YE9U(+d3sBY!*Z3PgXImBNlXzLi(AIIj56%E*L* z^4u;%Q1_<|x6^zL)KlSBiZaB%9UI}qBnmVGRi(9fg-3MkM+K*|(dD;jt_`ERQMr02@mqt1SKK5j#k!WD& z7J^~)H5>;4d{lGT*(&q%o`0nQ*=Jg-bf=pFzF41%6fhUt+HYucbLdl)EKH<;Q}1rn zX$jxiC=Mhh=DU7{9Zl7B^mFt(TCYQr;-s8$P6(&qEJ)&%&paIe*I;)=jgId#K%jQmQA zjEG)zvXTciGo!`;9aMfFzpc?=4_a3$rHQ_%?ccZ)!J}$hXdiV>GUbyN%p$ zSI_pH8&ZX62^j|kR#>$h14hOg`|Fx*pa!sS{hWw;Go8<_M4Z?*PvcG?1C(wM1lDc~ z8P@JOkmYC*gBKl~bO(&8AC%yU4p6FEs>Fo4IZt2(chJK-5D`sI!`TqO?zZCE3g*ki z2l%dgkN4}ul^f#a*U;A1(A2>FdCt~$-NZ(w+eFyW75wKDd`IhB?{|BrM~e~2Aeh-8 zAV0y+NTmQuJo3IBvtXcCM>}*}gv^N^l}od`{8geh#uJ4n?4VZUG@DL2fxPw)NUGd* z(O&2_90Iz$f6hF_AVW@ZdR5bUfKI&}nQNG?f^cpB-@lc7%rfRvI(SRe~!fxV1RJJHkE-)!0~)x)pqPVSXw{Jq_A39swGI4md9@ zks=+Vpc!T31CNyp?l*JFLo@X`YA~w4e0r+oDxTj=`UO5VH@Cfge7vA5K3SRI=&WC6 zcH9xH=N=;F7%hTFk?pV(X~qv__^#NoXsAN|M1SXSx#`S@(pIN5W3l})0^PpQ9@u*X zX7hyBFcT_uAH=)d)ZuBT*)#Xb^Pld}Fak#g1r8Ec)Wr+YpuJ7SY(~XHb7S3~-$gN^;7SI@@1D~J*(^AioniFo>fkARHz{xQ$)7#9jhIWN%oU7;8I=Q4Cr1$rW z<=91Ef)R7p=}fC0-&u_*MZm`fG_rSGl6frPwTt5GgXg;(U23_cLDwSa)Ybgn8Zc`w zh$(p(GXvq6p0<&V02`7zXUg{v8DvoDIjB~muHeyoNEs2LF9KrQt z&>26%K+$+a+vV8^dooSk$Bc*gU8-QoB^tF?#h4sv8qM!k>7ZdN`di4f;_>+@S}kq# z?3ko()7Y#t?}r0ZGB^3d?`qqYsw7)RrY9ghBLaO(^2!<^ad=Vk31{?5gIpHg@)6h} zjZvT3hb$&L4^d|WX5<6QXT~}mAWQ^h6kw(FA*g>0`K(+b=Vyr;PL+-=ctHl@< z#Ut@Y{afz7we{UgH|hMNjd4Mf8wS@U{CtDsnKqK2#H~7r7iw~wq&@pVZwb%{M9fk< zB50JH*1%7|z@@4(Dy1d=!3V3YuBc!93y+uTUHmlZ ziYeeEZ&iZ1rbQz~oSHaP6#^0Z3V4KraF>s)85@h$q^^ix)40e96zyyy}ds5@Z zeStqYluZ1x#oFG=0ro&wfS}M!mkDe&3Vq^=T(x3_+pzPNOokLEl^4e<)3#dfEr$Vm zi=1Tl08#XqZb1s~<^x-2j2z*Lc_72at{j2INBjC3wpC~orKU4?8H@!NgKLq@hvDr5E zi1ZwE=9Q`ygm>Ci3?N$~_pc$*xlX55sbD2U%0xDjvto9=!E|+twq3b77Cvx}GEQ=g zLOxyz@JsaSo;+5IyOrFKUftY#enX#WA{7 zqP*60WXp@wFrZmZ)8PjMnAcB@+9=?3?)1a_gbHqR4P%BFYd9k3Q_#1hL~3}uvY@*) zmub#JcwUK#`9;_=K6yI=NOvb-e6f5Qu~GnTXIEyV(u^6?PPBJv3JgXq(KP_=;H4`Dd5n``K3HZal#m4E(q8L4Vh==}=|=CvGpFL#K}N0=n0exZ7NAu!Z+<>4v?Tha6I zlxpTm#9Jl~tME5c6!FNAs7rnTY2zF26cg8E3V?JCPVxY%jis0LZ`O-mHmH_^Fz*?) zR@f-yC>8u}-6MsAeK(y(vX%Fg#7c#Ud5Gn(g4{$QEX<&&4f{=Y0yjbtWKzq~iu01T z`)TdC>WuI5edqLK5$RtNEt*9t0u}3oFd>ZIlh&~d5Gl?IZSIA4(Y{YKIxY>EzE6c| zL%(T?drFF=6Lb6o>ZU=w+fjY<%l66L{$1kIPhh{ZmBXz2Xsn+}!$>Sr7hPC^k5>Pe z!6lvxYq4V8Ho%hxb%TAEEW<|`@`VxSuWm>E^I|zu2YtW+t6DOAVyIw@L#Eyh#<+R( z*&@a(rn73woYmIspHI!nDd(>ek+LAEV3OedE9 zOG`*=3l@x<(U8@^u47No=q}%{))7Mj!D>*6+_eQ-N;_qMea#}IsEeo9EyGv{?}-^*O`h0(jM69F@ z#j!aYOa$FiFauBi?Q83P@0@C0%)}iWYK}((e2VZOn@V<+a=;1@iB+G5+>DJ=8X1QD?p@+}Gx zQ>Vw7`*TIn-a@5QyV3lQS1XWaYFC1qY}r1E*W~(Z&LWO+chfvS12^sm?pWThh)>im zyq@%yDN-5p#ZZcru>!dEJ}_*4W2J~kEraNZgXT)=jrvG4tqB(}$Z+B!!t)IU`E0o- zJGP>;SYkG>=?roo-pg!_l9{Sq*`^oq2MKj@i4|_ogvJB(5alY0=5s1%To=&%54FKpiz|?sf>tR-^$Z zGE{%F`Cd|#J%mVrZbHm*XoOW-f|P~(`JCAI08 z+p}ICfYtKZMTO@P;twMXN&DiARD;i>nCg7IKcI1`Zb-|7A|6@Af2}!~h2%=)lh;f^ z-3%a`ycpg8OH-AX@)Q7F;L+zVp@w3Zg(9wx6A;1n9Und(}%u(OI&M zPRA}?8I?r#{TsgBMQsjbNAl%mrD){VF8AgnZU6Rt_Ntx(wWp`mqs*pj4XoL#IoHF zbp?|wWUhzFBhN%t;dD^|J~V!~v;~0RFiq%0dVPB>o?!QO<5w0B3Wv|F=Y{OMA=VfCI(v#ZZ__ zt4fbcrY6n`RF5H3JZ6F-imarlv8vO&nNeQf|<*Z%)~DL9nHqT#vky zQ-Xiav>^TyO3E(zPXDbr>>Q0-dApu=^|rQV-y++l&X!{BqQOn#L-tE&WAZ||o z#^?gB!wT-lCX$~%p*h@ACOiG7%y`I6j%HXB<~BW{=t2RS^n5AHN;?Bz(_arJKL%xD z_z@4CP1MU2#2LM2?eRW4e%Uol^R-yg=)KwH5kyXKWrerhW5u=zxO{6LRrRK==N4#b$nqjP^fmu}H(Q@kP6 z3%PA!tx4IyRSG@9oyrF#uSV_0b)BxN zICvjH(2@orq)n4Q$3qJN1}c*v9DwMAbTtT57{-1%nGnVClIfV_lwaR_skEf-=$5o? zI{C!0t5xW?ftUA+gjfApTce-UmrN$0Ih$t*k?VQ9y}|mekmeQK8#GpC47$d=yNWR5 z5gR#x>N$(ZLTDFw?l-#HR%^}UxtlN-Q<%1aCP_7sw;ILH;k>djv%ulnC?l125I~Ti zFuAtI1H9Q9oD~d}cB*FY#jniQn@L&@-dv_ntfzC|Sqq`W$$8h8NtCw-cSLfk-Uj#T zWVJ0{@DgQqm}%O^Km0toE*w730y2RO^R@d;`MvWkJ7fK($5<+8iS zGg1+iRe);q`aSSgx&^C=t#6{9*=MgF-Z`ZdKnL4J74-P*Nu=gd;)x=|2$efSA-hC# zh0?b@e3Pox6uU0SH>R^q`l>!sdAh>7#fwl1lv^Sa9jVwVH^Rj_=BIh}XJG&YbVTpC zXlbL~Yz;So>v(wAulVe8<<8|~kDoP^kcaV4J$*5fdjF^5CBNd95Jn1W$vQ_5v6 zA7$_5GO;b6`HgMy?OSgB&Ey+lkt4rWfoLy?UbQKk{eRSn!it0Li`GgyH9D_ zD?FIZKG|leSl3B5FTkLf=%mjM;P>*}&?aWK2ZMo#T`cTz0|RKcrP<~AyL#8A`TvwNYG;qNEz zGRsI0^6C4w;miRBzx4n^&r%674`cNgmQw`-fN+1m|%H$g9y4t)?EMbBlvf;P0-06GoRy#qp z%{^rjXVy>fB+xSypt^LD#dlWGGzXCZ1B)mumt^B`Z5wZusB`>zh{wn&`iN1gON(s0 zk5(k?fv9+|3QBk+Yf+Xj-i?5NVL~@UjUj8=`ADASll?u>q0wh6PM-Y{cEya}58E8` z6`n|y8JdE+QiYEn0Ncj2rhnP^?9}vJwq#E!l1sh6RZb3>1v%G$mOELU=59d_h3SoN z5{uf{1-D2Wtq-Feu|Etsa_B`Al~2*)mb6)p@K0@3%;q!tmG-@OB+uQL=UvO=#~Ks6 z=c0&3XFnBr!*9>ow?QtYYqEt4dzDA%qzx0Ph*0(5A`#&Qj!c4>JGGDWiY~j}tz}v2 zk7z?2y(^EP3H2o)OZ#r>fS_7V|4g95yI&wAhY~TedS16WE+GC~OzRZ^^G)HcFE;^kioDeA(5{!k)Mb>)~T)9D(;*iqZIdGP|*t50k*7RiuyNJ~c{M&Nc@zyXQ)7NKn*B>|wB^^QPcPilMlg5Jg)>ed6%B{CnG6^-wK z982PLcseG3#;&vb`kdgr055h!`2zX?6cR`u8aQ#m_*}jYdW^RU&$y=<8pe5b@7iR1 zV9WiTZldrzHg!AA6Ib7TZK(f7`F7KPSJZ(L&j;f4Lrk zR#-j}x|#R|_%51xJw7AUO9^8hBzz>ftp}>|E=BgAK-(la%!FUCbXn>nv9AZvElLz7 z80kPG?-&ch{CWlURMe`@{7q6;zq@aCn0t4T;OhGl_MRQfz)*6jU!+P{oK-8Lk|MeJ zPY#o^G;N6XrNkC?yl1hWr9D!%Z&p>Q6`GaYAEQ{y8`zg@PN3t$Q`b!_dqV!hz%+s% zC>9;hZy^^toqJvDvw7~KWygwdeWfa&2!WxRX-l(Sy|lxp-hq&(Q_;cfv~!IU!GUAp z8fCppH_heBc~e@tutlf5wr#~#n#ZSrS0Jc_`!ZFses>k{?@q*Vc7!8ZNj zOfHd><}9ShO@Ldzf6mCKR>60lSQaem$6iWD=`=hlEFt?nHai3WcS9*Qq3xS;`;oel1UG zBoTxS{1whu-j++Gn8cmJy!qvL>W8eOEPhHgRq9{@PqzK(?CI|9`uh_nTzxk;F%4SS z=~=fAU0489$|&k{d9Z|@HJ){f5Zz`6fPAJ3V-&w zQ>O!s74m?>FK5}_VkV9hh#)JEk>xf31%Q51kY!thg7cd@PplPW2tVM##lo;+mF-M5 znI!}yB&#mg5E;Ync%#E%uLP06JM5N88*XfmJ8*}8R+nfDB2jKia~4IYO<%^+d*izp}E<$EX|j{ zr1gOmwqH&psDKnq@KVE@(#bm^t38yKTTWXUrTZc)=UH zh?89lAXZb^T35zqW79#LX+$ZIK{j@;R$`0;*|Kh`t#*_=AiynqM14uZVF7Ksc)+Ll zAgro&GY7^PfH9MrCHw@I>^Q5%j18S8|NJth!C!D{RzHf$0+GN;kkT;_v^D^2Yj|X_ zN9bM+4et@{m7Y&D1ZQQmpm$}LOZST}D}_J_X|SM5zp`x~Qz=JAjH*{W$S`IWJFo-o z)V&uMly;b%$7Egf#Zp=(`chfR?jMmax6;R7NpSSu+=?g_gYDY#Bu}V0GnSrp+Bu$@E;^em|({K`Bp$iS#uYUR| zV4N+-E-(6#OKR_(MI`}`iigG$B$W1F2qR$qxeoOMTZ#j67!3(=M{ zkK7XG!m$TW5>3#Z6qwUZ$Uh!^8JAhJTD1z8@1F&i`CZ~u0Z`ok8t~hXW`GGyoGFtp z4sLMq0&1j4kL{EW?F*-#q_b4s3S`oDBl&r_WMHLQ43o2@=+{JN6s=f%tYCd!npeaW z=RUgF_)~kX^1|e`zdzC147$QFC=d&vD)UZay{x_V41s}56C6_f&E*tX%UB#I0C|j3 zW*Kiwn(HUH%{wmkUUJ8h=md&?Pzr;Rb{{Wa@3U*DDz<<@|6CMkw=kKp3^{d#lY(?} zle1FcnuPw=*1$dph8g9>p#sxf;@)TtQQ@7&j!7j_jZilYY+~ksP-rjXJk*%<%GfL+ zKwJQ+q7jrkFfkBsaw5c$Hv`&u#(&f5>8 z(q!chUfGnvL+)uo;t%0)ah1Z1`GujwJ>bMZrlDX_m&i=CVO*kK95T?9;1lhLed@nn zdhFn#Q>@RBCR#U<0=cYM%m+8#X0D$d*|@$PeP2Z#eVv?sdt>9_dv^mBU&l|$*LQ5O z)xpyJUS-A2Fqy2Y?7!=eXKsIUMbLf0F= z*l5H*`#u_T^UE(%qW6$J(Ac$-1CFP8qCGt9h@HH_%m`!E0l!I{GpIoUnuW|*;O6nl z<2*-^rT^pn84{v2fNr{#TLE#2t?A{pv0{H@2A0U>tvvLlCk$;NKV>7x>FUgS$rMD z)$9XmtZ%>Snqe6wkWLtR%S5%9{+s_^R~}sbQq&Gc`(93RV<4%|M-OhabPo;pIymCk~2X|^OfvCbllr@P? zUy^!$Lu2R(v^q#2Ta>@1SGh8-A7v0v>K=w@0R_10ZJQZjwJTupLLH*nV2WbrCg^d| z(n|B0N^9!sZvy0bfhMm(6Iw*%o40mES8-+ROwhd!v}_G7p2i4n5Ck4sT4W`gq$bn+ zvtxOprkM$7LBU;z76Ey*_&XcYfj@GcYf=^AWdMhwt#k=JzMV9w!*U{t;ThGscHDvxILV< zCi=Y1*7DZ<7$=5jS0THO%k*PPand@*qvHgoF;UlpE3d6q-@%!ve;6E%!^`E-rIO%q zDLqIJH94puyMDJ#g6ny3bev^s9ofB_O9^5c--ah%vVP{@0YA9tK~pN~` zl#eZocpPjqXbDB8U%2FO@eUtw4L;V&Hog5Dam_{ZECJOzAM+HOHr#E^o}3FJzd~%O zUj<5Mf8Uz~ye`quLt{-GH7^QVnMaqoWl+Fy+oPBB;BCMn#O$r^bwLWJ=(j%L_Is~S z-dJStC*J~(c3HV8Zu<}aCDp8Z|XI02DtuWKReH9^7%5exO|~#(Phw5 zbkY?io}q%FHw2{jcIztrAyG_3oUwX_eArzsQrLRi8n>FUhuyuiVrO}RTsXtrtzhI` z!EGL!HD#UM$#xc~y~^GT!@e=^xq3&IQjTKMm(It>{YJbO8HEvYz(T8i<(?r;ZGhUm z5vTrpD%t#KJ?Cr@#h9tD!Q)QVb~$8}Y+-CUHl+pTXhK%3K@*d7A=4}m=$BY7HX`b=G22+TGYJQo^6FHhogw|W8D4U_1>+WVXv~PKnW}m zhWirpP=TVprui$TNaf6!@ZlEl>}m{e*$w`21wea?=TRqSIG=ao9$~sT`(u9G z`ZM6TAeDj%w?SzK3mkM0nKU<+A%y2S{|*F^YkYxFpHm8jmBvgR%aZK~VS7lPhco2S z^u?WjYz)`_X)pQy*Ac5Tc&9749d~;jnmHKM^n7sJ3~?9Hj;4=~8;UQ%_#7+YjTpNH zc6>_W@$n{HWm1b>vN~Jur$wdw8ULb25f0pycx8y}##OL8*VExjHfqf4P5A;n%f}Rm z;1at&w%?si6e zcK^<)sqnD*k8xJS5sIuKWe^07*B{L1tb+IykIu5n7Sxq?F~l7Qe2JH;4nOh_2iE;V zqmNG({Lc$y5aqmfnC8hG{$Y3Gas9%U#ucwcEnR{fhd+ZQjYZ|VDWvU@lyauH^9k4F z2+v4_=Um|q^j}m-$_iedLZz+p7Q{rWu_h|TJ8hIE{!t$8J`S{48w_cAX1$luS0fxueZY84sTLm`r<7#2RxTVFuwVtsXMZyc*4(9g7x@Mp!79ioEXVoFG zy7FNqo*R3r%(>+H39%k(=<96j&PB1g(BY#eHT#av$ycdUnE}~zos?lf$t@sAMzBss zF$DjXr>MlK&}|p_s$2i%PqjoC($}U`o%1Np*ooK(#yKVG_V21?(`+Kb*Nln=aQm<7 zBsbJVH;(@31)_NweTDC9Jh6voPSFq?Tu3R;Ao0Tx?FRr z7ZOJyB@7?VYtoT;vg|I!jh zf%|hvq$tQ8ILn#t^|_}t)h{4CPD?{NI)V@x6)0>vbeV?NYFYFK;@0*;fQ`==1p+|^ z=J8CZI0|`+r>7K;ocI5s_=W+lu_*=Swq#y5SFmDi;EW)Y<& zVHr`=51*Wk3?KJg?r&5W4?At3rnV?d*9a*cw;qz)qMC8Ss2?3`rL~q(qfK0(E63Mc z+Kn7-Mx77qY);osJo=iLXH5HA~ z_#>YRS}A|AnSKxuywc8p<};8vZg+D-w8?vjaTp7^alfoguY{J+!%I1?`3zw;moqy= zsyH@layeBnZx9dC)8@r4hZ6;a>UaMuKC&?Rwk!=cR4O z)-MvI9}i=8N_7Ii7iV{O_N2WZ#G28!aEdKG_P35~5ECYR1dN_j9ngS^tu!tQ?ST79 z<4HUFymXVy*m=KQh;bfWv5N$QHK){KEcX5tE|I#(+^Lhq!CLrV28#o2nCQ5JkDVKV z&4rPPvqvWnKq6r2YicUxeJ4rfmH5#Lyf_9`Yt(Dtu&EIEQwGPxoK@7_~s zkYF%dDD~(k_3_;MDwR4_X&UbVF(O=Qb!$==uvWqj(8((HMW_r_e6Tc8B<1(@1_hfc zD+;L`l^G+-Xy_6SRG7;kLr{vcxKa~R?(OjCkoyVh&6aoroJZMjDSiYy^4h&Ob$5yM zxYROlU`IdE?omVaUiTQbJlHC>MH9qRe6SKBXSO6i{&P%-#emC1jv8qES(41Bq+e;*OrSvP|2T`5?~Th zse)aEuL!nOqOH%Be7asU0T!r4V!$*tovMUHGcnuit4`WQ1GXX;-;Df&@=})D3D8so zhd!)fI63aI5c1GQZGTfXHZCzvNp`1%+3f?8h^GZ2d}Wum^=G>f&HvE|Tryu;7oI+B zb5)|RzY-EEbea;pLa58`BtTdnyN^)4IVF4FEqW{7%ifVqK#)$@h|fl%mMi=G{A^*#7a`d=e(M(tLspXLcH zUV1%ZsWOcX61t4_s;4-EpAvly=BHcEOQLAL5i_JiGq{}^dU(SrKEuQpr)9yKY&?km z`ENbh^U}eu8*ewYkuy^|NyO{n~HO}wo0x<41c*S{BGi^e|EWHppi3-;~ETbZks ztvN3M+ALr&NCN@hHSy=%oFg1q%v}_1;9xXZ!VCk(c`;4`9Xe5mg$SeCfVky`wn*NW zlmvI6#-jzTMDO4(*qqp>w)9oT7YZ& zxDZZW-Q;cAKiU<0kkiGsNSRYOY0naD%rzJTs`{u}&u#Jzy#@G5NQJ+g>ctv%p}(@} z_9KQCqvDbHCK=ANm~!#I>g-k#Wz@XoZvn74Wt-E)bo1Qw49mfIM2N?wBYEE5>e_ z4NROG{`cahV`0)5{7-;;fd&HN{ZE?fb}mjo59|L0PSvEbVZR}PS1NVoMnqd~G+DK`AjkUqzcoHGI>>en^ z)*kVQQ#pVSPqtXN&V;ebUfD_=V5hA?hY~*i+Gr}?xgdA!I>`pOrt4KiK#Y&-Pt%b_ zJ|dr?sUOJX-2JOV`_@#F>lr2ISr#FdOY4rTqzUr(rshNmANdAIgjOG}w<3P#kde@&|PzPJmLjv!+tA&gLBIMI&a25}hHQjSHP}KoLW0s#czq zqeP-qj0o#F28|)j%UQn^l5`Bdnq}yg$-YL@BHg%nWvoKoKAJzPcc)Z#YEvkth80R` zt~-NV0%Fy8tvFxWlu#eY=5l0bItH`-D*WaSsn;v4?dR7kR3(tH1a4>s7GHpj)Fcl*~G=z1cVvo=bNI9@33^a0`B5isT)2QtG(aA4SX&;6Bkf-!o| zj*k|2$Byq?aEMs-W%D>~?)c?n=FE*FfSE)Zq`~vwoz=_t+1X1AArdo){WMUIfFIj^ zU$JT_NV8p^atc05p2p>4p0GbXS7NK5e<;C#@PWFe9O0(7TX0a&QP%jr7fwc3W2m0!|-WD3H6>1|AS@cLWf z)^wai4~=g!>|4BNnq$A)eb%Z7W2(w4zSjf|p^55|BZjyRl zb8|gGcz#o?hSI@!{gz^|DbxO?6Shn>_1EtuU*<~by2E$XD|o7GJ){#9%HaTSnQL_` zVo?8Y8ji7&1x`Y<*zK@zedfPnPt@~&QXfRHUi@LyUSX|0!#iEOj5F!~!t6Z2Bj~gk z{+T^mH%Rn4LRK?x%`JnDq~hb4Wno_NQe3q*ewona0QLm&MKMVeHtDMsN`xMNmA3o} z0Tr&(-HxgF?rE~OiU^G&ot{~K*CKZM$q#4+1ZVuLN0(>Ry$Oa-ljh z)&ZJTQ$LUy zzLA)fF7=+`W?gTX+Ih@*cRM zk_06d<0_1f^&IFW=0f){2r~2}rVL6XO43);oIzZ+?tjY<9qtyQ8ERdkibmZ)u2 ztkk62_V8dACqr^q61`^gTwRV@%`KH+HNyXOc8rQbwrP+jtlpKGfelg%6;}h2!}8FT zVTdhw7j*8=%@CcrNf|Zb7q0ucfIK%fLPN~U2Te~~HUZ##16lVvP|}xfI|&}Mms={&uW~Qe)RZMI)uv3Gui?XG$!!fSRq?|E{w}{f<+b#_ z$H48w%k|6^D*!t>iydsN((uhj1LC(M2qUA?w2z>^_wn}<`FpP)riwQI1}*qXTsUf&a5Vi=wu~{`F&W z82Jez|B=bT*~8J?(A43lP0QfuZuY-1HqBM!uD8Iq~%ytSXi@v{~hUPz$p^Oh`r+_(2 zkFij}mzUd?E0u!9>=*bQ(~xlNx~DKvN*EX3R_8s?BgT6TfH(O3j4PIC8#y=|>vhxZ zNc*iI^`z(1)%uYMj@kFWP&GwDx~HESH_wWKcJ89Axy;D4*c`Sh%hV@3W2ef z=W@jukKj458R;_amUOe`LMAe`)Gi+-O>&5~91g_0*w8NmXTCJH`ciDk(>ko~8fBJP{(*;^&dEkaXu4 z&optIsx-=9$lFK;`q`~WB?Z!0b#XV)dz_I^vaAx8SH+>`i*>#csq1emxx9xvzy#M-T{cl@tlJ938Kq7JH zjrRCZ1z1i>s~)f|hTEo5cNoJFk=1Zm;t1_|S% z6GB)-e6wZOgp4wnMXad@+;PH$Gj9DW6G?T`~21C$h_A9iFL!iu@Tkk}Fm z2i6f1N~6{{9tegsZ>!z^tqrUzku#AC^B1?2X?iiT44QN(a+6S+FIMh!vgAG}b#7N1 z7q`EkVyU9TiCmAA@Gn>xl$I$Y#>h! znN}#^YLx)(z|fRD#8mTdjFY|v>6|T^x5uTqx3{^qyN$xThaT?-cpUKQ6gS}W5y#+G zZ5?M>88+YvNcL%xIysPC?%GPIew$jBv}@uL-&f{4_Q>n_;A`*-mp}f2MJL)|R09<) z4|zb{$D*^&1uHK<)WPHVm$^luOOmuQI^d+|rzZEn%86vXux?C?NqTj$ycIYrl+pm0 zgVkv2g|<%F{WI%42mkpAHW()Yf?q};#2fhp{+{^gVtEU&OfX-sX^fEQ2*;X< zQ@@Cq0MOmC|5UWy>0trtgHi9&-Xk`>hz`Kl1tKJnlq!h2)C3x-zs)2%z1%!)A^e{T za+QOG`N*{dd;ENvkZvt%^&pgkr{x}a**q`k1NEkC&h?Z=YJ6t%E)w!`&2{uIdv33G z-l6Fm_G%g}Qe(=U-#*iKT31o3_#2p!YYe=PHhNmXxaI5fW>Mc-fU(WV%d-FL^(lOs zr*+;rEa)=?L$braW6FWlcp80%eKRJbkLSu#fp>U9zEQSr)UbZwAIW_l27rw@@j0t6 zFNYR$!bBXYIo&g`aBhMJwDI=XtQ}<62-2~!?iF#1<-wlW9iXANB#-9($<9WOgrUbS2=Wpjl#!F&0jMfJh@85Y~$f zRCo_RmA@qAIBuSk0ZqUKf_2DqXqdGpL?L4zCb+*@RDfxUTij>lpWOEqLa1mg6psn> zVvhN&fzez7oqnc0gNsPdvPjO=eSWso>kC$yAc76@{R;3G*lfp(ACz7~v&}0PgE*e& zJ%L>zcmF$3#7p`+wYawWc5AZ@Zn%dq<{!Y}Gsf?UoKHS*hztg7Rs}D-ALqr;`tfR_ z(o^6zirY9qt~LoQ@D7gy3d-2%gO0f~j_4ge%4|sO7+Czz5MOVbhsN|V5Cj;Ma?E}& zuj)L^wFX)p5EFs=V6BG)9IQl`cDiW^(47PR?cf<&LS8F!?Z8*+UGzQHvwk>&DO4X7 zMu4UGLb#dK>P`e>Tb8gEe1cw-BP!7#-qd8GEl4Ku+LE;%cMP*h0F44Nu)tf|;tLi> zum;h0A^^-E{RA${e~wGTI#z=D*8rEcFdhVOeS*Rj$fw%&d-ntykWny@9yHotl#q}g zYR6H)@BYR-IhYmU#^0?24c3rA{z4PT4mZB9OLR<9lOvA)c7I}Tw77mQ`^a6RKpOVx z0JSrV4FqoF80W$R77S)}qA?dae5^oiYiy#agjr2;D9S52EznRJ#&GyPN1Tl=D~vqM={k;PM1RYAzE%3r3c>Oq`552Hn%S@D^Oj;y${_C_h|0Y|6npTIqN|j2cuL zuVsXj7n#c6XCSWdszy3>&eqUG6+WOB$|!FlX4H3`VyGjDeaZG>iOhx(k?1#C512G{1ex0j1|Fr^IsnN|>f4hO zbN7mPWDU84J)GEP8g<+o)VVrsL>|0m;#v-#ctZ``3#14#zRQ`hMq>ZodK9m?k*9;= zMX?Bf)gCbyPlW1}NB(Czo2VAd4Yduz;EK`QL>|AnWc?JF2h1v+e#_Vt?Q|@rCT!lp z>+{I{pY@cbH&|f?C(=)PvNAk>m=Mwu@sEROLqF@8?Y_Vonw#bJ+z-Jpum==@ErKNV z_LU&dxvX;k8v6%mjqqs@#FMk~z#5|9 z3m!>GL7w7o?fsp5gF&fxz+D#dc7HU10JxU__Jv73#JjZiYH*6Pfz6k#>Y-@0A^7>lDQ)owFc2ne0 z;&3_Ei!7EXcKSEy%1Y^O$<65tl8d6f)BA|_IodwYl>HFix1m{dkbX$8&#urV%_bP# zyE(eT(D+Op1SGUfHJFyh^;l_b{b+_p99Be&=Q}uVvqD*!givdH7TMCWL=&K=&q?u* zhYS54#7ViU86ueQVYL~{6^zBnVQDVocEhYso+Y#>GsUiS5HNozE{0C{#ZK)vl30SF zIiI~II-9tBbfy}xpX13*Y z@WLc#WI;r%4~7FT!>VFtU*nkPPK>iQNhG<0lx@{^Jd^iFEULYPKtDBKw~U|d9k~Kz zlt9g+w_hMoGvxlD6&P*VH^4<-TcpE4byZ}FECYhhMX3Yg=na{ z@%aT!D+5*`hO4FyV!~iD3>BU9s)=z;}7(-M3U|HK5grw*3`*x3n@d)zvfSt?$!=?`^YPM9TBL;LLX#6_R z1#NSQE&&|uk>Piu&mZXrn=gEg?VOc9Q|dzU)VW&fsydvL4gDjNA};TM74Hzyi=|ZU;lO=Plk=GGlVuveWv7gv zQDh0-35Ld~@iScqXG`FWNGTl!6s#rmb?{#Eun|nl5lZ2iskRGgBFjM=y`>q2mZ|)M zP(8n3CH(&nXXnr)3b3Tnwr$(CZQHhO-L`Gpw%xaF+qUhwZ#Hi^5wocJ1yxZ|nddt> z?Rwd~+qB*#7LCnPFoxv?0Bu(TcAAQW(I@m!$6BZsSPM1BSQhZifNBxmF@t^|tw{BN zQMrOxgZ&-IzHB@yxX11>Quqh+K9@vUac-7m+9ZU$-!N)Yd#pTflx(X;bA#jEMk5>I zFTXct3`JUyV~|D2{e0c-+^f_Hmh~JsAXQfH25g40h89 zYxO^M&|c~PRLovD9xV?x%vo<|y=hg!=IF`7yON^(>Em>%2L>IAUmCl|S&+Bc{VO;v zyW_v^f%HWR-!wohVhq@0DC`UdL%Ph5G6TWzK^vuvzBuG6;~p{amPbg*5-SQdm6#Kx z3LZG8%UTSx8@)u+3HH%fi@=6`cV~|;OPf9cwQ#%U_ ztnv`{4ZrkEBY9Ru=PZcV%B2v5h<^UuL)UxxIM$B!;Q@o;@L||pGjv;}cu%%wr!aN_ zy0o%*Zx?)U8D4cJz~)m!swa!B1I!ciO=B(D&fqXc>;O38Kp3<(5bz!2!_;BcCDUGb zX`Q%BRDme@FtbK0tSu(^b5}rfH|N{o#~?fr&K=8u%ddQYC1JBXXrw%%;y6SQP}|P9X7Ex=YuWNXy4`Sjh(&`ZAf}Qlo8kOA5W0MBLJEljl}bdA3HG4S#Q(K=frElA?yLC*VaiK1PCoVWsld#(>GCRpR#wE%& zzHRnuW2q2FACjHh^j=!plJlBL)3@Jfs}hR6-e*Qlkk@T{v`X1yPn*DnR7<)@k0^x^ zqvF3YjnshQt6zTELZrL#3%2%+i4J#?=7^Y7Rn#lAJw@OHv)evo!bM9VLG!joI7+8e z+bBnn>;sfOwDttq>k5`mLB(qzP93;EVsGh1HaD*H-Ph{MAfg!@WOnAYMrEUhx;4WY zw24z5vQFiqX6HV4eJdop?l84sHMcNz>vpnscx&5@{P!Ax3CoW08h*4I5Y7VpYGMGZ zv*#tx?RHhIc|Sot^A8MPjM(OJ;OIAc(mxoRTZUk@DkLL-iczvLqNx-hHsI~hWUVv; zmPL+UfG+$p1s|DpTA!U(EvAy?@?npf+KyTo3u9nRtc(Kw`(m%M z>D;PO$DBQ}3z5?g*o0){$+*2CQUw`M#J;$|@lI4^xXFd2&;yP5Ii6RzIMt{XAif?v zTdTZA*!mBco61=ls}?P0kF|}?C<6Q?>?c|?8gGH0e?FZ@?3)6zOTT+&;EBOGehvwj zQV*QEE!>b@ZxCoxy}BI%%j*2*H7Up|-TrhI_4iOuxMhPj!I?Hox@!%&mYM!mhbo@8dCwRJ-Kxtxd4g6GPLjr9 zpDfi%b{LO|xgGV^qQ3A&Q~^(y5AI@n@ptlowt!W~1!ladXpX->56{^a&Ow{=)~?sP zmY4p=%yde;w~beVP7SWeBW_%GZ5p$1#1G!+m)ri8ksf-WCSZWx=`YB)965YsD&9V# z$VSHidWVyMn-%;$;&*2g_I>vcoXvf{U8PEXxS8{xEZSmbEHCqx_T3AQoZ)SH1Xhan zQd`;{f5Nf&;ufp)5Ty{TXHpGE^jvpw1y^U&58j@3DOidKKl*T;vU?d;Z< zAistJ>xXi4s zjdUffeg7D|lAABNzZ$%DfA1B#zYTZp&j6g=HPmg5qL-N-fhSedSPfm=Oj3zKw$943 z#K+K88Fk=;mf{9MslkeaveS+J`i7()bG=O@Gu!nErOK9hM0-lm+jX(*R|VZSRTaDm z@%S~gfn)x8-dhLU_H79Jt+@gEq7P}>g)(cSy zV8-`!m>l=c+jX5D1r3L93W>6S59pgBJ^JkR=?%!z&Z6)I5J@y8LS{QX%WLLW`@0XN zOwEEKKm4<^liRksb30B-EXR?zJ6A){v0O*nmY*fR>6aUh{Q(oL*Ha;rQj3{@7ugzb z*WCzl-$q9O_ep5+Y%N3_5a@4?BthjQvaLB}n0pUUzs_rvN&5`GB4{hpNwW6Vwq!H; zk6J^>c@Wp7&_Cn@)rGIwHcl|jsd$4!PuUJDbuPJA^KqwL$W_~)VvI%GJ*7lPO3+NM z+?xR=-bef{ly6?TacfU7WDjZ`(ei|3Bh--6x^+T7(JRsyx;#kK2dx%YOIcXye|a%j zmOQ_+FwKzi>tfSX&lqv{wbg?#-L-W$`;EG+4ebVB)|)=^6a9Y9)-&An%@PMaa>Dpo zz{ccRV-y^T^<2D0e>?!S#pk&Xy0Hb>T+q)IDEs;P_og4BTi;Rm%hgaSs$ePD@qz!7 z00$Tx3bnt%LF7?Snxf5we8v&b-V@7zR$@ea+y4O3QS8M52P3f~#Y6{0=RDhBBreN3!7w|RXA?T~AV0r+l4ThTVg{{}wvpUZ(3&p&;?Nd5$7Bn#Rh zacQh(*QBy?w}wUOPFrp3jn0xUpL79)7Q9?`f{Hq^`(>E+g#L#$FcX`NAX4h%$1N%s*>pnEmizlnCvDuU1=_gD@`J}0q zeUKV4L0u#^@YlBh=G$KsyEl0!3&!o&4V$(OSK6Yd@~q0>BmNatd#l6V{p~d4(Qhad z&3L89$O0vPFb|LQYHMMtU4GqT{hCG2*{Qf|3)c9b8OJ;-E$qOv%lN`?vnzFn&5pq5 zwoZcrqS48wz|qO+KxYBv3|KVOk`}VkdY~NVFVO#nziIg@> z#|aO|=>%ILN-#g?b*QtuhbeZ!^QH0-WhU`{k806H?s`>>Nm|QpE9QcL-%Bgxb|3-d zPX!v>qV#MNwELD8AkQ3*K8}B6B>ns364J!s-tLT{cr2Z6Ef@+1imLKX`c06b-E6BF zI!;W=$R`h0Jpz-8iZj_CO!iZKAXq4RS_H=K3+5atxSH5!5t*cJz3RxC{kd!ixJ0+% z+&RfQF%>;*et2K(oNe;$)-#oZS0_z1_q{7=C4wA@Jwt%f79}{NnnoKks)-VL37(Tt zcv~Z3$}yE^W)Co@Lj4glh+P1I|J0>~@)N?W-lQ=Fon zOw2*C4DeiaO*q?%x^S1^ZHKIXZG+J1c%@1;svzh7Y@=+N=$8$)k{MSIyLuYHVCldH zfiwQ10*nn&6(ydA=B zvcS4S^HbMen8ro^`%tRrYBZi7KEw$vk;Q$=Y+B8bU37ADTVEScglFa0m*eZ;&qL15 zgb$Yn&Dt%926_Yl2K4mFtA-BEI$Ev0 zh_=}vNzX=uynhz{j#QOn7y$V-Wr{LWa5Npww}h_5anxw6RIs(|(`R)Od>p}KKjmjT z0)&8m0uCfB-~uTWmI_ZOj2=E2iBy)G8%eqt!;`We*tGv42W&fbyBBxYH=h%-vA~#! zX&Y|&o2ZYcg0(RsJ4Ux;?)lezdqmNjRY?;j~QQl z^V>oc9!*B_Weu@@A4_&nkZM7OEi1V)Ki445l{p5h@rw!>L>SCqen<`mf}0k0>Jh6% z=7Ad8JB6MyDOI0O?y%_R6LQJ%Ml{@$)();hQGRD`Km7jb7NfAEI$5g!?-2NlCmVXQ z!BJDK_9!EE#yE-szZHcKtzcq;WFLDUa3Gl102=n7ep^FvE7LfKX)IOIP#B5I@rp2N z{`Ges3qMYFuU9`qT|?3CAtRwJIm({z{W3mn(@rz%r8YL z_^7n+zaUD1j&2#Xf`)(tL!q32BASQ->%ZRb_Uq^v(BgWlX*FQ+1d&y| zT6f*UbL0A&(4Yj%R47wPx(C}AexIM8peVp;ox96oM)GMt#^oTmp`VFf?yBi&7k?mj z49Cf#sM@>#E(3QLo=P2%OH)-+@Ed{H`YGKK%+|Nk`*T#ukQ!tq%aZR*5fjME!h@@W zQ@jAw`~okf4ETOEE|4g5oQGYX+A8i|V{5wsAXE1zCjV-L{njejeYs5>vK8?DU&iAh zLYA-@HCj`?OUSybai{vWp5k0r^-Rm5XtkgHTU-?pV&kJ*Y}?YzP< zU*T;T5$eWct5rw`sOlvpHhfp50=R7c`(9*19u^@-`f#@>LVz!XNt z8(x7sV^Esn^6JLQwR3}>GGVH0Xn9sRqzr^jJ7@qY0S_CA&Sk5@%IjgBW=o)gH;tK$ z`=bKxKQZYzgBbj`8-wvO#(qJpKdEB&Y6ArCxz)K#K8ov6Y=^fgF+_Vz1;nC7t=@#Q zopFMmeXW^}8e5)1Xbt?KZPy4U@Kn8xA|14WPUwO-z{`zCDI{FrJf9#~K76KOrk*_} z0L_alzNog3V9nQu!#}^-HX`-tBqLWyy(I3_j{CzE*s4Whu6ob#XMXCrm#jn8#>-)a z3HAoy- z%|E)fw!&T~8hqRuudnz!rkv$$?GP|Gknk}vK;fUv`rRCl?BU;z_N|{5l4Z2$#B5Kp zNFZ=x2bIUf5*CpUJ)o>1Y<41tU$hkB0t6cBpdJ-}-VP~DjxxK*oD$}*X^TaW0Vj;h zQ~^`8Wy2L9vfN)X>%9N!+8S1T7+jI5@d`DsJwU4%6)#9A+y)L=LAKl9QQHJVLzBT} zNs*j5xTtmj7IioBPuhgA4n~>^k8LV6BUk^cdj_ibTDpNKR7RD07~__WC_gebJeE|2 z@xo|ZMY8NrTwD*dSm{fY=7*4weBlLL5aT>10K*(GZVV3l4I$_3M~RaQ?4};ir-oR6 zFs?vRic~0_iqo^_BuwT93~t27s6(i$0<#nok)+ofa61SpE)JSsL3suSR%! z(LTe_VtL$5bY&ZOrXkmPFeiR?v{P&3C29xERf#G~c(wys!x@B!#4on+bo?MzEXMVb zffrV?3Y37|%F+f0a6PV;GhD-ZR~1MUHwYmtmM}9`zrhmP_xw4TW~|fE7F5wmS>Zmu z{Q>}@QKE&q#~OoYm7|7W>+Km!1Roc#yD{LgmH zYwe6Zny~v!{ZcLkY0#f?#L-(L# z$_Egicy)B6U)?u;t*hJH+1}IM8QiA_UY52C z?$_Dw!wP{C4;iX>n+AFSFF__Dgi73Fm;8W`eiE(Fp^{{hDPy3wZi70l!Z3NOnlRuJ zSVDy+83C;cy3-PN*w^cqa`iww~$Qr_e~NYFDMB3HlNtiyl zW#nKWj*MymQS?^sC8}8=!)NOIUt;2hKX6x>0AG3?nNvaKFtL^yapL<`P0A>@mpESf zr-vk;Xq$Foe}v7ku*o4Wp{R&}G$9(7jbMKJC)#AVG6R07sDS{e>jhkqt1O$P0#KeV zYs9tR^lVe}qZeF)CX&e0Fa!O$h&sT~N_H4291!G(fX$3Ls>Dt&AMI=HW&`HvyCA1+ zHz;)HWsGqMfn=2e$10IPVhs9&`6x3cLFW!mNt}BtCrH#9HfQ&d`elFd8%ezA)&}dY4F*0V5nKQKJK*X77ImkU6JtZkQmVy0^d5 z2sdTEQ5J$M{hI3O_MhaVg**G`p?J<;dIOKVb9o=duRS{ki8*d@mcH-svAMKZ*(BEb zr;CPHK}pHh7$SB6va!dD$O8h)I|u`W^n+OAy6tkJSOIdM2}_Rl7`em8Q(nacF)U#t zG|{&tH|#xO$*67S?zn`L*&z&l39MqkglN3vZmruu`0v;US|k7iq!R%EE&|e)O6H>1 z%ZjHU63i=lMK;cw3@}kunlwrk8TrGqHDAjuh(ZPS;XgOsAFA9l!oxY0)Q&odWl(PB)x0Zj)-`v zz$#Y_YoQqh$9sMTRSZhSp;9!E&|7VIg!?GN8Pd7uC(=L4Xrf`mCDBxb3Lh|pwcX9m z-G+lDOy+2HU8go!tJXj@BV4#aKUjunsmnj&?<3jtAaL3QM`xRWK`8hsT4im;53{C< z1pw9P`ug|NK9b%^mz0sv3_DYElZj{g7Wg|F4SD=W|47qPmG1N zkF5VJpU`$C6d6ILF$^&r7aST!8=PgKlydSB;g12iM1%sk+H8-?7n_8^z!-R9BhHbm zgU{a~>Qoj^jNV5or4-RQ9mbXM@F-&=eUrWb!F1?H64~WG`%ra~=bCN=bA^lrYs4Eym|hqX06?!=I-S&!;h`4i*BANjL=5aODw-jM3N)5`~h!Vp6WKROY%9 ze@4aNR3^KurIJViY_fRItnOb38E{Pt2mt$l$w1jE@nOvZALYO$xQ-!Or3*GU%hvM? zzEINeX7Ta{KG8#7h2P`2>m_#|bZQadml<{?T|rQ`HH?fGTa?O{OUT&6RuwJJ7R)ND zub4_WSR?Xcv_-U}o!|rR*T57b8!2B0l`m?GfU}qm9M%@nHtr5A7177^&kcZv%c4PQ z<2QhlT6@qt*&7Cf%VOdtPtnsOBZS}^6?E{_l57GwXTbK|mbq!M;emBmxz+4)rx`)i z>7d!ZW8(59o5Y{;RqA(>|McBN<3}JiL_=i+RX12-orp7Mr#`2C2wG`unc`NfHTfKB zIoXbYQ=XITru^#*F|w@BsHFJp_2-2&)S!f<`RvSy&qIBmx6K&F&Srybkw7)$3+s!~ z!*W1F0geml>8uB%{W~@=&D;L0&}bW$oy7dz*bYFRGIF#by@)AOh}!^mu3-<%WR?nn zZ0#G3gv8h(+W=QO`c@Wd#S0F{)`eCbG(I4vVult&@;a%yl_E5;Pb_bRXK7MJpK}qw zf*1JO{Q#k-HTZ5#pKItuo%~_Un~1s}upV!?599>Ed;T|2Us|3!R!Z7+3+yZm|J4af z(r@*8fX(fZqwPnzeZt?DSI7h5>D3jUe__v&DJPIWbZR?Ru&nNldCE5{?p5-Xo~n;n zpfT*EPZACCP2J)et7SL5sUE`lts17|TVM?+yFHqu@jdRTyl_R?o#l3>f(AXuV00JU9tGX9Zlt4J+6><1;ngI&fVQi(-U_ z0A1e(a;;j7xEM)n9Ju*NtcY(j31-XXl^8i#HoO$3QYT`o%A-FB9a2ntTrYFV`5O6< zypaW0oD`qk0R2Tcp5mYtUtCpHW0okm!BA{D{wj3=EAXtTFb#hzC<7Ik9Z4xfYJ7m~ z%!Ez;L@_lv3UD11OdnXRze!VtE0`4rNErBp6Jxv~v1`ReBp#Y4fWgoncHv?T z0!^%2Gc3i{paD~)8RXc22oy;eM(4=o0cg_*D zK{$SVo|PX3(wO_#Fn!GOkTl7WK~C_e8`EUJXUt}df|6!z^F^I+5N+Y+b9rmhD6+ol z&-JaYb`Kv}eVh+mC|G^a1sd%H(^C+2tW^ZLwk^#UtIHaM3}ze;JUi_?RwOy_n>-oZ z0Rg$WjgY^LNR%T?>}&&}iv7-Qn=LCIq+d8WKM-jpDhDqwi^?;CSJI$7k<>_z2=);z9UYA`EdV?WRzUfir*EJY}V@3%qp9^_-oP4XSV;G=rnKC;6Ef`f{ z$1nhkjl)7~u?H9AZ|C-scHbLi<22f;;dEclN8p={wo zIsl(zxWtMqlVGPgU?vK4NCq09476V)@>~~`Oi-e51nN401@yia@h~UnQ0TKm6ByN; z8vvI-E3I5XjK^BbDmoy&6hh~9b3q#+XhcI zQaO9?vFP-Klfa9;Of~7r(f!qcm(y36@-JisgV!romsYCN>Kd$X2!cv^KeU#8t$us* z`dhVg`7YRhDz&csYjmT+nQCTFvQMxu7jZ4yb1KH1%`Z9AViG)rkzK@2dId~rDJU5* z(hs<}GlyiD!#k%JSnea90NH?|3%fO5DM~Zpiy_O0Y8LU;cGN|He148XfSoce8fxGk zmHe(}NkFh%t!-tw!$$Kv-Z%P@@JXWi-dB%Bf#TaR78u^Ej6>!V80KO3(qt`J2zqPjbj)lw=AbnTuz@p2u~ zXy7Si(qtNw?q0Nwq;)C(xgplWMaA%^e8(Wugk5sR(d-KcmmScX)~>Ojdupz8vptrd zejyVTo+~YeA_fiK#efg^V~$bCcUaR|Ou8q)yvO=pfx9c$WGLGf7aw^yID3OqLuX4L zAFYFMy3nAJIdT?&0PAqO`Hc7By#n@qk3gis%ov9-Qs(qi$_Y72K2B!FUJ2m&( z%oHnsmBB=M9!H;c18y}yMH_Ezgtn(oX}_rhs?FrJDCR_v+9B^> z#T|2aFYs2zjUIR~Wc;e~R(Wy7$FZq6`j%@1+d2qh8{~wOSH^v+AzfuE{}{1wikG!I zb7Ay~O><#zEZVdvX|uYAEl<>Vb*ducC5q)X+V{cY)B>_YBq3eNAKd@myo>@Ficxzm z^u#sHmIAKuQTe7nEPRfWYwwi_dZy7h=2}0)Ja#EZv)EAFg=>11|G}QTRTzC(>ZQ%v9L)7JjTJtP(0xoJn_siuh1W$bUfgnQ{LtUVVXk<`cesoNzlYai< zm+#}|^?HA|uh{JBpxRPi5h!p3Z_1HT11*!@2)in=>PtMhMVx}p9w*f>!uaiZTNy|l z%xeVMwE67Z-x>qR2fH5qP5=Bv)*qXAen%k+#;a15zhQJrvFDV;op4GQh{0dBur!8S zW{8L7V+)z_eft{4*ZY#V;AkY2uiI6VJ@Z8f50$PzQ^K{&l(2&G6btgFW%P9W@OpcB zIN5w;ew>_MUe60A*BM8RkS;DaBbJ!6VFJwz}U%zw>By5R8 zJ7mU2rpUBp(nj5Qe>V2&uE=jFceaX^>HR?Z@Xt`Gm|Og1Eb}`b1OZPU4>i-&N+QFL zfx&6b6GLG=h}V=*HkV)BvARBE8V;W@lQxQyG{sDx^aq)R1yINz6)W6IC(9*XDKs@8 zq2fjC&dJ?BzJX8-u?=pWB*G}0KZ5&pDAWxS2x*FcC6u{VKbVw%2+_%DRgc{R9EMkk z=nc`ig%(cNuZ|r-Y3GMa&9Z{q%z^y@zJ$_-8`w(=&en?wz?H0Ket=&vrRWQ=tN4Cc zn4d6<9i(iih%sQ*SjFCI8Ir?VQT5wnWsCA%$j-1lqtM0u%Lz%sSfdV{MnmMlbfz-w z9VmV+?7UPvf0=%m&(|EgX_Agi>QhKBzvPpph;W*e(j(a4bxYEzjyNJCs5d+8?ZxEfgZwuL-$^+c>7)wme zP5&{m?4A^emGQ+tF_bQDGw5F^G3w8b@#(UM4;awDzz0M#K#_#4$5EL>gS;z;UiZS! z0A#b)HfYB~W;}oCVngq&3b~!UQ@PZJZQ*oJF`!YPGeD+#oskw8Jb=p3eT#f)9u-*Fa$Dq50ya(UM_g69$?suYS-KO!oB`^HqPD75 z=DzhWBu)U|)7g7qV7{A~!G`eIAO1au@NXL0?e@w{7{|}`cN;E_p9X?K+`Z4{Ne~c_ zck~K}%=g@0e=Fm!u*>-f7ybxeMUHJdjJs79>=PxX{z04jyoEl7O%f|GT*gi$>bP?G zZQZ>7hS)}Pq^8j~w=-hXG!H&&z{I^-k)H;7={(zE{7vL!M@?F7bg`eAH4R6_R-2O~ z9TdfBED%OCY9ey5FUx`xci=Q(13Jt(TR=b%7-`*J`K4C%BoXc3u3yx1W}Yu1@y6lk zl9cf$Cau5;O6yB60t&T4n-#aDwf-WKCRe4(m+=b;SJ)o4F2_y7n7o>M3G%xD^R!Jl zBKfZw@@MDqIPaQw)l)7WXEimL!Per3(1!4;$yRg9u%Bb9XFGr{_UMj(!KIhN2B6ml z09X#(`N_8%Do<|1J1l9D4Cr+Oky!@rP~luUTo4aQriU z5CFct3uTMngpZ2Z_$#W|Gw-0^Qpo0K^veNKQaS~_JvanI6{O-XzOv7}Oki9a~ zOG%?cpJC*2u%=bNxoCT3$vfkcZpU0;4;CakcWd;o=A5W23EVR^spnX>cBfSHSICU! z12;F7g9T_AJTP@uO3t0B=v6ILuo~iv+Y)TmEK=xg7Ad9p6eaz6$4JC;aklMJwNGGn zzG!v_D1;Gm3ofKJ_F>V3ZQqI1?MMvUB^v5^%RBai#UU8-7E=&vI|}3_ukE3(u#4|v z;4J9AgTRiTnW|c7@+~xK)`K7zH7GaBtvfC{t_3;wqTuZL+8)O`E6{qIMDE}E2$}wM zvt)nGAkX|A{b`uQNqx>IXS)#ja;o5*@YC?`LMQ|c4IVdNC zKG!B-sKr+Si|j<$Y+Q;fp`LpLCqmg(BmgUK~PRKscz2M>`bu$fAp1=W8KR5JLs54g0HilN*VgyJIo90(E?InQCD zFKiVHO;G%xK$9w->p@bm>lSM05T@7D4j@*rlaY_ZNHTm@hxHtejZ?(1qkTk`1P5<; zEoW9a{?#=v7LXzk)6ysT)t0fU{wUZVA)#t(b-0@c*A`f-M949&#Ycod!0}+549x=^ zUbdh}PI4A4h8Z<-;?oalWFZ}j&|a;4ReaXxudi@YBUH$yXr&@mXOrDJW(~3uDzHMe zC78a3itK^)Ti%9cCX?&??6im~UM9Hz?TkVmTZvAbD74;2F`|yCz#SGH!LJ6gW^o;H zct~#m%;9k6BnP%los&IVcJjBmQU40i^Qa&kSF6maihs}`Vs;rCQyezE8$QkP#hs^m z6TT~?)qdc1$;qgLYu`5_)nxSO!g9-{>QD#YF_lQ8DQ3uLEO9S@d&Ik8EMM(4w%P48 zV3EfGV~OK2AjhNoq@=Pf@8s3S)o_FKLMsq#Dk9ZVC#Rk`y-gDkgn2K`)5}c3g<0gk z<09(x?2a>70Y_(>Z-ENm@rK~MYwaQRpT{5&Yt*2{5RwJufwimC)5T>^KaZL{4zE@j z<}GB3B0IV1ST6$!ZtZ-p#W(FZnTp(tq$%uE=)}o0W0>>i89MrqBI9~kiY!HIq9YAK z`<|OufmhaQW&eC82L7u`=kF<-Qp3aolig`oIREonK_yM7f(xi>x5xW6MZ0Dn%d!U}Ur5?{_=jGKi;VGC{%)##4=n$=1kp*B+XQ5@Pn!QpDI-ry>eY2@}PO``p}a^q|Y5%twr<0B(ZCsYrg^D@X4*|s$41-uwOMo zbeGb2Xx-MI;e;ZUo=0K@{{qjytcShYr_RRmdc)1Z`F^_1eg=EFOTGl#0yihxsLtGg z&J$y6@7l|1_;jvGRmESHhV&H?*1-X(7DY%-EyLKq{pPQar-zREPYquL^&T7ak9!Y?fvy}R#jaummlj=yUz=S&H;&=UfNtB zlHg$0*g(ZdFwN&eL2)^y^b}UDxw<{Q{cIiJ`0i4C_7OE*obcF7c`nza9r(&0KiHg3 zvBGr*cJ7V41iDd=1l$|vbaE(Q*tOP7VFGf3H&5bVLUG&FTx~TtN-r}#D1a?!WUVtS zb%xn!8w$#8EJ$p`J+5Indw`7HmGTs_rgyFVPv%&vd>71nGbsSD06Iw5yNj>s_iImt zP(z1%E{@;F;pyvXL0kV@x|!Vp2z?e}7_nC12@n7u=~~~4IOHRv02H&6Z1^dAEz_eK zsLus&HWY=ctlMm)_h-vqE?|unh|6+<4pb(hhGmuc>MncpQ<*YiGjmRpo=x@%G$`U+_oC$U zf0B!vo5^ls79vz-p+B=~LP{@lkSQME1WW^-;d?OXF8#2zkWhYM$S@1XcJ+*jY7#l=G>=qqY^a_$W>V@VRUfS7n?0h^-IPPKF2esBX)vTt4dma zf5?y^^tuop{xmJ|zQu!GD|LBl6=6tf6*ap%HnIcfm+DYT=rI?(p zape!m%r|o0Tre#mEOL(nho`Anw`#Zs8BjY+TQw$VbbDxCo%>YJel^kHy(~5sfLEbD z;{xv?3BD#_a;IsboF3b759NU3-Y@u&`@XAm>=rDMJ1Ts~($_6UJ8oE^^2viP$=h-0 z4-tT6@mY^KV>(Dv)K_}6b^P7Ht`svxIz-yku)DpB*4 zCWV!h7-(YF0dA`UZig%|rR#L-9rnXrUuJFu-R-7fM2*h$p>%8`L*`m-tp>Kv&sfMm zNi8pTCu%l~2}ztD?S(Gxbw3PywZE0sHh8WZixUW4L>MkBXU0KXuDzJRW`YsxWx40{ zWte$XnMV_UM{mcD05zw~7?-|xs#GTm-sh6nPzxDKmZ~|Du2v9N{(%O|SCBJf@YvNg zBOW4tfye=>qRITp%uM_$EWMLU2Q-rpZm6QeyzEOh-h-{65AJqQaFR~npj~5G{y6nb- zt|5`XDN+WMKf2zHCZRpsB$oWqTjx4?ehpHyyOFuK7F^lZ9vj>?c202H>vXZFvgKB| zlzVK?DM9P_0Bue6Bb?@Lgl2;}IuQKs;TyqDN&B#fp*EGO-ZE~^r*(OQK6|%tN;|gR zi_A#umc)&+w)g{F*AVuhwHtj<@w*`ah5_Ght88Hd_RDmkk;r6qNz>V(%-97XiAGjo zC7ux_d&O8XIaHdeD!o=hJa~OrZWV>F5^I6^ZLT^Z$6&_Korj4o9dJl$_4#UyVw00r z0Ta-HmhYTC)Jt#mV4%732X)q@aKfomDyOi>E|-+(dq_z_ns7oj)|F-zcSWbI5#}3{ zu78E91_*Ttn?hNZ!t0R|f_29($~768zc$ii2(6##`SJ1>EG@l3ONKP|A+bp$21>e;X?fd$ z=7g9l?`FclczLl&YI1~&XL{k)mFjE|U5B@GRYjZ4_=HtPi4M43(dN{fIQbeBw&Ms= zGB1Lpz&NLtF|27{&al=;NI$^L zc+=2JNNrU#6bj)>@%eel2I*B>wdPhOR7!0(0qEbB3h`TlzQt!O!_@%=Y}8y@&Ua0H z-mshjlWxv|XQfH{Jv*zZ1AMQ^!iQVe+{loM6qF#3zl{t7Nfvi^u9Di-=KxTe>e2bl zR8!0pU;&k6f7;4LzEuNDVmdKpRosd-?h2+OC#z1RckBuyHKfk^5su`RSVlWZTt56! zQk9Ux22vn1B850tTAfzyEQ+vBz>?E`g0uNc)p4qyuG^Tz2Ou{5KL)xONBbN-`%m!ub7y zsDXB3J8LF;M>F)j966~;01gt%gmL6&a8GKN9qX)EoYQvj6AW(08gCFfLB+O_uq}pmFV!62^x-4J#&`Fuk zr|-75(Ta|COcCRXV_?!TuALi5+96`>=i&K~@_s;ztt_A;=0H9Z+3q#j+VrZjTZWwx zKxuCi^`UK>MQS)0Tk`%&*+CDrkemKUZ#>}C)r6Z!sYV4&Lu8g!?9iB`$~rf>tR#{6 zmFmj~Om4OdHV?eJ^_N1L5P>39zfO9Hbq4{|5Uof=&*>7wDUqR18sDDFXu?RXuUq5L zsd;Mpgb-foTgUt^@2VnPD>{7{7CG}zU>2Kac`7pSAtmBB}Ed?)A1`Zfte z_p=c6lNJpGJ7g7nv5tEGH7CGt4Wt7hFNPd3uS?Kgl_;}58Cc%Z<(?_eQx4ZIHPV?;uJc*AZ_ZIGW6qj$-8ey5V!l%Uc;4)Ei zX2+<{F3Q}<>-qq+7*wWlk!|%ZY3(%3VSEEoE=GRY>0ou9KJc*N&p}C^epJ1iGC2S- z?#p6ty{Cj_kbTq?)BPIV>gW$a0^p2fBZiDu@2d6uc8~?B5)p}(o{Upn6r=M#j~+$* zH&_T+g;2!wRWiZc1U7?_A!0Fk&GqIG#lsV3J!nDXUc7cQ6wk=M0}Hq$$Xj3|_2RgA znniz@mN+0SFQ}lif7Q(a*X_RbIDeoTK=qUc8(Q$Rjp=&C$l`L_$f8W71luztyRwI? zIk+t>4nD^EA4sb6IfB~z8rXJ6Wt*MpCBxO@WU`1{vVpg#Wd^K}&ieIw4kj+{Lw17p zkK6v6)MkhHh~96P>0*X}Xa%yOx@Q$w8#J^$-yxM6;5nJ8UppbFi(YU9jqQ>ll$3J& zG+!NlyCFR7t&`dtem`Q|;yBbwIK~Rui>AX{Kq${JNsBL8x(y6b`uKNG-5c=fs>%M& zcc>-Pa5*51K(fS4nwWWf0p<06;!weV{10kdcNOI@51JlMfPn!ymd3RZyB*0{~vc}lS*xUA8v=p zk%KTeYL8=*;X+NY?3Lymsq3$oWU)N4D3a}T8$;OYsrg4%koiF(17Ly9+~$?>P4RQ!D068 z8CjEx`3xybo1~Pil1zgXdIjUt+x^R0ChQ+M8p)&PI4uGHG=YP_glT2m?v1l0h^FNP zsXq@J<99k6(_aLr>{<4i3E*J)kHP12eEoW$`WqIZP4^Tdk3Raau&i7=rahzKX`ZL) zy6||BppPT-le)%C_AdMv`20>8ph2|CwrZCiG@&<@d2S80-d?5k7ox$)iBjd|9_#~o zvZB#D`-HkMygXM?N^|Tlf@Oc~?OlYqUpr}8{8V`3ypDU}DUa&GC0`Ai#rJ!G{T|nP z^XhDY;P;)s&L&XMtg7&JteuN*P22HTFV|l`nux9Ni6~f3W>hy7RS*3D{!7|PTZqZ5 zpV%q}{Ij8Y6Z}7osHS%RafvzoA6};xO6x zBOM0hr&1*b@nGp9ScubOY*~*#FMqtAfP{b!O^WExO3F^pSI_O>)~YNTqK|FtG~%%N zSiWipKPH^9!DE2k$ao8y?=%a25Sf|LuS^DbzfAPGu5!X#qEWcr@)*M)GS|z@K6m%( zqmj@aNR<-|BhU;8S+gbt7#vR{LJXA{>7mdAjwX<=)G|A^$&7(8uDFa5WnsWs!M}mz z7hSnVVnBJ=&F267ug&QgtsHx}sZ3C+I4}2EKL;+fOdP;^yiSFn*{tC$aSX#qE;X9o zfA|oY0OpN0%4o#Ap`RTYab;5jtHKPow5X!vxB&oN-wJtB(nuF0o_p`U*YEw~;3Bby-}Cdu#9^Huw=t^iYI!(+weLzF=I)7_8&jug_cKT@ z46+<@GrUVw<_%CO5h_If0}DzKh#^Kvz>AQFGX?KRsVR6CBukW?NDU#3Rs=^%Hgw7V zG)S62*s{IBo=T`A#;j?*Jsa96kAj{wh;3{{O;xYBf3u!lnmKH2{wV1#Iuz{wdf9Il zgHmL=dDDX*9crf_mS7?<36Z50wLpRFnBpc_Jo6F(;EFsM9Wi^M7d8%Duu>PWsfj)! zJPqc{gCaUyxf_;(5~!3QbGqIWxbMpJ=V&IFbFd~nXI{c^-D1g(rPL9nF+xShTdC3J zTW1T@sdH-G(7M0}?{t=L4;wS`i}T1VclWYn#(vs~(4|u_QG(yx*LC?8b?E^vzwGI< zx5IbdmO*Eoy<@Leeq~m`OUGJ+?!?Z>Bwm>lozaE#EB8qz%oP6Ar@@yWrQs;xOZcrX z%5?Om;d)_@(ucPH5(&wT!aUp!)fj(se<7ohzL^W5o0IE z7oheUzOpqbDi2(w_>k3t7EG~DBf~@t#!arJ8Zz$Ux(WjBVStZQFBC9&S>Vhoma6UHt=k@BY5Ej(KKP zii@@y6HWsGX0T`=8)7hvbAj-N;n{b1j8iEj>GDQlfCRTP5>jA0j}XJ&+6h-)y2B4C zRxG)u*Loaixe=G3U@lR0+{5LT?63mls2LaYw6#+RfM}_rp_cpUy8!4WckT%gg-Hwq zEr3iWY(7EBcW27w7W3T~coG#AWEO8BO|iMKDX9Ba2%Xwwt^V$@&1q2H5_AHB!sio8 z@wX~|Wgr1DHd1XS#_5>xu2{8A#zb@;@^8arW?gfIo}*M*WxR8!@&HqNN3bbMFp7nS zjvHO0F|B3cJV~>KC$GiB9~foJ>-&`YxDSS@cSUrI}I{$55$Sy)zLWPkd86W?!AKFPpxn z;X+@c+iEKGO1p;bsErVVm}I;-$>;Jd_@=NM6z$qgtO1vC|H5K2%>yoz5Pn zD|D3C8CvMgf}rwzUl;-(r3JmZDy9xPO&m!(qAOVUqy9g{fJ zP)b&d09pyn_#)m?F(s`l3d?ml(fqdJ!JV|y*opQjzSFi&yHP_gbx>Pj%wl;X%e5kK z&l0mnPoLquW%z4#8y7SKkLtVc>T$8RlspCQ+3kMylN?9ka+68nY~24{)D%`XKC{x} zY5zg&P`X*XQTd#*GzZ@yEic3A>L~gyC5(0v=H#B|>@uH_gE{>U`|L7&Wz=}2`&fh- zOGiby)$}<*xhK3IDCJ-ifKik>|537tx7g-Cz`N~H=C=@-Vs1BPdwCF6G=x;v%-U>g zBKCygxYAWX?qA+^U#y32lLMNaR0>J=Y_DuyIoSGPC8he?AWAJT^2czt%DsL?@!s}Y zMC)Ev=X#HOFX-7c3tUr7OvqMUO^@I<58X3Cj>^<0S4Sc&o72*v>2DAMGnQrm$hCGG zMfyfGX5nWl)_(0*%E0BczJowqGQ5|KB|o$+oNTnXd<%=q#x+-K!N!2*z2`t~6nc0? z2E&pNQW~6f$U;;Ric;lZZu7b_(I%34i_n|ibBbzO32G(Y5OG(50Slh~Ar}`v|Hbpj zVUD8G6r=>J0cUfl9j+ceZm4+JpGv)K`{&SyvB~bLw?VlAz22MLJohkRc9=mD$z-KC z1iMPja`R1Qvvp}^`Y2|1xrl}@JpStB7*~S@S8I>HQbA^$Dh65 zQ}W+W4_~hbH|c?(N}A_zvS{M^@GefA*K}Ub0X?2wCAy=eZ1FW_ePt#sHrB?%=+Rgf zBA*?@r=X-Y*KM|{VVhEYQ}OKB2o^UtdGiU6E&S-wU_!(e)h*q?rflJDyZ#=CzSK|3 z#0g5(69|pbl`;0~nN+oc82c2g))yilc9SC2T%9D2=06nfv?Y03$zFP+rpUt4b+?tgZq(3*sbfTin)9m9jUgGWlbezA_gr5Zm zw)n4*nn2GK)V>@`_`UpAmp~1P8-Ma!deoX+d*fRC&$)F!z$u}JTfXTFdN?arle9_` zO*jo$Yi<$TtuU_$(FJdys_s3yC1%1NQ`Woi_t#2#h(1UC<=e!8;p-CTI=#1e~(%K6-#yG9ZC7rce3+6p4a{`DAonvg1zjVi9D(ATyq%(pfa_ z&)&ygB19=lmj~)JfaV$Y9j11vy_`Hg&`E7+?Y4(w$CG7pC=2(tVz^haeDHAx;I^Qy zMKt%^t!(^WAjVAlXM_>UbkXq%_HcXE^t+-<-jtPsUV2@q^U$1 z=GnJug9a%Z$vxES-+FNy5e=j^=eCsGAU>p;L1Up*>C7cV$t_M#0eUx_)C#fV@uRnEsn^4RE5^OuHx zOP!and>q{YgPa~Js!f;kW#iWl0I(|9rB9%sF=?pj?fPY9{p^@%^TE@ggx)5>NRpdytNuz2xc zHS$OrghsZpOUzCfhc$a9n-#^}a0@e;fyY(jnoxKJp6c~;uQIwdrbQ&P-=F($9Z{a6 zPwx7haAw0Lf+$3in{*9~GRdmbF^jkLbHv(f5uNR0|xb_uhYqzztc6mRXX{q1;g zH{|L;5j|`Gzak@H(d=|KLX!l=kc9+L=cr0l3*FsOL~3sLs+?@`f`()egM5s-j29do zas%RtC9xz}2uKIfeePkKqbD2I!E2ZW)$+ zO8k9X3+%WhWLT1QePXToh{(9$*ia&z$e$++;j7T z$R;e&u`en^`F&s+zL};HW@e8c+rQS`8?llUKb}4TFQ7+%p6n&~aa3 zksTj!vZtpEBa4&;CZPd^$25ARbt*X!L{jDlbs58byu=W;zLR zv{c0cLJ`QT=_~Cx{8rl>|9sS7iU?tyYl=zx(+D=u`=64FZfqe{;ouXc1H3uZeL`c9 ztQwI<0RpO#1Fr9!Z$$nda1mbkA{d+ag~mh>kyjPgsj$J&Exm zHS(__XO5y-v<$y?A^c%}`amyfHsBW&_IU*4YvSf)#t8_a2osz#b~p;>fmMa^V4x$Il$uAJ?q4Q6OrTGVLT zmEa_8n~CUr!;H>&SjA2~ComufPAQSE9J0+T%yKaY3_Ker4)R7CWlTYUYVf1<;e*54 zj(1P6?)Kl()nC~GpVeGEZ|6v@dCN6NVfQcP=2W9gU0~X%XslMR?zm0VVtnmZY>&pr z)xrj&M0wd$vIVpmKh3te*g+=mNriS2RCcSHW3ew&S)I?@T@$dIoMJrGnJkuVH_wht zp1`HU{+$G=jpvhg53<@RFgZ^S)fjBPAn?6Aw+#|PX+@^$Nd4*nU7bv1JxJ9o^_P0#|t@;KkMXJj*1$6Ee z5Qe<-?LxG;LiL9xRfxr;dwFa{?S*P@`>}+S2)btPqN0Y)r4Fig5^?C!->ZTWCqQdCkpf^zKx41dAzY4m)ZH3#4_r@wz$KvA%OYYCf zPzUT)1z9&?S-bKRZ9_Q(dm$E9d<32J&{V@6|AOQWO>HnaX8I8{LRBIkCJd?=VVDSA z+@)NwVv#%mS?4xd(l(o%8qtg|osaXlNT@~OQ*fgvVV=;7T3wlVy)GjD@y+*po^;6W z{T}R;u8WfV5Oxm~-zl!7e*@6p*P6yK^?lFY6wxz274A#TB&M{q_yX4-tOQEs&pBWY zG1@i}&>n*q`7om_DNv`ck3S+zAr}7=7@2mOnMs@f#3~9jwb}xf`h~HZZ3!u>PN*E# zUhI+o%P>)s2yQNtm@KEpfpf9th6}ea$nOpA%7Jvah?tkTCm|yg8knpP_1jg|ZO_B&8vl#^a3W~~@cdm>819_+o~`^gfi%!4A>B`Z zamOkF+#+M7zO%f%6~%xxeB#Xaa^sEMtuje|J0!L>V^x%0L%V1vA*s~zj|tF=75kfO z*KGgrJ&Ks%9q#wlPJ^luG1z{sw`~A^Cd>Bt8K7;TS{1>AF7J~xq__OS&L-2jTqnQI zOhP3=q`V15vQm+E2heeN@1V28lc{C}UsQz-IyKH3d2L|9ZEL9>Q|id(Lpv~Tqpk2< zU{9~I=@ynMDTERE;t_LuwT2T16&)SxWL{f5XdHjw1zBSLmn~9^a<1a6tT~w-YTYQZ zB}F?Cq)5U&AvJa(tT1Z=Md%$Qk4d#`VAn2@1NdW2x6@^Ob|Gf5wln*jV7G5Tugsyi zSoFO?wR=YP)oaVQm$7?vzAPAX4eAIS95qX(oZrf4|s%jX(B-poSVR8l2EO7m_lbSR=w6e0uz2MMEw z_1&IrX8`6te-KMf`dTD4s)6~%#aEYEKz`zGhvwL>VOyb@?4b;j8RIONK+UrIs{80{ zZ)U8aB~r`1g>aJAWEb@rn4fTV+lK_PdUUX(HfISM5DtV|IIUkmj1@zv)R{~WL!pIq zASik==h?DgzL_&BLOlVqeWzn@IfkO6erY>Rb|A=jK|V;7ec1zmCg3~jwya8rAxVt> zR>=mh?`?N@czE0}#0|)zVRt_8$X;&zr;kC&Uu}Za0DVxt7-OXQJY{+yHv5$!O+Z@a zQ!t9B9B?GDAx#mK9a6%D4+PH$beatkxYO|t220>reDKBBqDsp5dTu=Tuth;XAu;0xY}i?_#eNG2cY8_qw+J5$nAwfs z!1R3VkG9D8KLiWHh)U?ktOY8>26#?^2dFuo&h}tFpixVj3JnmqmM8$+CZa_p=tV@v z1pnH@J*`hjQG3BaUX^QE=0=W!NP~|4wg)cNNmRn@z3E)oG%%3%O zhPt3L-G}cK8Eo%}SCMkvq_B&{vLj25$Ff1`epSQT0ss%Ev<+FUmVF|W_$mcwABZg4 zulW6Z#x9@Uqpw?x9>6gHPy`IJc7tP|Ass0kHGv$ugu+X>0jP{o0i}#FB&g;rxuQE% zkVRG`06V(=d9>NVRCgvobX_i`a!vW;C?Z=_5RHv4g>NhO%mrS8PEIyKm%Gi7y>h^R z(gv>QUt8RWcT^Kd{Q$nGM*j`_@88h(Cuky)#GPmZpUB#v=HVdgq;Tf^pB6e9>ss3$ zC!Q41Y@UsB7BqB&b7gXUb}g>(g8Z;ZSkHZ`XVc)5=Z8h32yGm-i-u zax9&$Ae5LuW;M?WvW!0WVt+7pN~p0>vm7oM%ob&CHXQ+x+c!YN_HqM98_U;3&7gWJMdp_&!U-|mqSZp4%_Lql`%OdMi$#Qos1d7SZnDpHH zJZJY4#ZqmJRa&prissYb=Tcz?O^ARc_#+3#CVW5|e;4zyV7(!0z|Z}s2cmr& zTLw{MO#nVF?e9LIm`eYC*L(k24in#>mS>RDE*D6fA#~O94-+CVzNk0*L!+V?@{A?G z_4UAC%bv4)0-6w5> z9bw0eZJC5kK7zQFLkfi8x078`3o##oOfF?4qg-9Ot>Vv-Kxzme>Nh$y=%|~|>-dI^ z>`5!xA;px&<}&ypNn{?nzCr1E`Nn8EYi5}pERDgng5ay`!3qQX1Jm2TLPdg!Yd8 z{&PH;lxE6BG#X^`R&-dF3?3JRXDvmdC~ZC*iF7gI@C`vp36nT<|9uyb21cC#i4^5J z&t<`iVjOu!1!OE#i1H5EY5Q~fxaNh|`m;hZ3rW6pqGq&-i-|%w(&v|xCq!SDIeWNx ziYVkw6YyI1RyQxMCU-<*ydy8&QSc$w0kIZK08+i-mPjI`?Z68$n7K{qqJ_DSX5uIp z^&F$V)*Xmp;nt1GArC%+c-C0L3C#>=k^cm34B}o`2ySX`MqnA;nL3z2XpMqODjX0l zJ|LS82a1iwZ>xlZ+TZ}j_z=%4=sD=y$d4B{BKtximliKwbAPVtfF2Al*r*7Hx+RIA zs~rjKJm6Vgo}EQ{i}-s+FGB4>Uh}jX4!yIFzw^C|*@?hqC(K?%UfAgUN#?T3;|3_X zz;^AeSK2hr9?vogr5Q*x^%jsdlzGafJp%m>2EA-h&=b9geFm~)J<-Rwec)tZC?~Wi z4x!Yt7s!6iVC8A;V~9Csi;5e{Zpxc&b-)K|)IKKv*n=XD;?)FLbrzL9P1cB`7_a~V z;W2IE{PA@+Ox*ql&u^xK%L5OeW&9UKD8j`+4y_XOnlHe(@3_p^>{{;%bPu#Y zS}PjUCk=qPg-PI?W$>4qr4O#!EOV;YcbNtfMnJ_Il-CYMj0WLv6LWxL6;rC)^@EQ$4)J*g*DA+u8G@p7KHuSd9|6~886W+z!i@i3+>%_%=P+$2T?Zj;Lhd<38U#N? zOg}ZkXryBTM`Sr**9z$-<0^$V3Uk#d6?`ymFKDubm;T)m81DXJt4hw>SFU_kqT2RNK1R-pr-@SlK&nOCriYw zi};&M6Pxoaq<>30JO|^G@eUXXF7ni7Yc3<`lWdSvEFOpb{(ys3J`pF+;<3o(SU|); zBh2uDO$OMV+>0*T#BK5g0!jsBl(VB)aKLQzG#s&&wV^sk<1X{B*Mj3i#uBWZlt~3y zP-Eea#}Di<*BPb$YGUF%TkmlNll7c|IRVgt0}cWXLPW+bzrw=kOS28Y2CbGgP*Ze) zXP|ZW`9zb9wMG`Nk^IrdmB|LKd@bED)m7O3*=PZ8he@X;-xI&AgJ%MnT93hYuz;oC zRUA3MybX2SuwB-`;r`eN`}R(9W%Y7}-p%*=teUDY8KSFhN-d}vL0e1G_L)E-tzF^1 z`a}gOtWHntC+<@(VrL-PeEb-&*c#_1YJ1Pj*8Um97^g z2dh`^>OFK#F`2-*$P`ae3a!?%hQCa7$Crw%q|ZW0D)w9a!ji7D($5}_lbLWfOFob`Km68dPc1&yb3aRoH%GF zZ!jaEH7VX(gt!z?37c*%kQIb|NKl|S)zt$2`tFlE)rY{IQ5}EQ>EBDXy?1%}Z{c@s z@1`=YN=#fHK(TVWeu(T|2u+K0d>uu8t`$v4!{(Tg8i{@Mg3eP_=j*iZN|5#$1zwl+KE z^QNR2D~FDmUXbeIrkER-$}y8b#~ zMq(mic`Ln$Xo23!^}*2DK>xOUt*W#;3JspG152{>Tl*}&Mr@@YEjlk}=V~nrisB<^Z$nvxF61rUil?D2*RNOm zo}?=VQ)QUo164)aw0YWP!T#Q_;Qvvv&8zybPFi504E+l+^7;UBK`951HKRR6snX$g z^)18+!hwaSU#HnkjkoVazeTTppqHw5)pCZ~#lU+^f`(HnfwD=Xy_qxASPEZ5uYyR?MeJ}d{ksa zj!~akX0MexT0nB&C+vD_wuR;p{xuq?@pv|7Vr0z26tQ z2~&~I34wNVfPm3_fcz`m0fCQ2YDooC|9nhHs0!30ABIXP>b?BH}7C&Sx%D8pl4K-Vn1h3(xWHf zXhyKIs$%QBSBLY%;E8n!@ZF?sh?1FcaMT!)A3Ao$OV~5o;LkUA`5=W7s{miZa9ZdC zf3?FSW2sm?bs2+c@OrpPM6K03GdD>%uATNb3x%q5Y)_~VKLn7mY#4_|V6ft-$Cop? zZO@i=7JbwPOC(AtI%6WErY$lRzN~VK%jU_!b`!-^|NBc8jL&XClhui+^$rz{dC=2Z zl+A@`5M5=q`@jNv3yi-3H;ChoPIwu>`Uat$su{ag64QW}WnW8jIh1sJFIGL_Dal}7 zGZpa`9(JU%z7})U_mJcEd71Qg`BfW#J6GPu+dbUpp@jn(JSL8%{YVi$`NMSm3mE*~ z1OO4C91DbI#CR+H(@Xd3cNCr6U)Wt!2tBID55}b_)~H7HkOpea zUlj`ahoHZ+V+aaHcR*RDd5e3#d+mdBvc3?ejSiv(P!NQd?nLjbh&0XAeCY-e@>3LA z`&zU7D*@u!Bs{rfXf_AyVv`;ANg?<`q3l4saAg*LBfn&hw)-%m_fsg9{Bv& zITrPLD`-j;gKYn@8v{n)XGBbD+(SY4@t0sq8bTdd%~mj9<}Objd_c=f4H%#>ir552 z5yaz;4}3>JMFny%{!k_9PP$V~Z$Q@}$37&Z5R2R!c#bFBiGM@EU`V>aQ29n8q3WWq zweXs-a`#a7T~YXiPeP9I8EwymU_HCo-5g9)8+S}`JiYvwz*HytN@6&P2V2hf^!ees zEfeTwg{J!wj*h4nfo}UWwPX|DlQiCS6I}}6_@gZ`8tEsU-T->e)WjqeV-tlL*;m`v zRIINfx7kF*x?q|(xW?%9h~2AMH2Y_WlV3w}qZU4=Mlt&A;bWJtbwW1ytFLnE?gvc? zL3cMzbUK0FteB^)_{TJdKYqM1>(d|U8!gsZ%9?-#k&(M*lFWBt1Quenl0n%?2?qP+ zn}&Y7I7pyd+n|vQWmReYGIqSZx(*ed((gk>k%@pMyo(jQ zEM;L@3A`VJ)j+FY5a+{KCZ~TS(dc;Rv7Q6)d8nuhi2eyjMi#!O^gDSYV3+McdBCRu zZP$fXCh)Mz_05HMZaZZZiB0!?7Rtm;)LuVK`ccW)?|`i=r$DK0Lr&nVU;fCE2wAB*DFVT+lH8$AU zYBls=CY)!Y5ICV4?7x6Bs+GpK4wbUoUfpcdcS93+k z*ni2fj4b&JQ&Sz|>Bsr9>SQB3wNU+BYp~+}U@NAtvDIWo>%W(K z6+XemRw@a8+Cd{`*I6<=l+6txulC|;^j^I$6dt*l#jB&1V&E22b6NX{#nwU-pj#3@ zO-FStf4mwGeV27=vSmuSplzW+Pg<;6kb!O~&`xltH`*Fx6pk zJzSNJRDB}z$D)Jd%?^)TiM_rxj!fcntEi__lg|iAC=$|qv8TRj0J6b0Re@Ez>gKprumvy5(-%ky=>s2TO9r=2BCX@!e)@9E2*jG13->G;Mst zmo8Vay8Y+t7S=Mg?T2Q?ro|3^$33sc654uYdGQ8|R+ZA`wv6lL>#&8vcI`};SpNt4 z$`on&Tz!4~DOw!~rAYINEJ=5V9Q2j#7qr>VCYot8#GBycW+7xLUQmq_{_4{k6Awe6X6mlhjV)Fh3)w z|1d{|An;9zFAvY1D&L(IfH^e?Zm6Xs>2Bg9`5T|8@7q%Hn-#`_{FG)Su%g=pyM-!( ze!>b|sU>8m2F>g9cllO#zhBcw1J!f=s-@$4+C`~|W@lQQU5cD_0>TPsyALP;)wS%( z`U8yQSgGFYZO^r;iv`f63};Cj_fFoCiX@3Uk)qZ$K2p--F?qj-xZV;4tXSWWadJ&R z1GoBAi6n6AZVU(X;=0QO?Ax3l(GV%`0Kl-x*7-_L>5KTLS=~|f+HS|v`H{~%YZAd5 zH??Bps~eqZBxkH|IY?Rm%*#95YNKUeTfB3hXXb@>7YvF$PiWl`y#;_>7CkS$HP?n` zQ4xeK4d-ohS^nOgfcU4?2W9##s&`5k^CqcV_obN1!xZ?8%r`X-g*|dC_rM?Qhb@(D z`C`LIgE;IBe6*YkPxBTdG}5+F`^E!DSX76f%lEw=rg=A0(D3md=6F{h;C{JEVWtV{ zSk`^RTc4NJB`DmMZ<&S+HMf|H5(Vq{1)IFc>7VVq)A-t!c+_{c_O4Weq`$qVMH>X# z*jWI}d-0$pHbFd!=xiO?JR9a6J7}1v*C|3BLH>C4+LqE}$6T`#Oi z#_1o|jlIkPS6f`GX4>=dtMP=~GTHIFf8_V=q^y+sqvoB1G)9C(nSa^vJXb=fz} zbC!HqKC{71n)HO_KpoGV>da~Lr1j}TupPZwtj+G5QNi2d#qH1()6b^QpCjQ`VS5=y zD+WHjQUx*3BXar^eEd29jxrqnMj8+zJeAs@bT~~Q`GO}X*brV zr-^z4ioLLIg}+aF!#t#%$+IES(VqNM0eVyHOnL2w1x}wu<7+BZ!c8KVfe(r{zfP)l zn*!GEd$~~$g0->9dDZ`zh&4=!q>CwHS)jh#vUTT{3o`#bFPL~?8>cTrDV+!zFaC}x*y9XU%y^t{ z)@NAW4+rN}Fd4tK4;A4b=J*oK^KM(*9?e656nStV1ExX#yG=6chRaNIaGZZKxn%{P z+WWW>=(OCe^lAZ6WZY96lI>XckBK7asGx(v^lHszK7L-Fz@oQ&tC(^L7FWj=+^%!5 zus-$4dyZj$58ZgNP_j9?D9UBzY38Pfe-xt?w#lSxt07PQ=qb z`$}i8T2$T^+U5L1TY)MRRf74*LI%DxHK{2aY-wHOKw=(&+#2qt?0p;ftxt;` z-W}NkkRZE8SJNuTr%v6TW08);2M4<5=<0XCI_8GpJ)cgy{FlofoYGl8Sat!R{w6kYabacM9A5ej!z$C< zw<|b!uuJhWE$Ts;7CaGJBDeQ9Zk!)6kQ_v)1xtM^%;BkJXlxA3z+>mH>n zJ>x^1^r|_Zg(nFI^3i>oDkxAkT+@~V1DuDYo~e(y|WE?5FWanrt(S$8^<4paEj{7 zKzLnNPmRh#4jq47dccX#dK@O8R+i!dPX6)-2TU_mX?933IGMi-&5H;Jmg`$-^aOL| zSUFiktHjS2El8BAId_{5CVGv;t3%JWxKyPU{7rnoF^it)TQE>x-GViZTKz?|SfNM* z)!H1x3(DubPcvi{>(eND5RC=Pyl_i6eRf50l;O`4-rEH%SPg_e@bz&icb~q1)y8Qe zgkho^n?m!TY?oop@%7%yKbL%|_J{l$V+= ziuoxQ3$nsgR(H)(wfYhi$2B(qeyY}Ntl#rR?|s|Z&%n-D>sJa^ykq7qB-5XMib#Oe z^Enlxkz*S*jCxI6OGIGJ3g6}!A0FCHIwBYPI5FN0&cFbb$U~3Dm-SK+J{Jm^3>v7R z$!cm)l=gGEf0g}fts$~4MhE=mF$>Kdw3lT|V42)3YahXwYtb`0SNPO5Y$l!TsK{k+ zYfCBl`*J7j?(lT}Rn52iV3v{->ZP{6zGmYQI1&W#MnckVE8>jK*8Im87~?|Q^#rp) zR3FNuchgbiyT7JCf`vHFqb;m&w>#?fTV7F1YX9%w^&t1a;3sceuJ8*wLZoYk8?p)L zcISH-V>?BokJWysNVibN%JVZAkY*kFI_jFgw#XW&)Os0MM42h@Cu!<<*{gUsY+5LU z7^v2LJI%j*u zf?OnR<@Vba1IxV0V#uvYA0Ek2kD_x71mSGy{=itS8#@+rHJm!ZaYb(KsmpJxxZs)i zBKPP19Z#)zt}sHfM};JVQxvk~43F2?20d_NC_NCy6Ooc@(QLKm1xR;g4 zLrhn-5P+?o_Jb%Xgrh@q;c*H}mtPU*=CsMw<>a_;ZJwlB%bOluSmbXK3-$rpeS5@w ztr+@(kJNmARbf1-`+R-Nie0D%Jxg2cu`bC-YMFm#8j0o=RkIWQe&6kZ*RKgzQY@ht zauJr+j<-)e5Xq=gBIBbUgBM@5tkx+6lt@e94VjGc4BY#5;U3QoXZ6n~nkD~A^CL34 zItONRLB$rA(xFu~lO`$imqA)d1(Pi2^#qllXblTT^V|Mg{A)5qlY6>=>K5F5@L(A$ zp~3PMk#Q{N?=F>Gyt7W%@ekB%jkKZ6^po)917~3Epi+lu8zcWrm#sc(5)NjcECb-+ zgB$ER4`6XK(76uJx*n`n6MvSQ3ta<2z&+g^vwLn#$VOEOy=`g|9{WSHu}QBATV%VZ z7M=PqLVaJ?-R%{Ezl+}ic~mS=>;9E#hTeK2xJ|oPdfhvp-oXeKF+#=L6_(6CVdND| zy6cM&fL0w@=cSo$oY&QBhV#Skx^$hjEpshN`>eFS7VnTX$GPP&roea7F^%tMt!)ak z?feIBC#KG%J6j9UYOjfugdS(CHha_+%eU;rnVVLCj$C49wHp>HHJ*d0iU|3-mW(xq z+?^*N2Viq8Iw`^n&Ut~G688A@1*yhm8V5=>d$eM6wIpMu@6G|vZF+K1We_~|Pm5ss z1^lnZW}V8$lo%^qPV(`4 z+hZLB2pCBbg1$Y%%+}`D&Dze>fQ)_^iSY+A@7Aj7W{IhPVu7bl?|AYn*Y(`Y}bw^BpR{zRwl@y0d@j!m=a$kYS@0N59G z$ii%P#^_kXFR&ikDMuPRh<(*Xo7mnTug>imBOw-WpXsLDoke;Qan z{iCiL&0#;=MyM9f-Fgq;Hj;ZBhN(kZYHBokYPi`MZ_0mN9k#Ppu zOf!fH(x=o+xBDVc>xMGXr=v&k{sGjd(QFe~7@EX_Co2Fe*c|avP-y@bsrol2`61nNb z2qtt5ok@&sa8eV$WFCNdJ(k(%LnOgmZTiNW*@vyelUkWaoJ-E3n3zfJ@`Uz#u)+}) z?K)9p;pF0yasn8ko3N|@c!EO^am2WEB@aaMsCc`z%mpSN;PODgJz$lRnMz&9r3aDr zZ>sauzAcq;An_3e6v7Z4pMa&GI%l>{fF%Hf_!-|v4`8r!(V6iJ$bhp;!yyGJhqalu zN(VIh*20%iEbY<`{Rv-87P$Z^S8#w{VZHl(kMjyv5dZFb>td)u__(V9DBvw3~o!v=I zkIeCpGx5oMqG{~Mg`5T%7p3nBv)c|qA4jcp;Di*pL{(|6ZjOS0W;%456_CE zh~`n}O+O$Cb~^u%tk<|!kb1!R%FCe)fJ!EhFB8i6u2*eUT^ zsiKak)%LtS!k;hnX8e>5hHE`t=U}vUT6WOj^;NNZ8Q>Yl0_}FO_BEBMy96oA8N{gf znqVJ+1oF=sf?7D)`-XpWDht}alD~vm0oAWZ7Tb5w7$!9106<$6?z2Z(<4AVj*bdfI>>vYZxA7{OqDlVmTu+PRbwk>~7{ z1E5t|L-(cunnx2+t#3CQI;I7se6q-kmmYt$wc{K}Km-`Jw>(vb$E&}AZ(q=>{ZE~} z!6*cH1*ar7q$rRXaon&7KcVkQY)GC7l_H1`zR4m-SZC3d6<~ELD=hU_MWQf-YM{F8 zmR^)suJ=J!^O%Nmf0BEcDMLd*(INtgI{-=49l`dGyI%=7)mu zL>}i#1^~cTNu%8NkXTyaWoLyRX1Vf)x4L*o98T`>KMQ6#K-k5_`RW0e>S(l7uyRqO zoVIZ5IdshfkNBXHMxFTml!%JktAL5nuFF9m1Q^2!VN3tb4Vq=J!9TVe6$sYYL?~Jq zjBvD)iEeL399>CSdJq)CQ-g z3uFk1jz{FiWDp`4*7n8c2ft@t^{!a3y$T9Ik7AZauzL8rkdp!0coUAnL1(09{8n5$ zMNroCaavY75`~`QD->|B3BxM(rcYKMc#)!C)ds_l9unSXrq?}?g?qKs&Hu49_Prv8 zk<9yuIi(9%2lq!}-GV?Vzk;MHt4kD4k?or)CLV!0XufrPMy-5_oR|$M>oS zuO4*v1F{W`WsYZD_}l)$99c<J3)O#z_oor?ncxtd(itHm+O(#xlN%gY5YHO_2sxomDI4knZw9W=^v4p_I64cC zc)lGNSKx@7Ob4-DFxf|w#VD&~BUwXePh(N=w>F5}m;=eJH)WW$jv5RGf)Y99|Go& z!ssVE0#alwK&edde#VT41l5z51Kvr`IQTWO?cdV-M@RiiS%uYDUn(0WMih~N9z%Ft zou)YaA1-O9y@vV%prO#N+)h(5o48%HgFhncqHfpOQ#cMUSaQjc=PDg&gxi_yl}w`K z%?aiET=?_2_L!! z2~BMjSRW3yp<+%Q4 zsO%<`q!>KwS#bEQ!a64q?7^-$eolWiFg4sC&K1Legnjw=$tRe+d( z)Whha(j3*MVtsgMG{qZq`FZt|ZDP>$D!GHArn{<6`dp)j@jA%_>|W6*7?D9OJ=?J6 zvTF}ItBUuKpG|vW+?$t@8!`hsPyY#uHqL|hFlaBPr&h`5QH8#Af;}g-U=d44z|EWl z+U8g5%?yfot)^_>x~%dlVJAa%bYOz2eFA4T%v^)v85`3ho)jcgls?ce>1R_`H=W&l zIi|8hw%5D6!kaltIdk%^Ac5F1$bQavi)o<0t@#VV5|D|6=}!;iMUKRC@Zpy}HI}L1KUr7;5N{kWT3qkdW?fDFqZkQo2I~ zL=Z%Q?;X$a9EW@FC(b#n^{xefJ-_|zC->g$o!R$hGCh8tBKuMEvxNlcRKR-DQ=)5? zM7dcsbl#eS0y-$oESe!+yJn=YZCj2vH`AwFYyFvrY^1EhWcRtq$gYGa&Xl<0 zU3{IPW^Q4$MI)Tdxw=0msPkCkwwhEt2eE*CIQ^(q!b@I{z{RJ$^*sZ8eCp$sQD_mj ziqXy)ZfH~$B;a{|e6Klv|47ReYbA3b9DkcxZL*tA@;>e)b>ZVzl^%TNwmWTX(?cZ; zP+xhmAMJR7q8u&2ty3sO36X&cJRr!*%HW zApt{oZ#MBr6wSY>ND8c3lFH+)mH3I4IHhTr$NKHd1ft7x8oQ4c#=MOratsZO+Y{>b zb*~Rp-V+wH41>ij*Nhh`pv;h$vd>v?VSF5ZzF2=T-`>{YXgSz1&eY0VfGG3aL9@d# z%{6R}U;Zn9mp z&y?8|4ZbvZTXq-XSYMYPJER#w4K5UgY$JM@REMF z-AEFLESEgrPugxlMzdm7RME}&0C!Nnk$S9#p|RpYVq6+}wOXH*7(UB*XOuV(Z+O_M1$+x5tHjU{NqrL%Q0uLMO*xDC$GKF2=p zQ}txbfzo1hG57He`ex(l z{c8Sm_dLcKmTxFgyfDV;jW=Kw8NPJ+8|Mpt8Vl1n4dZ322Wz?`zPEYrSQgfcGKwfI zzabH@dX}amvles%?J}May^{fVE80qBoK_3on4u|VsIj51JdQHwy{Q&H;jbv*_hcqj zror5TDp|q9W30XIb>zw6be28hSc7VPmwAb`G24PgAD)po-^^7dcZG*&Yab%=cqQP9 z*h4Jp~{|eZ(h8+Q#~POiQlG#HH%%5jIvdCD6zp`!9~(XNj-d) z%=dlY7gym#y)pEgei2_oL*&GJT6gg}!sm@Y_uGCm=CG;n4{EU&(l;$jrdzgCIK2Eg z%B?|L%Fm%`#GLE%s$D7j?&rI?*$u4*-q&vntba`D5*y-7$#8GctPFd%`G}cyW7W;U zA7Vr6v1*8)vmg+WfX_s3!pDN5X1X334Ex&afML(^LfZl+EZ&@7kv~lPiR8uEAbB%A3m&ziOQSl#%4#cu8)5uT0;WDlc}+3WQi;lTSa3ukVQSq zChh%(D8QJJJDTcE-}!#($GrPyX#V%JW<^RIg@7&+nFZ?k0fTM$xB!z!s zFEZ<^FuttETbp#-?NwFxsbd*agGvQ|%!jVoHTz>W;iWn2I@On*rrf<4u3~Owe%;N_;VqhGDag$ptJcB~{n5Rzgc=mAo{oCMj$Iar5kvP*FCs%S6Jn|>{k|S|B+S4-2 zBqOGAH&dEArkl@aPI6_@;&P~;sNZ{Ttu!&Pvz0>}FPABmP+$ zD-Avi&gz6{B9hMTs3AgKHQb1AL|%lq8n~-42w%rWN;?<0-oK1{MU{tEYFR|f8#A7?d>YtRyJ{{ij2oPjmAQ$#{?R~-|(hE8?;%#}91 zuEu-H8pfALrS!W59w-$ruWEMmQ3iT{`Ra9BtAKd+Rg@lSM`o@(-dmCD@w5lWB}2G& z)2Mhn(Ouy&`6Yg#F%P!Tg_PyfNK|i#l`eFRzAn?eXCxJ?+SQVM2Ao(B5nZ`X`K-1< zzR%cLRp?Eot7eiC7DSHY6197L+#;Lad5#jI+IIrwU+b*I4I-{74jcM)3^I;l;M?!4 zMx@P690rMo3d-V*)&(1d5$iug!z$pxXB#HmNnQBplG@2CU?N8%TXQa-S*nSQ_Nthh zw#n#?(g&D^X%LAG&{U*|m%r$`;D4!GKkT{fJxZvJ~tnq(&j) zXc2jyh$5~SmeX-SHako)gEdNXHidI&>SfrG2>lDbQ@fwO< z$VgQJvjXp{wYPlARjj;kcRXQ_65bJB(FkIvVT+*8-^aDT8(1tf&UC->_9St*b1(;` z(KSxUh9@=Ua{;J)dH4i&SpTh&0j;2C=I2rCPy=CU364eRIMGZQog7z+6x=!L8wFRR z>BRAFkHw{QVr8k1e1yKH8C|QgE*KxBD&ZA6ddmI0A0+@!uF$XGdIV*%-gUJ)2aK_pm&cWs>)Cu^>#8?`+0Rm9;&OWGfbOtDw2%`S20~x~-dMX}kPp zTClU|l-1o>yzLh93_mP+XK#lR5i-7PFo`vN#Xh>u-ZMEHcF~zQBd5JPrVa;h%Jo=Z zvUUz9s_8;a`{+|sJV7UIas8T9$K~r}6TU_?gzMt9`qhEQ)V^s=o8+Am8pju|7VP5i zh`z39<4c#!^paxapLUAU+1JARQ=GfAjEC2JO@T>celvkT3 zD<;_)bw+WV7pnq-W^jFS$_hgsrMWCGWTWFJMcueK!r(e`TTpgR4{c+7nc$#iV1$YJ zzJRSvF}isk_H?SEQ^%+6uq(w$e5Mpj56~~JQ$K;y4_saUoYPB3j558z6D12PItP}d zB@K@vD&@LK%Oh^Q(G@KrKp@ln44zGyuSG=XYW)~13bkgBmaN%54KSz_-))-vpC9TZqI z_EE@DIVA0~qRB~>-|mv+6=hlu)idJ1SZv}B$F^%4zDKWQ>9(z}XTpW^^tGMK)7{y- zrI#$pG-QeW-hGhh&rg!xVB@>**UYGpqm!4Ux*h@5*Xx-PsuQ}{C#CV2B8SyCxRZ#-kSRy;g^#K(w=5_ORC zJPk|zb|DB=B`;W{t-2?GMAs~{{3ujx_v1$j4@LC=j>;`Qi5Bm~T{pF@`Am#ZpIVO^ zoezYsQAiBhzd8=S=<7%Khj>lLtvD~QzaYRs3$JUBh-oo_s*fnw!5!$Q;M@RsCZ1pu=bV7VQuY_Z{p6{OXhtoxtR43LgLDGzs*n5ISKVi5E*yPcW5iJ zsmGCt0t?j*_RFo)J9aLWo1%v57Uskf*7})7UTV!K8y8$~-t?+9OOSls$agsQ+B+&Z zekR62d!=D~dEyz5A;-I7$MPqRNw2v?B(&F#NdDf-Sp)WR782svM^J!wqR)arib(9` z=>M_h!wxnzWrILXtT@zFG;c{rsk6hJVBaq`a`IQX1OHCE8||wx4jt#jcJFH*bRQ)l zOyNPlGoC({dPd5G49cCQszI>#p275v5sMb)e(Uw+h`Gb@wNekgk>DKv<)JdY<7W}X z#k+0GIdyZCSWfCFvCEg5{G-+d=m&#?zd%B+ef*xAk5_9vKQ3(mgioDrF z()7+wN+17H?YC0O(>dKCVlwpHXma!plK0WCg7>H7H>r%xID};KP;RQg65Eu=g}a-L z2`i`+%;l8|NHJQTu{EJAn_T8EMdL1dY3>c(2vn#sx**A~yF=7%R#tFK;(~f>-L1Kz z4rhqnKSqTk)#!@Z3YxUA)|Kpug*@RmbDL&xNB`4S(> zx*k)?LN8y|VFSOxH{lIFMuzLgW^adRQ7dItZ|7q@DGR!~7$uMq{PakrG!(WJ%r2-@ z=eyw&%pMdS$@VU4BaP|tYczA4yA`RB0bj+dkL`@5KE>+_M2k#^h~K-IF7YHb&`T_d zaFhtoeOvQ(*1H+RKj}I21 zyA5u&R65Py+}sOBVv-m9m?wqP=<>10mNq6xI!pl*}lw$U5Ypc7g-`so4(xN(V& z-ksD1F3BRh1OmzZ8^)!oAj2l7EUohYSytoiJFeq3`0jhfE%b*?^tqA;1i(9z<65OD zm3)Egy04cM^jQNQF%Hs-p{0S3zNVsV0vABhS?;D?Ip>q$!^7`(A=ZCld$w5oO6~x@ z&=&6JY+-yFVrwHO(g!!ND;r2$iXRJX{>vTc|Pn#-63$TKkgyTXK)H-L?{%5>Ge)saKtX)_0}iqDYB`BXHV;U?+W_p+mt z8qjxqCV`l~%I7Sa2q%_wu`98)*1L2+Xos&rDqnxI^l0-Wz4OwHh(sbO#|=N2bdfTa zx{@_^OP1tLzeyi%qt8td)7r<7_I8O2^W%*H!CDL9er(3_nB0X0m>TO<7MkC9NSyXl zbJTF##||G?uYS75SBUXon^pZ?;=68%&0SFMjg0EJ9S33SD`d{aRg8(OI_b9A9rxHP z@3%0{Yq1-%TDA^@iWcAL1bynM;57stqnDQ$G%c>u+&a2_M*bPg)T@4Cv8#UYtCQ%1>taJnqbIx2^Lb-rJq!3exkJ8((nwh@9#B=6FTs+&s6ed2q zlXXuK$jeh>RG1XnAgGlbd>sw@zL=+Dj5&R*f}lUe{dZYfd)$QRg(lpuYX#|^=&&$^xUkb|mkJHli!d9O zF%&N?k?)s2GaJ#%>n`LqdrP|3LCTn%g}m%cxEf9~+N&nIOuNw)3quJkh21QILn{HC zl!Ywimws|Ew;yE=bSu^*FJx?q=MD2eZ+MMx)~urTeB+U7cdR_$m62D3 zp~SRmnv-F^M7!?>KnLlcxTDYZVrZ8qb$m*-h*obsn0#SBGTdUBOO^91EIPFb`(AL$ zdq~|jh!TwDMUdn($Sg%5eyQ-$)XUeZ!r%P7j6~zm1hZq!)(7s)zY|=b4zi3j?PWIM z-odz|jNR})WzW%TWaz-IQY4d?H1b|wcG*$3ZjIe+@- z!hJZV@cgO?^F#c6UwvG650y}dMYmu}ZSQvu)He=HqkE0ZcP9*&!K~N)a6b^E?Rm3p zzo?N=T1j4g@}MdI4!OGhJljsQVo_(x=lX54XCWVJLLd8cNjl}09Es;w;vX#n57h@V z4c&n#;trUTv3MTOc~)vUlTo*|P#Q=3-Cem{K5&#J>R2`ST;!a(m_qF0Gt`(@cQi&I z-qMe!Z}jx>`m;V8Q`eDNQI5US&p0n@HC{?!X+c3iI_d#hEn&8>B(xp3bfvLsyk z?EX0fJyjz-RruKasXt@%*7MNn#r05)GD!+)M{H<>LJzqpa3>X+NmloJ?pCH-Slya4 zw?PlP7Y1c{p!_1a1pO|JpKCs~O!l`gV$L@|NlsQWHI;BGt^I5iq-egTa5m28aZbtb zwNBulFiRfGP(GAh_rGV(7MN(JJ2+6(+96Gnc*qQ^OB;?wiz)I{*jRt*)Jb2?u>Q)4 zF9C;C=?Q(nZ1~9f+x@i%!i}lPuSPqFB=Q7P?y9h7@9)Nj<~$jzf4Ku|L8t66nWK8U zfQ6l%ppwz>ncHOKQV`cc=n++ff+Dqzd(OrF8u5!H5>?8XN!<6kAHX`eAA_kllb{Y} zUzG{Xvn1ZuE|I07ycDCivK%RhC8Q(yrVSDSO7h|5Biw4XUB{h|lbTI!q zPKAlc6?Jxm8z|^O@K5vK;4%hVgQ3R2E%na&77(bp-LLRY@SsY*=>@LG2f;t3e}l&s zY-10x1^<%64&rn}go|AO;6XtpNBd*uC^NSe0g%6bxIjQ{`u*nu=z|#im%sFS+S6>B z8fQRg6aU%b{C{G_5X71zA@)BSum2x|5D?S@KtIg-kBQ}@J+;tVQeu)yQtY;NK==OZd4-R4y+FKy2GB1( z@NpuHHIU`hV*fBEBNNl_Qcp2hZJ2`L3}9S-fX^aon3(l9rCK|mW~lIkuLjmwObO>e zpy2P9KK~Io{?M>KfbrXb!x9X$g&5iW@FDm~*kAY%4X)?C+h`yVJ^&$vpleGxBmfNU zO-$eqk2o2hMg#ER!Ez_Bk^Q?ffj}Y%0D?+L06^_6t(~PU3{36V%}$G;-}e#ZiVgw= z00~6`0bv>$1S4mdnH5yu(!kcv%m98DjD z;Oo37)C35!{)3>e1_=T~XBgNHV)Z)&VcH?r`_2Fr79I$61Hlv$8;~G?K`g;`Fayg| zTLN4Q9~KIA+9*7=m;;ZC?6*t&9y83gLfKRNxkFQfc^ zW-_vZLcvBbh!yk)!~b-{|6W%ANKWwf=z|!wBr*&(7GS6;1PbQ(9+3ZMf&F)d$rahj zz=R3{y#hQ@8etCN{u#i@(0mGOv2o|FQ26LckPmAA2n4}j{->?} zZ7NNLvW3yWpPV`%kSxOVDEj>)5K9B7fhpJ$427LiE%5o#sbB0kSOYyS@Al4TlX$Ck?Q%_8RD7 z5hGnE2o;HN|5vEBw*%Y4oUOsXVZ8hINZkB@%0mOFJP;!t|^)88TJ z3?;C*3TPk-(119C2Hc2{ATS0476CD^fVhC+Dd+T>h~1+xz7H^X13BkMHSkBTOmrCu zj=%e}(*_U5hf)^OKx3o|d_R`k{s35`Kmy>;=IGDX;PiS>k$8z$4ai*vtOC&@@+pzP z_^UNI9RWP)h9zvKZ)E{9(T{p!CPt}|ATYA9w}bupTq&?Zbc$rV>7VA24h;VZ&>JB3 zCL%A90D#9QefX29zY`cd9_&mf0TFYW_@p&x^&-LWec?hM?w;Q@aq33VgcKqr3oLi8 z0F4TwDlCSOzyR{Qk-e?$?^PSDa*Hn6z;uKIK^xH`S?7>o_$x-8zFl@mG&~mz#3*e_9_w(5UW%7DV71;BW6HI5CkRxarhlbN=BkV z!6D7(LGzT2Fa`F|pfFpoff3Lq{YD|9V%QXu4iqtq*G?8y4oAq57}z-jZ#&z9f1kx%kdY0( z$SVW%D2N4K94^}L7kGeiepiYGu;TpNdSM%HecugOiVnxgrj3ypIhOC1^7|m9sAYjD zfCF<=oNNcIXpjSeIYNM3X8ijo(8mOYo`9_oiJ$bIO*SM@PE=W^kEDJ5!(pKaR7)tV66kO)aw;T0^-NRn5Wy9fya=3Nb^P~kWbKo ze1cf&Da#-MfLQ9CeliF&hOQj~alH#bL5%C+3P@o5T<4sw+JozYQz*db3l#tnbTYf2 zQAPs7$i@QBaoSP~UT+@E&Gzlk14oLJPZGeaksz?PvVa&t{w6p@!8x{MspbV3qbub} zgPe*(f&e)B092d5lS&X@hy8gad@vmefS=!rJzZ8sd^+uCfb02409gP0klL>a;8uWm zNzl&(Z{8q7@Z-v$U$-3aO=-kq?LSLTUi*FN|8dp=;gCB(^otGalScyN5c={j>;D4L z=?)A0QT@Vk5Kk}ttf1lDe*x#jS*HI_MSn9Po?QBw3Hte8VEXUpm`;a=c!=p|w4;9m d4e?0R|3W$gET_QN*knMRv>;H}9*|~0{|Cc~@OJ hintedNodes) { - Set liveEndpoints = new HashSet<>(); String ks = mutation.getKeyspaceName(); + Keyspace keyspace = Keyspace.open(ks); Token tk = mutation.key().getToken(); - for (InetAddressAndPort endpoint : StorageService.instance.getNaturalAndPendingEndpoints(ks, tk)) + EndpointsForToken replicas = StorageService.instance.getNaturalAndPendingReplicasForToken(ks, tk); + Replicas.temporaryAssertFull(replicas); // TODO in CASSANDRA-14549 + + EndpointsForToken.Builder liveReplicasBuilder = EndpointsForToken.builder(tk); + for (Replica replica : replicas) { - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) + if (replica.isLocal()) { mutation.apply(); } - else if (FailureDetector.instance.isAlive(endpoint)) + else if (FailureDetector.instance.isAlive(replica.endpoint())) { - liveEndpoints.add(endpoint); // will try delivering directly instead of writing a hint. + liveReplicasBuilder.add(replica); // will try delivering directly instead of writing a hint. } else { - hintedNodes.add(endpoint); - HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(endpoint), + hintedNodes.add(replica.endpoint()); + HintsService.instance.write(StorageService.instance.getHostIdForEndpoint(replica.endpoint()), Hint.create(mutation, writtenAt)); } } - if (liveEndpoints.isEmpty()) + EndpointsForToken liveReplicas = liveReplicasBuilder.build(); + if (liveReplicas.isEmpty()) return null; - ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(liveEndpoints, System.nanoTime()); + Replicas.temporaryAssertFull(liveReplicas); + ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(keyspace, liveReplicas, System.nanoTime()); MessageOut message = mutation.createMessage(); - for (InetAddressAndPort endpoint : liveEndpoints) - MessagingService.instance().sendRR(message, endpoint, handler, false); + for (Replica replica : liveReplicas) + MessagingService.instance().sendWriteRR(message, replica, handler, false); return handler; } @@ -497,16 +509,17 @@ public class BatchlogManager implements BatchlogManagerMBean { private final Set undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>()); - ReplayWriteResponseHandler(Collection writeEndpoints, long queryStartNanoTime) + ReplayWriteResponseHandler(Keyspace keyspace, EndpointsForToken writeReplicas, long queryStartNanoTime) { - super(writeEndpoints, Collections.emptySet(), null, null, null, WriteType.UNLOGGED_BATCH, queryStartNanoTime); - undelivered.addAll(writeEndpoints); + super(ReplicaLayout.forWriteWithDownNodes(keyspace, null, writeReplicas.token(), writeReplicas, EndpointsForToken.empty(writeReplicas.token())), + null, WriteType.UNLOGGED_BATCH, queryStartNanoTime); + Iterables.addAll(undelivered, writeReplicas.endpoints()); } @Override protected int totalBlockFor() { - return this.naturalEndpoints.size(); + return this.replicaLayout.selected().size(); } @Override diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index a13070cc58..783dcc1a02 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -339,6 +339,8 @@ public class Config public boolean enable_materialized_views = true; + public boolean enable_transient_replication = false; + /** * Optionally disable asynchronous UDF execution. * Disabling asynchronous UDF execution also implicitly disables the security-manager! diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index af13f9c950..75b3fc3d88 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -59,6 +59,7 @@ import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointSnitchInfo; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.SeedProvider; import org.apache.cassandra.net.BackPressureStrategy; import org.apache.cassandra.net.RateBasedBackPressure; @@ -122,7 +123,7 @@ public class DatabaseDescriptor private static long indexSummaryCapacityInMB; private static String localDC; - private static Comparator localComparator; + private static Comparator localComparator; private static EncryptionContext encryptionContext; private static boolean hasLoggedConfig; @@ -991,18 +992,14 @@ public class DatabaseDescriptor EndpointSnitchInfo.create(); localDC = snitch.getDatacenter(FBUtilities.getBroadcastAddressAndPort()); - localComparator = new Comparator() - { - public int compare(InetAddressAndPort endpoint1, InetAddressAndPort endpoint2) - { - boolean local1 = localDC.equals(snitch.getDatacenter(endpoint1)); - boolean local2 = localDC.equals(snitch.getDatacenter(endpoint2)); - if (local1 && !local2) - return -1; - if (local2 && !local1) - return 1; - return 0; - } + localComparator = (replica1, replica2) -> { + boolean local1 = localDC.equals(snitch.getDatacenter(replica1)); + boolean local2 = localDC.equals(snitch.getDatacenter(replica2)); + if (local1 && !local2) + return -1; + if (local2 && !local1) + return 1; + return 0; }; } @@ -2308,7 +2305,7 @@ public class DatabaseDescriptor return localDC; } - public static Comparator getLocalComparator() + public static Comparator getLocalComparator() { return localComparator; } @@ -2459,6 +2456,16 @@ public class DatabaseDescriptor return conf.enable_materialized_views; } + public static boolean isTransientReplicationEnabled() + { + return conf.enable_transient_replication; + } + + public static void setTransientReplicationEnabledUnsafe(boolean enabled) + { + conf.enable_transient_replication = enabled; + } + public static long getUserDefinedFunctionFailTimeout() { return conf.user_defined_function_fail_timeout; diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index 79e19c1298..45db9470ea 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -202,7 +202,18 @@ public class QueryProcessor implements QueryHandler statement.authorize(clientState); statement.validate(clientState); - ResultMessage result = statement.execute(queryState, options, queryStartNanoTime); + ResultMessage result; + if (options.getConsistency() == ConsistencyLevel.NODE_LOCAL) + { + assert Boolean.getBoolean("cassandra.enable_nodelocal_queries") : "Node local consistency level is highly dangerous and should be used only for debugging purposes"; + assert statement instanceof SelectStatement : "Only SELECT statements are permitted for node-local execution"; + logger.info("Statement {} executed with NODE_LOCAL consistency level.", statement); + result = statement.executeLocally(queryState, options); + } + else + { + result = statement.execute(queryState, options, queryStartNanoTime); + } return result == null ? new ResultMessage.Void() : result; } diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java index e9257358c6..fa637ef8ac 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java @@ -261,7 +261,7 @@ public class BatchStatement implements CQLStatement return statements; } - private Collection getMutations(BatchQueryOptions options, + private List getMutations(BatchQueryOptions options, boolean local, long batchTimestamp, int nowInSeconds, @@ -401,7 +401,7 @@ public class BatchStatement implements CQLStatement return new ResultMessage.Void(); } - private void executeWithoutConditions(Collection mutations, ConsistencyLevel cl, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException + private void executeWithoutConditions(List mutations, ConsistencyLevel cl, long queryStartNanoTime) throws RequestExecutionException, RequestValidationException { if (mutations.isEmpty()) return; diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java index 96d9f5ace4..8f70ffc888 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java @@ -104,7 +104,7 @@ final class BatchUpdatesCollector implements UpdatesCollector * Returns a collection containing all the mutations. * @return a collection containing all the mutations. */ - public Collection toMutations() + public List toMutations() { //TODO: The case where all statement where on the same keyspace is pretty common, optimize for that? List ms = new ArrayList<>(); diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index 13fc6591bd..a8367f05aa 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -465,7 +465,7 @@ public abstract class ModificationStatement implements CQLStatement else cl.validateForWrite(metadata.keyspace); - Collection mutations = + List mutations = getMutations(options, false, options.getTimestamp(queryState), @@ -676,7 +676,7 @@ public abstract class ModificationStatement implements CQLStatement * * @return list of the mutations */ - private Collection getMutations(QueryOptions options, + private List getMutations(QueryOptions options, boolean local, long timestamp, int nowInSeconds, diff --git a/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java index 1def3fd0c5..6ef551d64a 100644 --- a/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java +++ b/src/java/org/apache/cassandra/cql3/statements/SingleTableUpdatesCollector.java @@ -82,7 +82,7 @@ final class SingleTableUpdatesCollector implements UpdatesCollector * Returns a collection containing all the mutations. * @return a collection containing all the mutations. */ - public Collection toMutations() + public List toMutations() { List ms = new ArrayList<>(); for (PartitionUpdate.Builder builder : puBuilders.values()) diff --git a/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java index 30db7ca5ce..c3dd334971 100644 --- a/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java +++ b/src/java/org/apache/cassandra/cql3/statements/UpdatesCollector.java @@ -18,17 +18,16 @@ package org.apache.cassandra.cql3.statements; -import java.util.Collection; +import java.util.List; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.IMutation; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.schema.TableMetadata; public interface UpdatesCollector { PartitionUpdate.Builder getPartitionUpdateBuilder(TableMetadata metadata, DecoratedKey dk, ConsistencyLevel consistency); - Collection toMutations(); + List toMutations(); } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java index 12e73d0db4..2f0c188d5b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterKeyspaceStatement.java @@ -17,16 +17,27 @@ */ package org.apache.cassandra.cql3.statements.schema; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; import org.apache.cassandra.audit.AuditLogContext; import org.apache.cassandra.audit.AuditLogEntryType; import org.apache.cassandra.auth.Permission; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLStatement; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.ReplicationFactor; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceMetadata.KeyspaceDiff; import org.apache.cassandra.schema.Keyspaces; @@ -34,9 +45,13 @@ import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.transport.Event.SchemaChange; import org.apache.cassandra.transport.Event.SchemaChange.Change; +import org.apache.cassandra.utils.FBUtilities; public final class AlterKeyspaceStatement extends AlterSchemaStatement { + private static final boolean allow_alter_rf_during_range_movement = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_alter_rf_during_range_movement"); + private static final boolean allow_unsafe_transient_changes = Boolean.getBoolean(Config.PROPERTY_PREFIX + "allow_unsafe_transient_changes"); + private final KeyspaceAttributes attrs; public AlterKeyspaceStatement(String keyspaceName, KeyspaceAttributes attrs) @@ -60,6 +75,9 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement newKeyspace.params.validate(keyspaceName); + validateNoRangeMovements(); + validateTransientReplication(keyspace.createReplicationStrategy(), newKeyspace.createReplicationStrategy()); + return schema.withAddedOrUpdated(newKeyspace); } @@ -84,11 +102,77 @@ public final class AlterKeyspaceStatement extends AlterSchemaStatement AbstractReplicationStrategy before = keyspaceDiff.before.createReplicationStrategy(); AbstractReplicationStrategy after = keyspaceDiff.after.createReplicationStrategy(); - return before.getReplicationFactor() < after.getReplicationFactor() + return before.getReplicationFactor().fullReplicas < after.getReplicationFactor().fullReplicas ? ImmutableSet.of("When increasing replication factor you need to run a full (-full) repair to distribute the data.") : ImmutableSet.of(); } + private void validateNoRangeMovements() + { + if (allow_alter_rf_during_range_movement) + return; + + Stream endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(), Gossiper.instance.getUnreachableMembers().stream()); + List notNormalEndpoints = endpoints.filter(endpoint -> !FBUtilities.getBroadcastAddressAndPort().equals(endpoint) && + !Gossiper.instance.getEndpointStateForEndpoint(endpoint).isNormalState()) + .collect(Collectors.toList()); + if (!notNormalEndpoints.isEmpty()) + { + throw new ConfigurationException("Cannot alter RF while some endpoints are not in normal state (no range movements): " + notNormalEndpoints); + } + } + + private void validateTransientReplication(AbstractReplicationStrategy oldStrategy, AbstractReplicationStrategy newStrategy) + { + //If there is no read traffic there are some extra alterations you can safely make, but this is so atypical + //that a good default is to not allow unsafe changes + if (allow_unsafe_transient_changes) + return; + + ReplicationFactor oldRF = oldStrategy.getReplicationFactor(); + ReplicationFactor newRF = newStrategy.getReplicationFactor(); + + int oldTrans = oldRF.transientReplicas(); + int oldFull = oldRF.fullReplicas; + int newTrans = newRF.transientReplicas(); + int newFull = newRF.fullReplicas; + + if (newTrans > 0) + { + if (DatabaseDescriptor.getNumTokens() > 1) + throw new ConfigurationException(String.format("Transient replication is not supported with vnodes yet")); + + Keyspace ks = Keyspace.open(keyspaceName); + for (ColumnFamilyStore cfs : ks.getColumnFamilyStores()) + { + if (cfs.viewManager.hasViews()) + { + throw new ConfigurationException("Cannot use transient replication on keyspaces using materialized views"); + } + + if (cfs.indexManager.hasIndexes()) + { + throw new ConfigurationException("Cannot use transient replication on keyspaces using secondary indexes"); + } + } + } + + //This is true right now because the transition from transient -> full lacks the pending state + //necessary for correctness. What would happen if we allowed this is that we would attempt + //to read from a transient replica as if it were a full replica. + if (oldFull > newFull && oldTrans > 0) + throw new ConfigurationException("Can't add full replicas if there are any transient replicas. You must first remove all transient replicas, then change the # of full replicas, then add back the transient replicas"); + + //Don't increase transient replication factor by more than one at a time if changing number of replicas + //Just like with changing full replicas it's not safe to do this as you could read from too many replicas + //that don't have the necessary data. W/O transient replication this alteration was allowed and it's not clear + //if it should be. + //This is structured so you can convert as many full replicas to transient replicas as you want. + boolean numReplicasChanged = oldTrans + oldFull != newTrans + newFull; + if (numReplicasChanged && (newTrans > oldTrans && newTrans != oldTrans + 1)) + throw new ConfigurationException("Can only safely increase number of transients one at a time with incremental repair run in between each time"); + } + @Override public AuditLogContext getAuditLogContext() { diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java index 3ec75b25e7..504411968e 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.schema.*; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.transport.Event.SchemaChange; import org.apache.cassandra.transport.Event.SchemaChange.Change; import org.apache.cassandra.transport.Event.SchemaChange.Target; @@ -360,6 +361,12 @@ public abstract class AlterTableStatement extends AlterSchemaStatement "before being replayed."); } + if (keyspace.createReplicationStrategy().hasTransientReplicas() + && params.readRepair != ReadRepairStrategy.NONE) + { + throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces"); + } + return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params))); } } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java index df41358d9f..dbca160e3d 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java @@ -28,7 +28,9 @@ import org.apache.cassandra.cql3.CQLStatement; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.QualifiedName; import org.apache.cassandra.cql3.statements.schema.IndexTarget.Type; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.schema.*; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; @@ -88,6 +90,9 @@ public final class CreateIndexStatement extends AlterSchemaStatement if (table.isView()) throw ire("Secondary indexes on materialized views aren't supported"); + if (Keyspace.open(table.keyspace).getReplicationStrategy().hasTransientReplicas()) + throw new InvalidRequestException("Secondary indexes are not supported on transiently replicated keyspaces"); + List indexTargets = Lists.newArrayList(transform(rawIndexTargets, t -> t.prepare(table))); if (indexTargets.isEmpty() && !attrs.isCustom) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java index 62fcafe6b8..be7907f2bb 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java @@ -27,11 +27,14 @@ import org.apache.cassandra.auth.DataResource; import org.apache.cassandra.auth.IResource; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.cql3.*; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.AlreadyExistsException; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.schema.*; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.transport.Event.SchemaChange; import org.apache.cassandra.transport.Event.SchemaChange.Change; import org.apache.cassandra.transport.Event.SchemaChange.Target; @@ -98,6 +101,12 @@ public final class CreateTableStatement extends AlterSchemaStatement TableMetadata table = builder(keyspace.types).build(); table.validate(); + if (keyspace.createReplicationStrategy().hasTransientReplicas() + && table.params.readRepair != ReadRepairStrategy.NONE) + { + throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces"); + } + return schema.withAddedOrUpdated(keyspace.withSwapped(keyspace.tables.with(table))); } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java index 5f6200179f..bf6bcff53b 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateViewStatement.java @@ -31,9 +31,11 @@ import org.apache.cassandra.cql3.restrictions.StatementRestrictions; import org.apache.cassandra.cql3.selection.RawSelector; import org.apache.cassandra.cql3.selection.Selectable; import org.apache.cassandra.cql3.statements.StatementType; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.ReversedType; import org.apache.cassandra.exceptions.AlreadyExistsException; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.schema.*; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.ClientState; @@ -107,6 +109,9 @@ public final class CreateViewStatement extends AlterSchemaStatement if (null == keyspace) throw ire("Keyspace '%s' doesn't exist", keyspaceName); + if (keyspace.createReplicationStrategy().hasTransientReplicas()) + throw new InvalidRequestException("Materialized views are not supported on transiently replicated keyspaces"); + TableMetadata table = keyspace.tables.getNullable(tableName); if (null == table) throw ire("Base table '%s' doesn't exist", tableName); diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index c8e464adf7..4e66307780 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -128,6 +128,9 @@ public final class TableAttributes extends PropertyDefinitions if (hasOption(Option.SPECULATIVE_RETRY)) builder.speculativeRetry(SpeculativeRetryPolicy.fromString(getString(Option.SPECULATIVE_RETRY))); + if (hasOption(Option.SPECULATIVE_WRITE_THRESHOLD)) + builder.speculativeWriteThreshold(SpeculativeRetryPolicy.fromString(getString(Option.SPECULATIVE_WRITE_THRESHOLD))); + if (hasOption(Option.CRC_CHECK_CHANCE)) builder.crcCheckChance(getDouble(Option.CRC_CHECK_CHANCE)); diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 5e3858480a..56851e2dcb 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -30,7 +30,6 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; -import javax.annotation.Nullable; import javax.management.*; import javax.management.openmbean.*; @@ -68,7 +67,6 @@ import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.KeyIterator; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.*; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -81,7 +79,6 @@ import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.repair.TableRepairManager; import org.apache.cassandra.schema.*; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; -import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.TableStreamManager; @@ -205,7 +202,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean private final Directories directories; public final TableMetrics metric; - public volatile long sampleLatencyNanos; + public volatile long sampleReadLatencyNanos; + public volatile long transientWriteLatencyNanos; private final CassandraTableWriteHandler writeHandler; private final CassandraStreamManager streamManager; @@ -384,7 +382,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean viewManager = keyspace.viewManager.forTable(metadata.id); metric = new TableMetrics(this); fileIndexGenerator.set(generation); - sampleLatencyNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getReadRpcTimeout() / 2); + sampleReadLatencyNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getReadRpcTimeout() / 2); + transientWriteLatencyNanos = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getWriteRpcTimeout() / 2); logger.info("Initializing {}.{}", keyspace.getName(), name); @@ -454,7 +453,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { try { - sampleLatencyNanos = metadata().params.speculativeRetry.calculateThreshold(metric.coordinatorReadLatency); + sampleReadLatencyNanos = metadata().params.speculativeRetry.calculateThreshold(metric.coordinatorReadLatency); + transientWriteLatencyNanos = metadata().params.speculativeWriteThreshold.calculateThreshold(metric.coordinatorWriteLatency); } catch (Throwable e) { @@ -487,15 +487,15 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return directories; } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, int sstableLevel, SerializationHeader header, LifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header, LifecycleTransaction txn) { MetadataCollector collector = new MetadataCollector(metadata().comparator).sstableLevel(sstableLevel); - return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, collector, header, txn); + return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, collector, header, txn); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, MetadataCollector metadataCollector, SerializationHeader header, LifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, MetadataCollector metadataCollector, SerializationHeader header, LifecycleTransaction txn) { - return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, metadataCollector, header, indexManager.listIndexes(), txn); + return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadataCollector, header, indexManager.listIndexes(), txn); } public boolean supportsEarlyOpen() @@ -1402,7 +1402,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean // cleanup size estimation only counts bytes for keys local to this node long expectedFileSize = 0; - Collection> ranges = StorageService.instance.getLocalRanges(keyspace.getName()); + Collection> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges(); for (SSTableReader sstable : sstables) { List positions = sstable.getPositionsForRanges(ranges); @@ -1677,7 +1677,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean public void cleanupCache() { - Collection> ranges = StorageService.instance.getLocalRanges(keyspace.getName()); + Collection> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges(); for (Iterator keyIter = CacheService.instance.rowCache.keyIterator(); keyIter.hasNext(); ) diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java index d37da0a8df..35ba19802e 100644 --- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java +++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java @@ -17,16 +17,18 @@ */ package org.apache.cassandra.db; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import com.google.common.collect.Iterables; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.Replicas; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -47,7 +49,8 @@ public enum ConsistencyLevel EACH_QUORUM (7), SERIAL (8), LOCAL_SERIAL(9), - LOCAL_ONE (10, true); + LOCAL_ONE (10, true), + NODE_LOCAL (11, true); private static final Logger logger = LoggerFactory.getLogger(ConsistencyLevel.class); @@ -89,13 +92,13 @@ public enum ConsistencyLevel private int quorumFor(Keyspace keyspace) { - return (keyspace.getReplicationStrategy().getReplicationFactor() / 2) + 1; + return (keyspace.getReplicationStrategy().getReplicationFactor().allReplicas / 2) + 1; } private int localQuorumFor(Keyspace keyspace, String dc) { return (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) - ? (((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getReplicationFactor(dc) / 2) + 1 + ? (((NetworkTopologyStrategy) keyspace.getReplicationStrategy()).getReplicationFactor(dc).allReplicas / 2) + 1 : quorumFor(keyspace); } @@ -116,7 +119,7 @@ public enum ConsistencyLevel case SERIAL: return quorumFor(keyspace); case ALL: - return keyspace.getReplicationStrategy().getReplicationFactor(); + return keyspace.getReplicationStrategy().getReplicationFactor().allReplicas; case LOCAL_QUORUM: case LOCAL_SERIAL: return localQuorumFor(keyspace, DatabaseDescriptor.getLocalDataCenter()); @@ -138,6 +141,28 @@ public enum ConsistencyLevel } } + public int blockForWrite(Keyspace keyspace, Endpoints pending) + { + assert pending != null; + + int blockFor = blockFor(keyspace); + switch (this) + { + case ANY: + break; + case LOCAL_ONE: case LOCAL_QUORUM: case LOCAL_SERIAL: + // we will only count local replicas towards our response count, as these queries only care about local guarantees + blockFor += countDCLocalReplicas(pending).allReplicas(); + break; + case ONE: case TWO: case THREE: + case QUORUM: case EACH_QUORUM: + case SERIAL: + case ALL: + blockFor += pending.size(); + } + return blockFor; + } + /** * Determine if this consistency level meets or exceeds the consistency requirements of the given cl for the given keyspace */ @@ -156,40 +181,75 @@ public enum ConsistencyLevel return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint)); } - public int countLocalEndpoints(Iterable liveEndpoints) + public static boolean isLocal(Replica replica) { - int count = 0; - for (InetAddressAndPort endpoint : liveEndpoints) - if (isLocal(endpoint)) - count++; + return isLocal(replica.endpoint()); + } + + private static ReplicaCount countDCLocalReplicas(ReplicaCollection liveReplicas) + { + ReplicaCount count = new ReplicaCount(); + for (Replica replica : liveReplicas) + if (isLocal(replica)) + count.increment(replica); return count; } - private Map countPerDCEndpoints(Keyspace keyspace, Iterable liveEndpoints) + private static class ReplicaCount + { + int fullReplicas; + int transientReplicas; + + int allReplicas() + { + return fullReplicas + transientReplicas; + } + + void increment(Replica replica) + { + if (replica.isFull()) ++fullReplicas; + else ++transientReplicas; + } + + boolean isSufficient(int allReplicas, int fullReplicas) + { + return this.fullReplicas >= fullReplicas + && this.allReplicas() >= allReplicas; + } + } + + private static Map countPerDCEndpoints(Keyspace keyspace, Iterable liveReplicas) { NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy(); - Map dcEndpoints = new HashMap(); + Map dcEndpoints = new HashMap<>(); for (String dc: strategy.getDatacenters()) - dcEndpoints.put(dc, 0); + dcEndpoints.put(dc, new ReplicaCount()); - for (InetAddressAndPort endpoint : liveEndpoints) + for (Replica replica : liveReplicas) { - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint); - dcEndpoints.put(dc, dcEndpoints.get(dc) + 1); + String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica); + dcEndpoints.get(dc).increment(replica); } return dcEndpoints; } - public List filterForQuery(Keyspace keyspace, List liveEndpoints) + public > E filterForQuery(Keyspace keyspace, E liveReplicas) + { + return filterForQuery(keyspace, liveReplicas, false); + } + + public > E filterForQuery(Keyspace keyspace, E liveReplicas, boolean alwaysSpeculate) { /* * If we are doing an each quorum query, we have to make sure that the endpoints we select * provide a quorum for each data center. If we are not using a NetworkTopologyStrategy, * we should fall through and grab a quorum in the replication strategy. + * + * We do not speculate for EACH_QUORUM. */ if (this == EACH_QUORUM && keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) - return filterForEachQuorum(keyspace, liveEndpoints); + return filterForEachQuorum(keyspace, liveReplicas); /* * Endpoints are expected to be restricted to live replicas, sorted by snitch preference. @@ -198,36 +258,34 @@ public enum ConsistencyLevel * the blockFor first ones). */ if (isDCLocal) - liveEndpoints.sort(DatabaseDescriptor.getLocalComparator()); + liveReplicas = liveReplicas.sorted(DatabaseDescriptor.getLocalComparator()); - return liveEndpoints.subList(0, Math.min(liveEndpoints.size(), blockFor(keyspace))); + return liveReplicas.subList(0, Math.min(liveReplicas.size(), blockFor(keyspace) + (alwaysSpeculate ? 1 : 0))); } - private List filterForEachQuorum(Keyspace keyspace, List liveEndpoints) + private > E filterForEachQuorum(Keyspace keyspace, E liveReplicas) { NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy(); - - Map> dcsEndpoints = new HashMap<>(); - for (String dc: strategy.getDatacenters()) - dcsEndpoints.put(dc, new ArrayList<>()); - - for (InetAddressAndPort add : liveEndpoints) + Map dcsReplicas = new HashMap<>(); + for (String dc : strategy.getDatacenters()) { - String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(add); - dcsEndpoints.get(dc).add(add); + // we put _up to_ dc replicas only + dcsReplicas.put(dc, localQuorumFor(keyspace, dc)); } - List waitSet = new ArrayList<>(); - for (Map.Entry> dcEndpoints : dcsEndpoints.entrySet()) - { - List dcEndpoint = dcEndpoints.getValue(); - waitSet.addAll(dcEndpoint.subList(0, Math.min(localQuorumFor(keyspace, dcEndpoints.getKey()), dcEndpoint.size()))); - } - - return waitSet; + return liveReplicas.filter((replica) -> { + String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica); + int replicas = dcsReplicas.get(dc); + if (replicas > 0) + { + dcsReplicas.put(dc, --replicas); + return true; + } + return false; + }); } - public boolean isSufficientLiveNodes(Keyspace keyspace, Iterable liveEndpoints) + public boolean isSufficientLiveNodesForRead(Keyspace keyspace, Endpoints liveReplicas) { switch (this) { @@ -235,75 +293,92 @@ public enum ConsistencyLevel // local hint is acceptable, and local node is always live return true; case LOCAL_ONE: - return countLocalEndpoints(liveEndpoints) >= 1; + return countDCLocalReplicas(liveReplicas).isSufficient(1, 1); case LOCAL_QUORUM: - return countLocalEndpoints(liveEndpoints) >= blockFor(keyspace); + return countDCLocalReplicas(liveReplicas).isSufficient(blockFor(keyspace), 1); case EACH_QUORUM: if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) { - for (Map.Entry entry : countPerDCEndpoints(keyspace, liveEndpoints).entrySet()) + int fullCount = 0; + for (Map.Entry entry : countPerDCEndpoints(keyspace, liveReplicas).entrySet()) { - if (entry.getValue() < localQuorumFor(keyspace, entry.getKey())) + ReplicaCount count = entry.getValue(); + if (!count.isSufficient(localQuorumFor(keyspace, entry.getKey()), 0)) return false; + fullCount += count.fullReplicas; } - return true; + return fullCount > 0; } // Fallthough on purpose for SimpleStrategy default: - return Iterables.size(liveEndpoints) >= blockFor(keyspace); + return liveReplicas.size() >= blockFor(keyspace) + && Replicas.countFull(liveReplicas) > 0; } } - public void assureSufficientLiveNodes(Keyspace keyspace, Iterable liveEndpoints) throws UnavailableException + public void assureSufficientLiveNodesForRead(Keyspace keyspace, Endpoints liveReplicas) throws UnavailableException + { + assureSufficientLiveNodes(keyspace, liveReplicas, blockFor(keyspace), 1); + } + public void assureSufficientLiveNodesForWrite(Keyspace keyspace, Endpoints allLive, Endpoints pendingWithDown) throws UnavailableException + { + assureSufficientLiveNodes(keyspace, allLive, blockForWrite(keyspace, pendingWithDown), 0); + } + public void assureSufficientLiveNodes(Keyspace keyspace, Endpoints allLive, int blockFor, int blockForFullReplicas) throws UnavailableException { - int blockFor = blockFor(keyspace); switch (this) { case ANY: // local hint is acceptable, and local node is always live break; case LOCAL_ONE: - if (countLocalEndpoints(liveEndpoints) == 0) - throw new UnavailableException(this, 1, 0); + { + ReplicaCount localLive = countDCLocalReplicas(allLive); + if (!localLive.isSufficient(blockFor, blockForFullReplicas)) + throw UnavailableException.create(this, 1, blockForFullReplicas, localLive.allReplicas(), localLive.fullReplicas); break; + } case LOCAL_QUORUM: - int localLive = countLocalEndpoints(liveEndpoints); - if (localLive < blockFor) + { + ReplicaCount localLive = countDCLocalReplicas(allLive); + if (!localLive.isSufficient(blockFor, blockForFullReplicas)) { if (logger.isTraceEnabled()) { - StringBuilder builder = new StringBuilder("Local replicas ["); - for (InetAddressAndPort endpoint : liveEndpoints) - { - if (isLocal(endpoint)) - builder.append(endpoint).append(","); - } - builder.append("] are insufficient to satisfy LOCAL_QUORUM requirement of ").append(blockFor).append(" live nodes in '").append(DatabaseDescriptor.getLocalDataCenter()).append("'"); - logger.trace(builder.toString()); + logger.trace(String.format("Local replicas %s are insufficient to satisfy LOCAL_QUORUM requirement of %d live replicas and %d full replicas in '%s'", + allLive.filter(ConsistencyLevel::isLocal), blockFor, blockForFullReplicas, DatabaseDescriptor.getLocalDataCenter())); } - throw new UnavailableException(this, blockFor, localLive); + throw UnavailableException.create(this, blockFor, blockForFullReplicas, localLive.allReplicas(), localLive.fullReplicas); } break; + } case EACH_QUORUM: if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) { - for (Map.Entry entry : countPerDCEndpoints(keyspace, liveEndpoints).entrySet()) + int total = 0; + int totalFull = 0; + for (Map.Entry entry : countPerDCEndpoints(keyspace, allLive).entrySet()) { int dcBlockFor = localQuorumFor(keyspace, entry.getKey()); - int dcLive = entry.getValue(); - if (dcLive < dcBlockFor) - throw new UnavailableException(this, entry.getKey(), dcBlockFor, dcLive); + ReplicaCount dcCount = entry.getValue(); + if (!dcCount.isSufficient(dcBlockFor, 0)) + throw UnavailableException.create(this, entry.getKey(), dcBlockFor, dcCount.allReplicas(), 0, dcCount.fullReplicas); + totalFull += dcCount.fullReplicas; + total += dcCount.allReplicas(); } + if (totalFull < blockForFullReplicas) + throw UnavailableException.create(this, blockFor, total, blockForFullReplicas, totalFull); break; } // Fallthough on purpose for SimpleStrategy default: - int live = Iterables.size(liveEndpoints); - if (live < blockFor) + int live = allLive.size(); + int full = Replicas.countFull(allLive); + if (live < blockFor || full < blockForFullReplicas) { if (logger.isTraceEnabled()) - logger.trace("Live nodes {} do not satisfy ConsistencyLevel ({} required)", Iterables.toString(liveEndpoints), blockFor); - throw new UnavailableException(this, blockFor, live); + logger.trace("Live nodes {} do not satisfy ConsistencyLevel ({} required)", Iterables.toString(allLive), blockFor); + throw UnavailableException.create(this, blockFor, blockForFullReplicas, live, full); } break; } diff --git a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java index 72b5e2ae89..acfe71aeae 100644 --- a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java +++ b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java @@ -19,8 +19,9 @@ package org.apache.cassandra.db; import java.util.ArrayList; -import java.util.Collection; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +31,8 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.PendingRangeCalculatorService; import org.apache.cassandra.service.StorageService; @@ -68,7 +71,7 @@ public class DiskBoundaryManager private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs) { - Collection> localRanges; + RangesAtEndpoint localRanges; long ringVersion; TokenMetadata tmd; @@ -87,7 +90,7 @@ public class DiskBoundaryManager // Reason we use use the future settled TMD is that if we decommission a node, we want to stream // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places. // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled - localRanges = cfs.keyspace.getReplicationStrategy().getAddressRanges(tmd.cloneAfterAllSettled()).get(FBUtilities.getBroadcastAddressAndPort()); + localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort()); } logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion); } @@ -106,9 +109,18 @@ public class DiskBoundaryManager if (localRanges == null || localRanges.isEmpty()) return new DiskBoundaries(dirs, null, ringVersion, directoriesVersion); - List> sortedLocalRanges = Range.sort(localRanges); + // note that Range.sort unwraps any wraparound ranges, so we need to sort them here + List> fullLocalRanges = Range.sort(localRanges.stream() + .filter(Replica::isFull) + .map(Replica::range) + .collect(Collectors.toList())); + List> transientLocalRanges = Range.sort(localRanges.stream() + .filter(Replica::isTransient) + .map(Replica::range) + .collect(Collectors.toList())); + + List positions = getDiskBoundaries(fullLocalRanges, transientLocalRanges, cfs.getPartitioner(), dirs); - List positions = getDiskBoundaries(sortedLocalRanges, cfs.getPartitioner(), dirs); return new DiskBoundaries(dirs, positions, ringVersion, directoriesVersion); } @@ -121,15 +133,26 @@ public class DiskBoundaryManager * * The final entry in the returned list will always be the partitioner maximum tokens upper key bound */ - private static List getDiskBoundaries(List> sortedLocalRanges, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories) + private static List getDiskBoundaries(List> fullRanges, List> transientRanges, IPartitioner partitioner, Directories.DataDirectory[] dataDirectories) { assert partitioner.splitter().isPresent(); + Splitter splitter = partitioner.splitter().get(); boolean dontSplitRanges = DatabaseDescriptor.getNumTokens() > 1; - List boundaries = splitter.splitOwnedRanges(dataDirectories.length, sortedLocalRanges, dontSplitRanges); + + List weightedRanges = new ArrayList<>(fullRanges.size() + transientRanges.size()); + for (Range r : fullRanges) + weightedRanges.add(new Splitter.WeightedRange(1.0, r)); + + for (Range r : transientRanges) + weightedRanges.add(new Splitter.WeightedRange(0.1, r)); + + weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left)); + + List boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, dontSplitRanges); // If we can't split by ranges, split evenly to ensure utilisation of all disks if (dontSplitRanges && boundaries.size() < dataDirectories.length) - boundaries = splitter.splitOwnedRanges(dataDirectories.length, sortedLocalRanges, false); + boundaries = splitter.splitOwnedRanges(dataDirectories.length, weightedRanges, false); List diskBoundaries = new ArrayList<>(); for (int i = 0; i < boundaries.size() - 1; i++) diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java index c1626978cc..436b7ef9b7 100644 --- a/src/java/org/apache/cassandra/db/Memtable.java +++ b/src/java/org/apache/cassandra/db/Memtable.java @@ -503,6 +503,7 @@ public class Memtable implements Comparable toFlush.size(), ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, + false, sstableMetadataCollector, new SerializationHeader(true, cfs.metadata(), columns, stats), txn); } diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java index 8386048f60..9660f658dd 100644 --- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.db; -import java.io.IOException; import java.util.Iterator; import org.apache.cassandra.exceptions.WriteTimeoutException; @@ -38,7 +37,7 @@ public class MutationVerbHandler implements IVerbHandler Tracing.trace("Payload application resulted in WriteTimeout, not replying"); } - public void doVerb(MessageIn message, int id) throws IOException + public void doVerb(MessageIn message, int id) { // Check if there were any forwarding headers in this message InetAddressAndPort from = (InetAddressAndPort)message.parameters.get(ParameterType.FORWARD_FROM); @@ -69,7 +68,7 @@ public class MutationVerbHandler implements IVerbHandler } } - private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, ForwardToContainer forwardTo, InetAddressAndPort from) throws IOException + private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, ForwardToContainer forwardTo, InetAddressAndPort from) { // tell the recipients who to send their ack to MessageOut message = new MessageOut<>(verb, mutation, Mutation.serializer).withParameter(ParameterType.FORWARD_FROM, from); diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index 2bfb434bff..7eab016d12 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -24,7 +24,6 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.filter.*; @@ -61,6 +60,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR private PartitionRangeReadCommand(boolean isDigest, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -69,7 +69,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR DataRange dataRange, IndexMetadata index) { - super(Kind.PARTITION_RANGE, isDigest, digestVersion, metadata, nowInSec, columnFilter, rowFilter, limits, index); + super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index); this.dataRange = dataRange; } @@ -82,6 +82,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(false, 0, + false, metadata, nowInSec, columnFilter, @@ -103,6 +104,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(false, 0, + false, metadata, nowInSec, ColumnFilter.all(metadata), @@ -151,6 +153,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR // on the ring. return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -164,6 +167,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -177,6 +181,21 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(true, digestVersion(), + false, + metadata(), + nowInSec(), + columnFilter(), + rowFilter(), + limits(), + dataRange(), + indexMetadata()); + } + + public PartitionRangeReadCommand copyAsTransientQuery() + { + return new PartitionRangeReadCommand(false, + 0, + true, metadata(), nowInSec(), columnFilter(), @@ -191,6 +210,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -205,6 +225,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { return new PartitionRangeReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -406,6 +427,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR int version, boolean isDigest, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -415,7 +437,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR throws IOException { DataRange range = DataRange.serializer.deserialize(in, version, metadata); - return new PartitionRangeReadCommand(isDigest, digestVersion, metadata, nowInSec, columnFilter, rowFilter, limits, range, index); + return new PartitionRangeReadCommand(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, index); } } } diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index 026214084a..736e3a3055 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -34,7 +34,6 @@ import org.apache.cassandra.db.transform.RTBoundCloser; import org.apache.cassandra.db.transform.RTBoundValidator; import org.apache.cassandra.db.transform.StoppingTransformation; import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.UnknownIndexException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.IndexNotAvailableException; @@ -68,6 +67,7 @@ public abstract class ReadCommand extends AbstractReadQuery private final Kind kind; private final boolean isDigestQuery; + private final boolean acceptsTransient; // if a digest query, the version for which the digest is expected. Ignored if not a digest. private int digestVersion; @@ -80,6 +80,7 @@ public abstract class ReadCommand extends AbstractReadQuery int version, boolean isDigest, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -104,6 +105,7 @@ public abstract class ReadCommand extends AbstractReadQuery protected ReadCommand(Kind kind, boolean isDigestQuery, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -115,6 +117,7 @@ public abstract class ReadCommand extends AbstractReadQuery this.kind = kind; this.isDigestQuery = isDigestQuery; this.digestVersion = digestVersion; + this.acceptsTransient = acceptsTransient; this.index = index; } @@ -175,6 +178,14 @@ public abstract class ReadCommand extends AbstractReadQuery return this; } + /** + * @return Whether this query expects only a transient data response, or a full response + */ + public boolean acceptsTransient() + { + return acceptsTransient; + } + /** * Index (metadata) chosen for this query. Can be null. * @@ -210,6 +221,7 @@ public abstract class ReadCommand extends AbstractReadQuery * Returns a copy of this command with isDigestQuery set to true. */ public abstract ReadCommand copyAsDigestQuery(); + public abstract ReadCommand copyAsTransientQuery(); protected abstract UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadExecutionController executionController); @@ -569,6 +581,16 @@ public abstract class ReadCommand extends AbstractReadQuery return (flags & 0x01) != 0; } + private static boolean acceptsTransient(int flags) + { + return (flags & 0x08) != 0; + } + + private static int acceptsTransientFlag(boolean acceptsTransient) + { + return acceptsTransient ? 0x08 : 0; + } + // We don't set this flag anymore, but still look if we receive a // command with it set in case someone is using thrift a mixed 3.0/4.0+ // cluster (which is unsupported). This is also a reminder for not @@ -592,7 +614,11 @@ public abstract class ReadCommand extends AbstractReadQuery public void serialize(ReadCommand command, DataOutputPlus out, int version) throws IOException { out.writeByte(command.kind.ordinal()); - out.writeByte(digestFlag(command.isDigestQuery()) | indexFlag(null != command.indexMetadata())); + out.writeByte( + digestFlag(command.isDigestQuery()) + | indexFlag(null != command.indexMetadata()) + | acceptsTransientFlag(command.acceptsTransient()) + ); if (command.isDigestQuery()) out.writeUnsignedVInt(command.digestVersion()); command.metadata().id.serialize(out); @@ -611,6 +637,7 @@ public abstract class ReadCommand extends AbstractReadQuery Kind kind = Kind.values()[in.readByte()]; int flags = in.readByte(); boolean isDigest = isDigest(flags); + boolean acceptsTransient = acceptsTransient(flags); // Shouldn't happen or it's a user error (see comment above) but // better complain loudly than doing the wrong thing. if (isForThrift(flags)) @@ -628,7 +655,7 @@ public abstract class ReadCommand extends AbstractReadQuery DataLimits limits = DataLimits.serializer.deserialize(in, version, metadata.comparator); IndexMetadata index = hasIndex ? deserializeIndexMetadata(in, version, metadata) : null; - return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, metadata, nowInSec, columnFilter, rowFilter, limits, index); + return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index); } private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java index c919d257de..7597f82cff 100644 --- a/src/java/org/apache/cassandra/db/SSTableImporter.java +++ b/src/java/org/apache/cassandra/db/SSTableImporter.java @@ -349,9 +349,9 @@ public class SSTableImporter } if (options.clearRepaired) { - descriptor.getMetadataSerializer().mutateRepaired(descriptor, - ActiveRepairService.UNREPAIRED_SSTABLE, - null); + descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, + null, + false); } } } diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index 97ab210c5b..c81185e7b0 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -36,7 +36,6 @@ import org.apache.cassandra.db.lifecycle.*; import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReadsListener; @@ -71,6 +70,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar @VisibleForTesting protected SinglePartitionReadCommand(boolean isDigest, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -80,7 +80,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar ClusteringIndexFilter clusteringIndexFilter, IndexMetadata index) { - super(Kind.SINGLE_PARTITION, isDigest, digestVersion, metadata, nowInSec, columnFilter, rowFilter, limits, index); + super(Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index); assert partitionKey.getPartitioner() == metadata.partitioner; this.partitionKey = partitionKey; this.clusteringIndexFilter = clusteringIndexFilter; @@ -111,6 +111,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { return new SinglePartitionReadCommand(false, 0, + false, metadata, nowInSec, columnFilter, @@ -286,6 +287,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { return new SinglePartitionReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -300,6 +302,22 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { return new SinglePartitionReadCommand(true, digestVersion(), + acceptsTransient(), + metadata(), + nowInSec(), + columnFilter(), + rowFilter(), + limits(), + partitionKey(), + clusteringIndexFilter(), + indexMetadata()); + } + + public SinglePartitionReadCommand copyAsTransientQuery() + { + return new SinglePartitionReadCommand(false, + 0, + true, metadata(), nowInSec(), columnFilter(), @@ -315,6 +333,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { return new SinglePartitionReadCommand(isDigestQuery(), digestVersion(), + acceptsTransient(), metadata(), nowInSec(), columnFilter(), @@ -1064,6 +1083,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar int version, boolean isDigest, int digestVersion, + boolean acceptsTransient, TableMetadata metadata, int nowInSec, ColumnFilter columnFilter, @@ -1074,7 +1094,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar { DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readValue(in, DatabaseDescriptor.getMaxValueSize())); ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata); - return new SinglePartitionReadCommand(isDigest, digestVersion, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, index); + return new SinglePartitionReadCommand(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, index); } } diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index fb9e889404..ff070a3b27 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -29,13 +29,15 @@ import java.util.stream.StreamSupport; import javax.management.openmbean.OpenDataException; import javax.management.openmbean.TabularData; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,6 +73,8 @@ import static java.util.Collections.singletonMap; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal; +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; public final class SystemKeyspace { @@ -95,12 +99,10 @@ public final class SystemKeyspace public static final String LOCAL = "local"; public static final String PEERS_V2 = "peers_v2"; public static final String PEER_EVENTS_V2 = "peer_events_v2"; - public static final String RANGE_XFERS = "range_xfers"; public static final String COMPACTION_HISTORY = "compaction_history"; public static final String SSTABLE_ACTIVITY = "sstable_activity"; public static final String SIZE_ESTIMATES = "size_estimates"; - public static final String AVAILABLE_RANGES = "available_ranges"; - public static final String TRANSFERRED_RANGES = "transferred_ranges"; + public static final String AVAILABLE_RANGES_V2 = "available_ranges_v2"; public static final String TRANSFERRED_RANGES_V2 = "transferred_ranges_v2"; public static final String VIEW_BUILDS_IN_PROGRESS = "view_builds_in_progress"; public static final String BUILT_VIEWS = "built_views"; @@ -110,6 +112,8 @@ public final class SystemKeyspace @Deprecated public static final String LEGACY_PEERS = "peers"; @Deprecated public static final String LEGACY_PEER_EVENTS = "peer_events"; @Deprecated public static final String LEGACY_TRANSFERRED_RANGES = "transferred_ranges"; + @Deprecated public static final String LEGACY_AVAILABLE_RANGES = "available_ranges"; + public static final TableMetadata Batches = parse(BATCHES, @@ -207,15 +211,6 @@ public final class SystemKeyspace + "PRIMARY KEY ((peer), peer_port))") .build(); - private static final TableMetadata RangeXfers = - parse(RANGE_XFERS, - "ranges requested for transfer", - "CREATE TABLE %s (" - + "token_bytes blob," - + "requested_at timestamp," - + "PRIMARY KEY ((token_bytes)))") - .build(); - private static final TableMetadata CompactionHistory = parse(COMPACTION_HISTORY, "week-long compaction history", @@ -256,14 +251,15 @@ public final class SystemKeyspace + "PRIMARY KEY ((keyspace_name), table_name, range_start, range_end))") .build(); - private static final TableMetadata AvailableRanges = - parse(AVAILABLE_RANGES, - "available keyspace/ranges during bootstrap/replace that are ready to be served", - "CREATE TABLE %s (" - + "keyspace_name text," - + "ranges set," - + "PRIMARY KEY ((keyspace_name)))") - .build(); + private static final TableMetadata AvailableRangesV2 = + parse(AVAILABLE_RANGES_V2, + "available keyspace/ranges during bootstrap/replace that are ready to be served", + "CREATE TABLE %s (" + + "keyspace_name text," + + "full_ranges set," + + "transient_ranges set," + + "PRIMARY KEY ((keyspace_name)))") + .build(); private static final TableMetadata TransferredRangesV2 = parse(TRANSFERRED_RANGES_V2, @@ -366,6 +362,16 @@ public final class SystemKeyspace + "PRIMARY KEY ((operation, keyspace_name), peer))") .build(); + @Deprecated + private static final TableMetadata LegacyAvailableRanges = + parse(LEGACY_AVAILABLE_RANGES, + "available keyspace/ranges during bootstrap/replace that are ready to be served", + "CREATE TABLE %s (" + + "keyspace_name text," + + "ranges set," + + "PRIMARY KEY ((keyspace_name)))") + .build(); + private static TableMetadata.Builder parse(String table, String description, String cql) { return CreateTableStatement.parse(format(cql, table), SchemaConstants.SYSTEM_KEYSPACE_NAME) @@ -390,11 +396,11 @@ public final class SystemKeyspace LegacyPeers, PeerEventsV2, LegacyPeerEvents, - RangeXfers, CompactionHistory, SSTableActivity, SizeEstimates, - AvailableRanges, + AvailableRangesV2, + LegacyAvailableRanges, TransferredRangesV2, LegacyTransferredRanges, ViewBuildsInProgress, @@ -1270,36 +1276,38 @@ public final class SystemKeyspace executeInternal(cql, keyspace, table); } - public static synchronized void updateAvailableRanges(String keyspace, Collection> completedRanges) + public static synchronized void updateAvailableRanges(String keyspace, Collection> completedFullRanges, Collection> completedTransientRanges) { - String cql = "UPDATE system.%s SET ranges = ranges + ? WHERE keyspace_name = ?"; - Set rangesToUpdate = new HashSet<>(completedRanges.size()); - for (Range range : completedRanges) - { - rangesToUpdate.add(rangeToBytes(range)); - } - executeInternal(format(cql, AVAILABLE_RANGES), rangesToUpdate, keyspace); + String cql = "UPDATE system.%s SET full_ranges = full_ranges + ?, transient_ranges = transient_ranges + ? WHERE keyspace_name = ?"; + executeInternal(format(cql, AVAILABLE_RANGES_V2), + completedFullRanges.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()), + completedTransientRanges.stream().map(SystemKeyspace::rangeToBytes).collect(Collectors.toSet()), + keyspace); } - public static synchronized Set> getAvailableRanges(String keyspace, IPartitioner partitioner) + public static synchronized RangesAtEndpoint getAvailableRanges(String keyspace, IPartitioner partitioner) { - Set> result = new HashSet<>(); String query = "SELECT * FROM system.%s WHERE keyspace_name=?"; - UntypedResultSet rs = executeInternal(format(query, AVAILABLE_RANGES), keyspace); + UntypedResultSet rs = executeInternal(format(query, AVAILABLE_RANGES_V2), keyspace); + InetAddressAndPort endpoint = InetAddressAndPort.getLocalHost(); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint); for (UntypedResultSet.Row row : rs) { - Set rawRanges = row.getSet("ranges", BytesType.instance); - for (ByteBuffer rawRange : rawRanges) - { - result.add(byteBufferToRange(rawRange, partitioner)); - } + Optional.ofNullable(row.getSet("full_ranges", BytesType.instance)) + .ifPresent(full_ranges -> full_ranges.stream() + .map(buf -> byteBufferToRange(buf, partitioner)) + .forEach(range -> builder.add(fullReplica(endpoint, range)))); + Optional.ofNullable(row.getSet("transient_ranges", BytesType.instance)) + .ifPresent(transient_ranges -> transient_ranges.stream() + .map(buf -> byteBufferToRange(buf, partitioner)) + .forEach(range -> builder.add(transientReplica(endpoint, range)))); } - return ImmutableSet.copyOf(result); + return builder.build(); } public static void resetAvailableRanges() { - ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES); + ColumnFamilyStore availableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(AVAILABLE_RANGES_V2); availableRanges.truncateBlocking(); } @@ -1405,7 +1413,13 @@ public final class SystemKeyspace return result.one().getString("release_version"); } - private static ByteBuffer rangeToBytes(Range range) + @VisibleForTesting + public static Set> rawRangesToRangeSet(Set rawRanges, IPartitioner partitioner) + { + return rawRanges.stream().map(buf -> byteBufferToRange(buf, partitioner)).collect(Collectors.toSet()); + } + + static ByteBuffer rangeToBytes(Range range) { try (DataOutputBuffer out = new DataOutputBuffer()) { diff --git a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java index ea5ff59b04..e0a58baf63 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java @@ -18,6 +18,11 @@ package org.apache.cassandra.db; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +50,9 @@ public class SystemKeyspaceMigrator40 static final String peerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEER_EVENTS_V2); static final String legacyTransferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_TRANSFERRED_RANGES); static final String transferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.TRANSFERRED_RANGES_V2); + static final String legacyAvailableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_AVAILABLE_RANGES); + static final String availableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.AVAILABLE_RANGES_V2); + private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator40.class); @@ -55,6 +63,7 @@ public class SystemKeyspaceMigrator40 migratePeers(); migratePeerEvents(); migrateTransferredRanges(); + migrateAvailableRanges(); } private static void migratePeers() @@ -181,4 +190,40 @@ public class SystemKeyspaceMigrator40 logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyTransferredRangesName, transferredRangesName); } + static void migrateAvailableRanges() + { + ColumnFamilyStore newAvailableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.AVAILABLE_RANGES_V2); + + if (!newAvailableRanges.isEmpty()) + return; + + logger.info("{} table was empty, migrating legacy {} to {}", availableRangesName, legacyAvailableRangesName, availableRangesName); + + String query = String.format("SELECT * FROM %s", + legacyAvailableRangesName); + + String insert = String.format("INSERT INTO %s (" + + "keyspace_name, " + + "full_ranges, " + + "transient_ranges) " + + " values ( ?, ?, ? )", + availableRangesName); + + UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000); + int transferred = 0; + for (UntypedResultSet.Row row : rows) + { + logger.debug("Transferring row {}", transferred); + String keyspace = row.getString("keyspace_name"); + Set ranges = Optional.ofNullable(row.getSet("ranges", BytesType.instance)).orElse(Collections.emptySet()); + QueryProcessor.executeInternal(insert, + keyspace, + ranges, + Collections.emptySet()); + transferred++; + } + + logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyAvailableRangesName, availableRangesName); + } + } diff --git a/src/java/org/apache/cassandra/db/TableCQLHelper.java b/src/java/org/apache/cassandra/db/TableCQLHelper.java index 550a6d62fe..f97bebc4f4 100644 --- a/src/java/org/apache/cassandra/db/TableCQLHelper.java +++ b/src/java/org/apache/cassandra/db/TableCQLHelper.java @@ -310,6 +310,7 @@ public class TableCQLHelper builder.append("\n\tAND max_index_interval = ").append(tableParams.maxIndexInterval); builder.append("\n\tAND memtable_flush_period_in_ms = ").append(tableParams.memtableFlushPeriodInMs); builder.append("\n\tAND speculative_retry = '").append(tableParams.speculativeRetry).append("'"); + builder.append("\n\tAND speculative_write_threshold = '").append(tableParams.speculativeWriteThreshold).append("'"); builder.append("\n\tAND comment = ").append(singleQuote(tableParams.comment)); builder.append("\n\tAND caching = ").append(toCQL(tableParams.caching.asMap())); builder.append("\n\tAND compaction = ").append(toCQL(tableParams.compaction.asMap())); diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index 59bdce66b6..28ea90a072 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -530,12 +530,13 @@ public abstract class AbstractCompactionStrategy long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, MetadataCollector meta, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { - return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, cfs.metadata, meta, header, indexes, txn); + return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, txn); } public boolean supportsEarlyOpen() diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java index dc16261abb..24bea06404 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java @@ -158,11 +158,11 @@ public abstract class AbstractStrategyHolder * groups they deal with. IOW, if one holder returns true for a given isRepaired/isPendingRepair combo, * none of the others should. */ - public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair); + public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient); public boolean managesSSTable(SSTableReader sstable) { - return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair()); + return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair(), sstable.isTransient()); } public abstract AbstractCompactionStrategy getStrategyFor(SSTableReader sstable); @@ -193,6 +193,7 @@ public abstract class AbstractStrategyHolder long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, MetadataCollector collector, SerializationHeader header, Collection indexes, @@ -203,4 +204,6 @@ public abstract class AbstractStrategyHolder * if it's not held by this holder */ public abstract int getStrategyIndex(AbstractCompactionStrategy strategy); + + public abstract boolean containsSSTable(SSTableReader sstable); } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index a872fea9e0..2a56650255 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -23,7 +23,7 @@ import java.lang.management.ManagementFactory; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.LongPredicate; +import java.util.function.Predicate; import java.util.stream.Collectors; import javax.management.MBeanServer; import javax.management.ObjectName; @@ -34,6 +34,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.*; import com.google.common.util.concurrent.*; + +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +46,6 @@ import org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.schema.Schema; @@ -71,7 +73,6 @@ import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.CompactionMetrics; import org.apache.cassandra.metrics.TableMetrics; -import org.apache.cassandra.repair.Validator; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; @@ -81,6 +82,8 @@ import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Refs; import static java.util.Collections.singleton; +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; /** *

@@ -509,7 +512,10 @@ public class CompactionManager implements CompactionManagerMBean return AllSSTableOpStatus.ABORTED; } // if local ranges is empty, it means no data should remain - final Collection> ranges = StorageService.instance.getLocalRanges(keyspace.getName()); + final RangesAtEndpoint replicas = StorageService.instance.getLocalReplicas(keyspace.getName()); + final Set> allRanges = replicas.ranges(); + final Set> transientRanges = replicas.filter(Replica::isTransient).ranges(); + final Set> fullRanges = replicas.filter(Replica::isFull).ranges(); final boolean hasIndexes = cfStore.indexManager.hasIndexes(); return parallelAllSSTableOperation(cfStore, new OneSSTableOperation() @@ -525,8 +531,8 @@ public class CompactionManager implements CompactionManagerMBean @Override public void execute(LifecycleTransaction txn) throws IOException { - CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, ranges, FBUtilities.nowInSeconds()); - doCleanupOne(cfStore, txn, cleanupStrategy, ranges, hasIndexes); + CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfStore, allRanges, transientRanges, txn.onlyOne().isRepaired(), FBUtilities.nowInSeconds()); + doCleanupOne(cfStore, txn, cleanupStrategy, replicas.ranges(), fullRanges, transientRanges, hasIndexes); } }, jobs, OperationType.CLEANUP); } @@ -574,9 +580,8 @@ public class CompactionManager implements CompactionManagerMBean logger.info("Partitioner does not support splitting"); return AllSSTableOpStatus.ABORTED; } - final Collection> r = StorageService.instance.getLocalRanges(cfs.keyspace.getName()); - if (r.isEmpty()) + if (StorageService.instance.getLocalReplicas(cfs.keyspace.getName()).isEmpty()) { logger.info("Relocate cannot run before a node has joined the ring"); return AllSSTableOpStatus.ABORTED; @@ -643,7 +648,11 @@ public class CompactionManager implements CompactionManagerMBean /** * Splits the given token ranges of the given sstables into a pending repair silo */ - public ListenableFuture submitPendingAntiCompaction(ColumnFamilyStore cfs, Collection> ranges, Refs sstables, LifecycleTransaction txn, UUID sessionId) + public ListenableFuture submitPendingAntiCompaction(ColumnFamilyStore cfs, + RangesAtEndpoint tokenRanges, + Refs sstables, + LifecycleTransaction txn, + UUID sessionId) { Runnable runnable = new WrappedRunnable() { @@ -651,7 +660,7 @@ public class CompactionManager implements CompactionManagerMBean { try (TableMetrics.TableTimer.Context ctx = cfs.metric.anticompactionTime.time()) { - performAnticompaction(cfs, ranges, sstables, txn, ActiveRepairService.UNREPAIRED_SSTABLE, sessionId, sessionId); + performAnticompaction(cfs, tokenRanges, sstables, txn, sessionId); } } }; @@ -672,49 +681,70 @@ public class CompactionManager implements CompactionManagerMBean } } + /** + * for sstables that are fully contained in the given ranges, just rewrite their metadata with + * the pending repair id and remove them from the transaction + */ + private static void mutateFullyContainedSSTables(ColumnFamilyStore cfs, + Refs refs, + Iterator sstableIterator, + Collection> ranges, + LifecycleTransaction txn, + UUID sessionID, + boolean isTransient) throws IOException + { + if (ranges.isEmpty()) + return; + + List> normalizedRanges = Range.normalize(ranges); + + Set fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID); + + cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables)); + cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient); + // since we're just re-writing the sstable metdata for the fully contained sstables, we don't want + // them obsoleted when the anti-compaction is complete. So they're removed from the transaction here + txn.cancel(fullyContainedSSTables); + refs.release(fullyContainedSSTables); + } + /** * Make sure the {validatedForRepair} are marked for compaction before calling this. * * Caller must reference the validatedForRepair sstables (via ParentRepairSession.getActiveRepairedSSTableRefs(..)). * * @param cfs - * @param ranges Ranges that the repair was carried out on + * @param ranges token ranges to be repaired * @param validatedForRepair SSTables containing the repaired ranges. Should be referenced before passing them. - * @param parentRepairSession parent repair session ID + * @param sessionID the repair session we're anti-compacting for * @throws InterruptedException * @throws IOException */ public void performAnticompaction(ColumnFamilyStore cfs, - Collection> ranges, + RangesAtEndpoint ranges, Refs validatedForRepair, LifecycleTransaction txn, - long repairedAt, - UUID pendingRepair, - UUID parentRepairSession) throws InterruptedException, IOException + UUID sessionID) throws IOException { try { - ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentRepairSession); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); Preconditions.checkArgument(!prs.isPreview(), "Cannot anticompact for previews"); + Preconditions.checkArgument(!ranges.isEmpty(), "No ranges to anti-compact"); if (logger.isInfoEnabled()) - logger.info("{} Starting anticompaction for {}.{} on {}/{} sstables", PreviewKind.NONE.logPrefix(parentRepairSession), cfs.keyspace.getName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables().size()); + logger.info("{} Starting anticompaction for {}.{} on {}/{} sstables", PreviewKind.NONE.logPrefix(sessionID), cfs.keyspace.getName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables().size()); if (logger.isTraceEnabled()) - logger.trace("{} Starting anticompaction for ranges {}", PreviewKind.NONE.logPrefix(parentRepairSession), ranges); + logger.trace("{} Starting anticompaction for ranges {}", PreviewKind.NONE.logPrefix(sessionID), ranges); + Set sstables = new HashSet<>(validatedForRepair); + validateSSTableBoundsForAnticompaction(sessionID, sstables, ranges); + mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), ranges.fullRanges(), txn, sessionID, false); + mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), ranges.transientRanges(), txn, sessionID, true); - Iterator sstableIterator = sstables.iterator(); - List> normalizedRanges = Range.normalize(ranges); - - Set fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, parentRepairSession); - - cfs.metric.bytesMutatedAnticompaction.inc(SSTableReader.getTotalBytes(fullyContainedSSTables)); - cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, repairedAt, pendingRepair); - txn.cancel(fullyContainedSSTables); - validatedForRepair.release(fullyContainedSSTables); assert txn.originals().equals(sstables); if (!sstables.isEmpty()) - doAntiCompaction(cfs, ranges, txn, repairedAt, pendingRepair); + doAntiCompaction(cfs, ranges, txn, sessionID); txn.finish(); } finally @@ -723,7 +753,28 @@ public class CompactionManager implements CompactionManagerMBean txn.close(); } - logger.info("{} Completed anticompaction successfully", PreviewKind.NONE.logPrefix(parentRepairSession)); + logger.info("{} Completed anticompaction successfully", PreviewKind.NONE.logPrefix(sessionID)); + } + + static void validateSSTableBoundsForAnticompaction(UUID sessionID, + Collection sstables, + RangesAtEndpoint ranges) + { + List> normalizedRanges = Range.normalize(ranges.ranges()); + for (SSTableReader sstable : sstables) + { + Bounds bounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken()); + + if (!Iterables.any(normalizedRanges, r -> (r.contains(bounds.left) && r.contains(bounds.right)) || r.intersects(bounds))) + { + // this should never happen - in PendingAntiCompaction#getSSTables we select all sstables that intersect the repaired ranges, that can't have changed here + String message = String.format("%s SSTable %s (%s) does not intersect repaired ranges %s, this sstable should not have been included.", + PreviewKind.NONE.logPrefix(sessionID), sstable, bounds, normalizedRanges); + logger.error(message); + throw new IllegalStateException(message); + } + } + } @VisibleForTesting @@ -736,8 +787,6 @@ public class CompactionManager implements CompactionManagerMBean Bounds sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken()); - boolean shouldAnticompact = false; - for (Range r : normalizedRanges) { // ranges are normalized - no wrap around - if first and last are contained we know that all tokens are contained in the range @@ -746,23 +795,13 @@ public class CompactionManager implements CompactionManagerMBean logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r); fullyContainedSSTables.add(sstable); sstableIterator.remove(); - shouldAnticompact = true; break; } else if (r.intersects(sstableBounds)) { logger.info("{} SSTable {} ({}) will be anticompacted on range {}", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, sstableBounds, r); - shouldAnticompact = true; } } - - if (!shouldAnticompact) - { - // this should never happen - in PendingAntiCompaction#getSSTables we select all sstables that intersect the repaired ranges, that can't have changed here - String message = String.format("%s SSTable %s (%s) does not intersect repaired ranges %s, this sstable should not have been included.", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, sstableBounds, normalizedRanges); - logger.error(message); - throw new IllegalStateException(message); - } } return fullyContainedSSTables; } @@ -914,7 +953,10 @@ public class CompactionManager implements CompactionManagerMBean { ColumnFamilyStore cfs = entry.getKey(); Keyspace keyspace = cfs.keyspace; - Collection> ranges = StorageService.instance.getLocalRanges(keyspace.getName()); + final RangesAtEndpoint replicas = StorageService.instance.getLocalReplicas(keyspace.getName()); + final Set> allRanges = replicas.ranges(); + final Set> transientRanges = replicas.filter(Replica::isTransient).ranges(); + final Set> fullRanges = replicas.filter(Replica::isFull).ranges(); boolean hasIndexes = cfs.indexManager.hasIndexes(); SSTableReader sstable = lookupSSTable(cfs, entry.getValue()); @@ -924,10 +966,10 @@ public class CompactionManager implements CompactionManagerMBean } else { - CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfs, ranges, FBUtilities.nowInSeconds()); + CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfs, allRanges, transientRanges, sstable.isRepaired(), FBUtilities.nowInSeconds()); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.CLEANUP)) { - doCleanupOne(cfs, txn, cleanupStrategy, ranges, hasIndexes); + doCleanupOne(cfs, txn, cleanupStrategy, allRanges, fullRanges, transientRanges, hasIndexes); } catch (IOException e) { @@ -1104,22 +1146,33 @@ public class CompactionManager implements CompactionManagerMBean * * @throws IOException */ - private void doCleanupOne(final ColumnFamilyStore cfs, LifecycleTransaction txn, CleanupStrategy cleanupStrategy, Collection> ranges, boolean hasIndexes) throws IOException + private void doCleanupOne(final ColumnFamilyStore cfs, + LifecycleTransaction txn, + CleanupStrategy cleanupStrategy, + Collection> allRanges, + Collection> fullRanges, + Collection> transientRanges, + boolean hasIndexes) throws IOException { assert !cfs.isIndex(); SSTableReader sstable = txn.onlyOne(); // if ranges is empty and no index, entire sstable is discarded - if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges)) + if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(allRanges)) { txn.obsoleteOriginals(); txn.finish(); return; } - if (!needsCleanup(sstable, ranges)) + + boolean needsCleanupFull = needsCleanup(sstable, fullRanges); + boolean needsCleanupTransient = needsCleanup(sstable, transientRanges); + //If there are no ranges for which the table needs cleanup either due to lack of intersection or lack + //of the table being repaired. + if (!needsCleanupFull && (!needsCleanupTransient || !sstable.isRepaired())) { - logger.trace("Skipping {} for cleanup; all rows should be kept", sstable); + logger.trace("Skipping {} for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}", sstable, needsCleanupFull, needsCleanupTransient, sstable.isRepaired()); return; } @@ -1150,7 +1203,7 @@ public class CompactionManager implements CompactionManagerMBean CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) { StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, txn)); + writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, txn)); long lastBytesScanned = 0; while (ci.hasNext()) @@ -1218,11 +1271,18 @@ public class CompactionManager implements CompactionManagerMBean this.nowInSec = nowInSec; } - public static CleanupStrategy get(ColumnFamilyStore cfs, Collection> ranges, int nowInSec) + public static CleanupStrategy get(ColumnFamilyStore cfs, Collection> ranges, Collection> transientRanges, boolean isRepaired, int nowInSec) { - return cfs.indexManager.hasIndexes() - ? new Full(cfs, ranges, nowInSec) - : new Bounded(cfs, ranges, nowInSec); + if (cfs.indexManager.hasIndexes()) + { + if (!transientRanges.isEmpty()) + { + //Shouldn't have been possible to create this situation + throw new AssertionError("Can't have indexes and transient ranges"); + } + return new Full(cfs, ranges, nowInSec); + } + return new Bounded(cfs, ranges, transientRanges, isRepaired, nowInSec); } public abstract ISSTableScanner getScanner(SSTableReader sstable); @@ -1230,7 +1290,10 @@ public class CompactionManager implements CompactionManagerMBean private static final class Bounded extends CleanupStrategy { - public Bounded(final ColumnFamilyStore cfs, Collection> ranges, int nowInSec) + private final Collection> transientRanges; + private final boolean isRepaired; + + public Bounded(final ColumnFamilyStore cfs, Collection> ranges, Collection> transientRanges, boolean isRepaired, int nowInSec) { super(ranges, nowInSec); cacheCleanupExecutor.submit(new Runnable() @@ -1241,12 +1304,23 @@ public class CompactionManager implements CompactionManagerMBean cfs.cleanupCache(); } }); + this.transientRanges = transientRanges; + this.isRepaired = isRepaired; } @Override public ISSTableScanner getScanner(SSTableReader sstable) { - return sstable.getScanner(ranges); + //If transient replication is enabled and there are transient ranges + //then cleanup should remove any partitions that are repaired and in the transient range + //as they should already be synchronized at other full replicas. + //So just don't scan the portion of the table containing the repaired transient ranges + Collection> rangesToScan = ranges; + if (isRepaired) + { + rangesToScan = Collections2.filter(ranges, range -> !transientRanges.contains(range)); + } + return sstable.getScanner(rangesToScan); } @Override @@ -1291,6 +1365,7 @@ public class CompactionManager implements CompactionManagerMBean long expectedBloomFilterSize, long repairedAt, UUID pendingRepair, + boolean isTransient, SSTableReader sstable, LifecycleTransaction txn) { @@ -1301,6 +1376,7 @@ public class CompactionManager implements CompactionManagerMBean expectedBloomFilterSize, repairedAt, pendingRepair, + isTransient, sstable.getSSTableLevel(), sstable.header, cfs.indexManager.listIndexes(), @@ -1312,6 +1388,7 @@ public class CompactionManager implements CompactionManagerMBean int expectedBloomFilterSize, long repairedAt, UUID pendingRepair, + boolean isTransient, Collection sstables, LifecycleTransaction txn) { @@ -1335,6 +1412,7 @@ public class CompactionManager implements CompactionManagerMBean (long) expectedBloomFilterSize, repairedAt, pendingRepair, + isTransient, cfs.metadata, new MetadataCollector(sstables, cfs.metadata().comparator, minLevel), SerializationHeader.make(cfs.metadata(), sstables), @@ -1347,16 +1425,19 @@ public class CompactionManager implements CompactionManagerMBean * will store the non-repaired ranges. Once anticompation is completed, the original sstable is marked as compacted * and subsequently deleted. * @param cfs - * @param repaired a transaction over the repaired sstables to anticompacy - * @param ranges Repaired ranges to be placed into one of the new sstables. The repaired table will be tracked via - * the {@link org.apache.cassandra.io.sstable.metadata.StatsMetadata#repairedAt} field. + * @param txn a transaction over the repaired sstables to anticompact + * @param ranges full and transient ranges to be placed into one of the new sstables. The repaired table will be tracked via + * the {@link org.apache.cassandra.io.sstable.metadata.StatsMetadata#pendingRepair} field. */ - private void doAntiCompaction(ColumnFamilyStore cfs, Collection> ranges, LifecycleTransaction repaired, long repairedAt, UUID pendingRepair) + private void doAntiCompaction(ColumnFamilyStore cfs, + RangesAtEndpoint ranges, + LifecycleTransaction txn, + UUID pendingRepair) { - logger.info("Performing anticompaction on {} sstables", repaired.originals().size()); + logger.info("Performing anticompaction on {} sstables", txn.originals().size()); //Group SSTables - Set sstables = repaired.originals(); + Set sstables = txn.originals(); // Repairs can take place on both unrepaired (incremental + full) and repaired (full) data. // Although anti-compaction could work on repaired sstables as well and would result in having more accurate @@ -1366,101 +1447,111 @@ public class CompactionManager implements CompactionManagerMBean cfs.metric.bytesAnticompacted.inc(SSTableReader.getTotalBytes(unrepairedSSTables)); Collection> groupedSSTables = cfs.getCompactionStrategyManager().groupSSTablesForAntiCompaction(unrepairedSSTables); - // iterate over sstables to check if the repaired / unrepaired ranges intersect them. + // iterate over sstables to check if the full / transient / unrepaired ranges intersect them. int antiCompactedSSTableCount = 0; for (Collection sstableGroup : groupedSSTables) { - try (LifecycleTransaction txn = repaired.split(sstableGroup)) + try (LifecycleTransaction groupTxn = txn.split(sstableGroup)) { - int antiCompacted = antiCompactGroup(cfs, ranges, txn, repairedAt, pendingRepair); + int antiCompacted = antiCompactGroup(cfs, ranges, groupTxn, pendingRepair); antiCompactedSSTableCount += antiCompacted; } } String format = "Anticompaction completed successfully, anticompacted from {} to {} sstable(s)."; - logger.info(format, repaired.originals().size(), antiCompactedSSTableCount); + logger.info(format, txn.originals().size(), antiCompactedSSTableCount); } - private int antiCompactGroup(ColumnFamilyStore cfs, Collection> ranges, - LifecycleTransaction anticompactionGroup, long repairedAt, UUID pendingRepair) + private int antiCompactGroup(ColumnFamilyStore cfs, + RangesAtEndpoint ranges, + LifecycleTransaction txn, + UUID pendingRepair) { + Preconditions.checkArgument(!ranges.isEmpty(), "need at least one full or transient range"); long groupMaxDataAge = -1; - for (Iterator i = anticompactionGroup.originals().iterator(); i.hasNext();) + for (Iterator i = txn.originals().iterator(); i.hasNext();) { SSTableReader sstable = i.next(); if (groupMaxDataAge < sstable.maxDataAge) groupMaxDataAge = sstable.maxDataAge; } - if (anticompactionGroup.originals().size() == 0) + if (txn.originals().size() == 0) { logger.info("No valid anticompactions for this group, All sstables were compacted and are no longer available"); return 0; } - logger.info("Anticompacting {}", anticompactionGroup); - Set sstableAsSet = anticompactionGroup.originals(); + logger.info("Anticompacting {}", txn); + Set sstableAsSet = txn.originals(); File destination = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(sstableAsSet, OperationType.ANTICOMPACTION)); - long repairedKeyCount = 0; - long unrepairedKeyCount = 0; int nowInSec = FBUtilities.nowInSeconds(); CompactionStrategyManager strategy = cfs.getCompactionStrategyManager(); - try (SSTableRewriter repairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(anticompactionGroup, false, groupMaxDataAge); - SSTableRewriter unRepairedSSTableWriter = SSTableRewriter.constructWithoutEarlyOpening(anticompactionGroup, false, groupMaxDataAge); - AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(anticompactionGroup.originals()); + try (SSTableRewriter fullWriter = SSTableRewriter.constructWithoutEarlyOpening(txn, false, groupMaxDataAge); + SSTableRewriter transWriter = SSTableRewriter.constructWithoutEarlyOpening(txn, false, groupMaxDataAge); + SSTableRewriter unrepairedWriter = SSTableRewriter.constructWithoutEarlyOpening(txn, false, groupMaxDataAge); + + AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(txn.originals()); CompactionController controller = new CompactionController(cfs, sstableAsSet, getDefaultGcBefore(cfs, nowInSec)); CompactionIterator ci = new CompactionIterator(OperationType.ANTICOMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID(), metrics)) { int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet))); - repairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, repairedAt, pendingRepair, sstableAsSet, anticompactionGroup)); - unRepairedSSTableWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, ActiveRepairService.UNREPAIRED_SSTABLE, null, sstableAsSet, anticompactionGroup)); - Range.OrderedRangeContainmentChecker containmentChecker = new Range.OrderedRangeContainmentChecker(ranges); + fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, false, sstableAsSet, txn)); + transWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, true, sstableAsSet, txn)); + unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, sstableAsSet, txn)); + + Predicate fullChecker = !ranges.fullRanges().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.fullRanges()) : t -> false; + Predicate transChecker = !ranges.transientRanges().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.transientRanges()) : t -> false; while (ci.hasNext()) { try (UnfilteredRowIterator partition = ci.next()) { - // if current range from sstable is repaired, save it into the new repaired sstable - if (containmentChecker.contains(partition.partitionKey().getToken())) + Token token = partition.partitionKey().getToken(); + // if this row is contained in the full or transient ranges, append it to the appropriate sstable + if (fullChecker.test(token)) { - repairedSSTableWriter.append(partition); - repairedKeyCount++; + fullWriter.append(partition); + } + else if (transChecker.test(token)) + { + transWriter.append(partition); } - // otherwise save into the new 'non-repaired' table else { - unRepairedSSTableWriter.append(partition); - unrepairedKeyCount++; + // otherwise, append it to the unrepaired sstable + unrepairedWriter.append(partition); } } } List anticompactedSSTables = new ArrayList<>(); - // since both writers are operating over the same Transaction, we cannot use the convenience Transactional.finish() method, + // since all writers are operating over the same Transaction, we cannot use the convenience Transactional.finish() method, // as on the second finish() we would prepareToCommit() on a Transaction that has already been committed, which is forbidden by the API // (since it indicates misuse). We call permitRedundantTransitions so that calls that transition to a state already occupied are permitted. - anticompactionGroup.permitRedundantTransitions(); - repairedSSTableWriter.setRepairedAt(repairedAt).prepareToCommit(); - unRepairedSSTableWriter.prepareToCommit(); - anticompactedSSTables.addAll(repairedSSTableWriter.finished()); - anticompactedSSTables.addAll(unRepairedSSTableWriter.finished()); - repairedSSTableWriter.commit(); - unRepairedSSTableWriter.commit(); + txn.permitRedundantTransitions(); + + fullWriter.prepareToCommit(); + transWriter.prepareToCommit(); + unrepairedWriter.prepareToCommit(); + + anticompactedSSTables.addAll(fullWriter.finished()); + anticompactedSSTables.addAll(transWriter.finished()); + anticompactedSSTables.addAll(unrepairedWriter.finished()); + + fullWriter.commit(); + transWriter.commit(); + unrepairedWriter.commit(); - logger.trace("Repaired {} keys out of {} for {}/{} in {}", repairedKeyCount, - repairedKeyCount + unrepairedKeyCount, - cfs.keyspace.getName(), - cfs.getTableName(), - anticompactionGroup); return anticompactedSSTables.size(); } catch (Throwable e) { JVMStabilityInspector.inspectThrowable(e); - logger.error("Error anticompacting " + anticompactionGroup, e); + logger.error("Error anticompacting " + txn, e); } return 0; } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java index 8fba121654..8ce93fa88c 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.UUID; import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.SerializationHeader; @@ -71,11 +72,19 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder } @Override - public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair) + public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient) { - Preconditions.checkArgument(!isPendingRepair || !isRepaired, - "SSTables cannot be both repaired and pending repair"); - return !isPendingRepair && (isRepaired == this.isRepaired); + if (!isPendingRepair) + { + Preconditions.checkArgument(!isTransient, "isTransient can only be true for sstables pending repairs"); + return this.isRepaired == isRepaired; + } + else + { + Preconditions.checkArgument(!isRepaired, "SSTables cannot be both repaired and pending repair"); + return false; + + } } @Override @@ -206,7 +215,15 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder } @Override - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, MetadataCollector collector, SerializationHeader header, Collection indexes, LifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, + long keyCount, + long repairedAt, + UUID pendingRepair, + boolean isTransient, + MetadataCollector collector, + SerializationHeader header, + Collection indexes, + LifecycleTransaction txn) { if (isRepaired) { @@ -226,6 +243,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder keyCount, repairedAt, pendingRepair, + isTransient, collector, header, indexes, @@ -237,4 +255,10 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder { return strategies.indexOf(strategy); } + + @Override + public boolean containsSSTable(SSTableReader sstable) + { + return Iterables.any(strategies, acs -> acs.getSSTables().contains(sstable)); + } } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index 9766454055..afe628b82c 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -56,6 +56,7 @@ import org.apache.cassandra.index.Index; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -112,6 +113,7 @@ public class CompactionStrategyManager implements INotificationConsumer /** * Variables guarded by read and write lock above */ + private final PendingRepairHolder transientRepairs; private final PendingRepairHolder pendingRepairs; private final CompactionStrategyHolder repaired; private final CompactionStrategyHolder unrepaired; @@ -156,10 +158,11 @@ public class CompactionStrategyManager implements INotificationConsumer return compactionStrategyIndexForDirectory(descriptor); } }; - pendingRepairs = new PendingRepairHolder(cfs, router); + transientRepairs = new PendingRepairHolder(cfs, router, true); + pendingRepairs = new PendingRepairHolder(cfs, router, false); repaired = new CompactionStrategyHolder(cfs, router, true); unrepaired = new CompactionStrategyHolder(cfs, router, false); - holders = ImmutableList.of(pendingRepairs, repaired, unrepaired); + holders = ImmutableList.of(transientRepairs, pendingRepairs, repaired, unrepaired); cfs.getTracker().subscribe(this); logger.trace("{} subscribed to the data tracker.", this); @@ -176,7 +179,6 @@ public class CompactionStrategyManager implements INotificationConsumer * Return the next background task * * Returns a task for the compaction strategy that needs it the most (most estimated remaining tasks) - * */ public AbstractCompactionTask getNextBackgroundTask(int gcBefore) { @@ -188,18 +190,16 @@ public class CompactionStrategyManager implements INotificationConsumer return null; int numPartitions = getNumTokenPartitions(); + // first try to promote/demote sstables from completed repairs - List repairFinishedSuppliers = pendingRepairs.getRepairFinishedTaskSuppliers(); - if (!repairFinishedSuppliers.isEmpty()) - { - Collections.sort(repairFinishedSuppliers); - for (TaskSupplier supplier : repairFinishedSuppliers) - { - AbstractCompactionTask task = supplier.getTask(); - if (task != null) - return task; - } - } + AbstractCompactionTask repairFinishedTask; + repairFinishedTask = pendingRepairs.getNextRepairFinishedTask(); + if (repairFinishedTask != null) + return repairFinishedTask; + + repairFinishedTask = transientRepairs.getNextRepairFinishedTask(); + if (repairFinishedTask != null) + return repairFinishedTask; // sort compaction task suppliers by remaining tasks descending List suppliers = new ArrayList<>(numPartitions * holders.size()); @@ -393,64 +393,28 @@ public class CompactionStrategyManager implements INotificationConsumer } } - - @VisibleForTesting - List getRepaired() + CompactionStrategyHolder getRepairedUnsafe() { - readLock.lock(); - try - { - return Lists.newArrayList(repaired.allStrategies()); - } - finally - { - readLock.unlock(); - } + return repaired; } @VisibleForTesting - List getUnrepaired() + CompactionStrategyHolder getUnrepairedUnsafe() { - readLock.lock(); - try - { - return Lists.newArrayList(unrepaired.allStrategies()); - } - finally - { - readLock.unlock(); - } + return unrepaired; } @VisibleForTesting - Iterable getForPendingRepair(UUID sessionID) + PendingRepairHolder getPendingRepairsUnsafe() { - readLock.lock(); - try - { - return pendingRepairs.getStrategiesFor(sessionID); - } - finally - { - readLock.unlock(); - } + return pendingRepairs; } @VisibleForTesting - Set pendingRepairs() + PendingRepairHolder getTransientRepairsUnsafe() { - readLock.lock(); - try - { - Set ids = new HashSet<>(); - pendingRepairs.getManagers().forEach(p -> ids.addAll(p.getSessions())); - return ids; - } - finally - { - readLock.unlock(); - } + return transientRepairs; } public boolean hasDataForPendingRepair(UUID sessionID) @@ -458,8 +422,7 @@ public class CompactionStrategyManager implements INotificationConsumer readLock.lock(); try { - return Iterables.any(pendingRepairs.getManagers(), - prm -> prm.hasDataForSession(sessionID)); + return pendingRepairs.hasDataForSession(sessionID) || transientRepairs.hasDataForSession(sessionID); } finally { @@ -682,18 +645,19 @@ public class CompactionStrategyManager implements INotificationConsumer throw new IllegalStateException("No holder claimed " + sstable); } - private AbstractStrategyHolder getHolder(long repairedAt, UUID pendingRepair) + private AbstractStrategyHolder getHolder(long repairedAt, UUID pendingRepair, boolean isTransient) { return getHolder(repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE, - pendingRepair != ActiveRepairService.NO_PENDING_REPAIR); + pendingRepair != ActiveRepairService.NO_PENDING_REPAIR, + isTransient); } @VisibleForTesting - AbstractStrategyHolder getHolder(boolean isRepaired, boolean isPendingRepair) + AbstractStrategyHolder getHolder(boolean isRepaired, boolean isPendingRepair, boolean isTransient) { for (AbstractStrategyHolder holder : holders) { - if (holder.managesRepairedGroup(isRepaired, isPendingRepair)) + if (holder.managesRepairedGroup(isRepaired, isPendingRepair, isTransient)) return holder; } @@ -1146,16 +1110,26 @@ public class CompactionStrategyManager implements INotificationConsumer long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, MetadataCollector collector, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { + SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient); maybeReloadDiskBoundaries(); readLock.lock(); try { - return getHolder(repairedAt, pendingRepair).createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, collector, header, indexes, txn); + return getHolder(repairedAt, pendingRepair, isTransient).createSSTableMultiWriter(descriptor, + keyCount, + repairedAt, + pendingRepair, + isTransient, + collector, + header, + indexes, + txn); } finally { @@ -1220,7 +1194,7 @@ public class CompactionStrategyManager implements INotificationConsumer * Mutates sstable repairedAt times and notifies listeners of the change with the writeLock held. Prevents races * with other processes between when the metadata is changed and when sstables are moved between strategies. */ - public void mutateRepaired(Collection sstables, long repairedAt, UUID pendingRepair) throws IOException + public void mutateRepaired(Collection sstables, long repairedAt, UUID pendingRepair, boolean isTransient) throws IOException { Set changed = new HashSet<>(); @@ -1229,7 +1203,7 @@ public class CompactionStrategyManager implements INotificationConsumer { for (SSTableReader sstable: sstables) { - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, repairedAt, pendingRepair); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient); sstable.reloadSSTableMetadata(); changed.add(sstable); } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 662384c4ac..591b7c4a48 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -29,20 +29,19 @@ import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.util.concurrent.RateLimiter; - -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter; -import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionManager.CompactionExecutorStatsCollector; +import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter; +import org.apache.cassandra.db.compaction.writers.DefaultCompactionWriter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.FBUtilities; @@ -339,6 +338,23 @@ public class CompactionTask extends AbstractCompactionTask return ids.iterator().next(); } + public static boolean getIsTransient(Set sstables) + { + if (sstables.isEmpty()) + { + return false; + } + + boolean isTransient = sstables.iterator().next().isTransient(); + + if (!Iterables.all(sstables, sstable -> sstable.isTransient() == isTransient)) + { + throw new RuntimeException("Attempting to compact transient sstables with non transient sstables"); + } + + return isTransient; + } + /* * Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java index 7b9123f687..92e44a7b51 100644 --- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -43,10 +44,12 @@ import org.apache.cassandra.service.ActiveRepairService; public class PendingRepairHolder extends AbstractStrategyHolder { private final List managers = new ArrayList<>(); + private final boolean isTransient; - public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router) + public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isTransient) { super(cfs, router); + this.isTransient = isTransient; } @Override @@ -66,15 +69,15 @@ public class PendingRepairHolder extends AbstractStrategyHolder { managers.clear(); for (int i = 0; i < numTokenPartitions; i++) - managers.add(new PendingRepairManager(cfs, params)); + managers.add(new PendingRepairManager(cfs, params, isTransient)); } @Override - public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair) + public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient) { Preconditions.checkArgument(!isPendingRepair || !isRepaired, "SSTables cannot be both repaired and pending repair"); - return isPendingRepair; + return isPendingRepair && (this.isTransient == isTransient); } @Override @@ -145,7 +148,23 @@ public class PendingRepairHolder extends AbstractStrategyHolder return tasks; } - public ArrayList getRepairFinishedTaskSuppliers() + AbstractCompactionTask getNextRepairFinishedTask() + { + List repairFinishedSuppliers = getRepairFinishedTaskSuppliers(); + if (!repairFinishedSuppliers.isEmpty()) + { + Collections.sort(repairFinishedSuppliers); + for (TaskSupplier supplier : repairFinishedSuppliers) + { + AbstractCompactionTask task = supplier.getTask(); + if (task != null) + return task; + } + } + return null; + } + + private ArrayList getRepairFinishedTaskSuppliers() { ArrayList suppliers = new ArrayList<>(managers.size()); for (PendingRepairManager manager : managers) @@ -218,6 +237,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, MetadataCollector collector, SerializationHeader header, Collection indexes, @@ -233,6 +253,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder keyCount, repairedAt, pendingRepair, + isTransient, collector, header, indexes, @@ -249,4 +270,15 @@ public class PendingRepairHolder extends AbstractStrategyHolder } return -1; } + + public boolean hasDataForSession(UUID sessionID) + { + return Iterables.any(managers, prm -> prm.hasDataForSession(sessionID)); + } + + @Override + public boolean containsSSTable(SSTableReader sstable) + { + return Iterables.any(managers, prm -> prm.containsSSTable(sstable)); + } } diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java index edc9a2f8ad..6763abfb60 100644 --- a/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java +++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java @@ -30,7 +30,9 @@ import java.util.UUID; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.slf4j.Logger; @@ -62,6 +64,7 @@ class PendingRepairManager private final ColumnFamilyStore cfs; private final CompactionParams params; + private final boolean isTransient; private volatile ImmutableMap strategies = ImmutableMap.of(); /** @@ -75,10 +78,11 @@ class PendingRepairManager } } - PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params) + PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params, boolean isTransient) { this.cfs = cfs; this.params = params; + this.isTransient = isTransient; } private ImmutableMap.Builder mapBuilder() @@ -156,6 +160,7 @@ class PendingRepairManager synchronized void addSSTable(SSTableReader sstable) { + Preconditions.checkArgument(sstable.isTransient() == isTransient); getOrCreate(sstable).addSSTable(sstable); } @@ -389,6 +394,15 @@ class PendingRepairManager return strategies.keySet().contains(sessionID); } + boolean containsSSTable(SSTableReader sstable) + { + if (!sstable.isPendingRepair()) + return false; + + AbstractCompactionStrategy strategy = strategies.get(sstable.getPendingRepair()); + return strategy != null && strategy.getSSTables().contains(sstable); + } + public Collection createUserDefinedTasks(Collection sstables, int gcBefore) { Map> group = sstables.stream().collect(Collectors.groupingBy(s -> s.getSSTableMetadata().pendingRepair)); @@ -419,18 +433,35 @@ class PendingRepairManager protected void runMayThrow() throws Exception { boolean completed = false; + boolean obsoleteSSTables = isTransient && repairedAt > 0; try { - logger.debug("Setting repairedAt to {} on {} for {}", repairedAt, transaction.originals(), sessionID); - cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR); + if (obsoleteSSTables) + { + logger.info("Obsoleting transient repaired ssatbles"); + Preconditions.checkState(Iterables.all(transaction.originals(), SSTableReader::isTransient)); + transaction.obsoleteOriginals(); + } + else + { + logger.debug("Setting repairedAt to {} on {} for {}", repairedAt, transaction.originals(), sessionID); + cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false); + } completed = true; } finally { - // we always abort because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll - // anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other - // compactions from marking these sstables compacting, and unmarking them when we're done - transaction.abort(); + if (obsoleteSSTables) + { + transaction.finish(); + } + else + { + // we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll + // anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other + // compactions from marking these sstables compacting, and unmarking them when we're done + transaction.abort(); + } if (completed) { removeSession(sessionID); diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index f97b69304f..aa41051e72 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -170,7 +170,7 @@ public class Scrubber implements Closeable } StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, sstable, transaction)); + writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)); DecoratedKey prevKey = null; @@ -277,7 +277,7 @@ public class Scrubber implements Closeable // out of order rows, but no bad rows found - we can keep our repairedAt time long repairedAt = badRows > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : metadata.repairedAt; SSTableReader newInOrderSstable; - try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, sstable, transaction)) + try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)) { for (Partition partition : outOfOrder) inOrderWriter.append(partition.unfilteredIterator()); diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java index 80453ef1d7..e1406aa7ed 100644 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@ -68,14 +68,15 @@ public class Upgrader this.estimatedRows = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables); } - private SSTableWriter createCompactionWriter(long repairedAt, UUID parentRepair) + private SSTableWriter createCompactionWriter(StatsMetadata metadata) { MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator()); sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel()); return SSTableWriter.create(cfs.newSSTableDescriptor(directory), estimatedRows, - repairedAt, - parentRepair, + metadata.repairedAt, + metadata.pendingRepair, + metadata.isTransient, cfs.metadata, sstableMetadataCollector, SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)), @@ -91,8 +92,7 @@ public class Upgrader AbstractCompactionStrategy.ScannerList scanners = strategyManager.getScanners(transaction.originals()); CompactionIterator iter = new CompactionIterator(transaction.opType(), scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())) { - StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(createCompactionWriter(metadata.repairedAt, metadata.pendingRepair)); + writer.switchWriter(createCompactionWriter(sstable.getSSTableMetadata())); while (iter.hasNext()) writer.append(iter.next()); diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java index db49369e5d..446d527bb1 100644 --- a/src/java/org/apache/cassandra/db/compaction/Verifier.java +++ b/src/java/org/apache/cassandra/db/compaction/Verifier.java @@ -350,6 +350,7 @@ public class Verifier implements Closeable public RangeOwnHelper(List> normalizedRanges) { this.normalizedRanges = normalizedRanges; + Range.assertNormalized(normalizedRanges); } /** @@ -457,7 +458,7 @@ public class Verifier implements Closeable { try { - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().pendingRepair); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient()); sstable.reloadSSTableMetadata(); cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); } diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java index 5ddd99c7b7..d72b236962 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java @@ -57,6 +57,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa protected final long maxAge; protected final long minRepairedAt; protected final UUID pendingRepair; + protected final boolean isTransient; protected final SSTableRewriter sstableWriter; protected final LifecycleTransaction txn; @@ -91,6 +92,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa sstableWriter = SSTableRewriter.construct(cfs, txn, keepOriginals, maxAge); minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables); pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables); + isTransient = CompactionTask.getIsTransient(nonExpiredSSTables); DiskBoundaries db = cfs.getDiskBoundaries(); diskBoundaries = db.positions; locations = db.directories; diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java index cda7e386d7..6180f96100 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java @@ -72,6 +72,7 @@ public class DefaultCompactionWriter extends CompactionAwareWriter estimatedTotalKeys, minRepairedAt, pendingRepair, + isTransient, cfs.metadata, new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel), SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java index 3959b4bc78..2b93eb47b8 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java @@ -108,6 +108,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter keysPerSSTable, minRepairedAt, pendingRepair, + isTransient, cfs.metadata, new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel), SerializationHeader.make(cfs.metadata(), txn.originals()), diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java index c4f84e86f7..df7eeaf12d 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java @@ -111,6 +111,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter estimatedTotalKeys / estimatedSSTables, minRepairedAt, pendingRepair, + isTransient, cfs.metadata, new MetadataCollector(allSSTables, cfs.metadata().comparator, level), SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java index a4af7839df..7533f1d8a2 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java @@ -107,6 +107,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter currentPartitionsToWrite, minRepairedAt, pendingRepair, + isTransient, cfs.metadata, new MetadataCollector(allSSTables, cfs.metadata().comparator, 0), SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java index 9064b0f75e..bed0958b8e 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java @@ -82,18 +82,6 @@ public abstract class PartitionIterators return new SingletonPartitionIterator(iterator); } - public static void consume(PartitionIterator iterator) - { - while (iterator.hasNext()) - { - try (RowIterator partition = iterator.next()) - { - while (partition.hasNext()) - partition.next(); - } - } - } - /** * Wraps the provided iterator so it logs the returned rows for debugging purposes. *

diff --git a/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java index 5f2e5a0767..fa2e65351b 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraKeyspaceRepairManager.java @@ -26,8 +26,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.repair.KeyspaceRepairManager; public class CassandraKeyspaceRepairManager implements KeyspaceRepairManager @@ -40,9 +39,12 @@ public class CassandraKeyspaceRepairManager implements KeyspaceRepairManager } @Override - public ListenableFuture prepareIncrementalRepair(UUID sessionID, Collection tables, Collection> ranges, ExecutorService executor) + public ListenableFuture prepareIncrementalRepair(UUID sessionID, + Collection tables, + RangesAtEndpoint tokenRanges, + ExecutorService executor) { - PendingAntiCompaction pac = new PendingAntiCompaction(sessionID, tables, ranges, executor); + PendingAntiCompaction pac = new PendingAntiCompaction(sessionID, tables, tokenRanges, executor); return pac.run(); } } diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java index 4e0f13d69c..a205c3cc36 100644 --- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java +++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java @@ -43,6 +43,7 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.utils.concurrent.Refs; /** @@ -126,17 +127,17 @@ public class PendingAntiCompaction static class AcquisitionCallback implements AsyncFunction, Object> { private final UUID parentRepairSession; - private final Collection> ranges; + private final RangesAtEndpoint tokenRanges; - public AcquisitionCallback(UUID parentRepairSession, Collection> ranges) + public AcquisitionCallback(UUID parentRepairSession, RangesAtEndpoint tokenRanges) { this.parentRepairSession = parentRepairSession; - this.ranges = ranges; + this.tokenRanges = tokenRanges; } ListenableFuture submitPendingAntiCompaction(AcquireResult result) { - return CompactionManager.instance.submitPendingAntiCompaction(result.cfs, ranges, result.refs, result.txn, parentRepairSession); + return CompactionManager.instance.submitPendingAntiCompaction(result.cfs, tokenRanges, result.refs, result.txn, parentRepairSession); } public ListenableFuture apply(List results) throws Exception @@ -177,14 +178,17 @@ public class PendingAntiCompaction private final UUID prsId; private final Collection tables; - private final Collection> ranges; + private final RangesAtEndpoint tokenRanges; private final ExecutorService executor; - public PendingAntiCompaction(UUID prsId, Collection tables, Collection> ranges, ExecutorService executor) + public PendingAntiCompaction(UUID prsId, + Collection tables, + RangesAtEndpoint tokenRanges, + ExecutorService executor) { this.prsId = prsId; this.tables = tables; - this.ranges = ranges; + this.tokenRanges = tokenRanges; this.executor = executor; } @@ -194,12 +198,12 @@ public class PendingAntiCompaction for (ColumnFamilyStore cfs : tables) { cfs.forceBlockingFlush(); - ListenableFutureTask task = ListenableFutureTask.create(new AcquisitionCallable(cfs, ranges, prsId)); + ListenableFutureTask task = ListenableFutureTask.create(new AcquisitionCallable(cfs, tokenRanges.ranges(), prsId)); executor.submit(task); tasks.add(task); } ListenableFuture> acquisitionResults = Futures.successfulAsList(tasks); - ListenableFuture compactionResult = Futures.transformAsync(acquisitionResults, new AcquisitionCallback(prsId, ranges), MoreExecutors.directExecutor()); + ListenableFuture compactionResult = Futures.transformAsync(acquisitionResults, new AcquisitionCallback(prsId, tokenRanges), MoreExecutors.directExecutor()); return compactionResult; } } diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index 5252187e5f..c688fdf7f3 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -68,17 +68,18 @@ public class CassandraOutgoingFile implements OutgoingStream private final ComponentManifest manifest; private Boolean isFullyContained; - private final List> ranges; + private final List> normalizedRanges; public CassandraOutgoingFile(StreamOperation operation, Ref ref, - List sections, Collection> ranges, + List sections, List> normalizedRanges, long estimatedKeys) { Preconditions.checkNotNull(ref.get()); + Range.assertNormalized(normalizedRanges); this.ref = ref; this.estimatedKeys = estimatedKeys; this.sections = sections; - this.ranges = ImmutableList.copyOf(ranges); + this.normalizedRanges = ImmutableList.copyOf(normalizedRanges); this.filename = ref.get().getFilename(); this.manifest = getComponentManifest(ref.get()); @@ -194,7 +195,7 @@ public class CassandraOutgoingFile implements OutgoingStream .getCompactionStrategyFor(ref.get()); if (compactionStrategy instanceof LeveledCompactionStrategy) - return contained(ranges, ref.get()); + return contained(normalizedRanges, ref.get()); return false; } @@ -251,6 +252,6 @@ public class CassandraOutgoingFile implements OutgoingStream @Override public String toString() { - return "CassandraOutgoingFile{" + ref.get().getFilename() + '}'; + return "CassandraOutgoingFile{" + filename + '}'; } } diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java index 43667d0610..6c2631c828 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java @@ -18,19 +18,10 @@ package org.apache.cassandra.db.streaming; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.UUID; - import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.lifecycle.SSTableIntervalTree; @@ -39,6 +30,8 @@ import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.IncomingStream; import org.apache.cassandra.streaming.OutgoingStream; @@ -49,6 +42,14 @@ import org.apache.cassandra.streaming.TableStreamManager; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.Refs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; /** * Implements the streaming interface for the native cassandra storage engine. @@ -96,14 +97,14 @@ public class CassandraStreamManager implements TableStreamManager } @Override - public Collection createOutgoingStreams(StreamSession session, Collection> ranges, UUID pendingRepair, PreviewKind previewKind) + public Collection createOutgoingStreams(StreamSession session, RangesAtEndpoint replicas, UUID pendingRepair, PreviewKind previewKind) { Refs refs = new Refs<>(); try { - final List> keyRanges = new ArrayList<>(ranges.size()); - for (Range range : ranges) - keyRanges.add(Range.makeRowRange(range)); + final List> keyRanges = new ArrayList<>(replicas.size()); + for (Replica replica : replicas) + keyRanges.add(Range.makeRowRange(replica.range())); refs.addAll(cfs.selectAndReference(view -> { Set sstables = Sets.newHashSet(); SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL)); @@ -141,11 +142,16 @@ public class CassandraStreamManager implements TableStreamManager }).refs); + List> normalizedFullRanges = Range.normalize(replicas.filter(Replica::isFull).ranges()); + List> normalizedAllRanges = Range.normalize(replicas.ranges()); + //Create outgoing file streams for ranges possibly skipping repaired ranges in sstables List streams = new ArrayList<>(refs.size()); - for (SSTableReader sstable: refs) + for (SSTableReader sstable : refs) { - Ref ref = refs.get(sstable); + List> ranges = sstable.isRepaired() ? normalizedFullRanges : normalizedAllRanges; List sections = sstable.getPositionsForRanges(ranges); + + Ref ref = refs.get(sstable); if (sections.isEmpty()) { ref.release(); diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index fccabfe08c..572c6482ae 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -156,7 +156,7 @@ public class CassandraStreamReader implements IStreamReader Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver); LifecycleTransaction txn = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).getTransaction(); - RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); + RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); return writer; } diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java index d35457edeb..09490e8bd7 100644 --- a/src/java/org/apache/cassandra/db/view/TableViews.java +++ b/src/java/org/apache/cassandra/db/view/TableViews.java @@ -60,6 +60,11 @@ public class TableViews extends AbstractCollection baseTableMetadata = Schema.instance.getTableMetadataRef(id); } + public boolean hasViews() + { + return !views.isEmpty(); + } + public int size() { return views.size(); diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilder.java b/src/java/org/apache/cassandra/db/view/ViewBuilder.java index c727f63e0b..67172973ee 100644 --- a/src/java/org/apache/cassandra/db/view/ViewBuilder.java +++ b/src/java/org/apache/cassandra/db/view/ViewBuilder.java @@ -43,6 +43,8 @@ import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replicas; import org.apache.cassandra.repair.SystemDistributedKeyspace; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; @@ -135,14 +137,15 @@ class ViewBuilder } // Get the local ranges for which the view hasn't already been built nor it's building - Set> newRanges = StorageService.instance.getLocalRanges(ksName) - .stream() - .map(r -> r.subtractAll(builtRanges)) - .flatMap(Set::stream) - .map(r -> r.subtractAll(pendingRanges.keySet())) - .flatMap(Set::stream) - .collect(Collectors.toSet()); - + RangesAtEndpoint replicatedRanges = StorageService.instance.getLocalReplicas(ksName); + Replicas.temporaryAssertFull(replicatedRanges); + Set> newRanges = replicatedRanges.ranges() + .stream() + .map(r -> r.subtractAll(builtRanges)) + .flatMap(Set::stream) + .map(r -> r.subtractAll(pendingRanges.keySet())) + .flatMap(Set::stream) + .collect(Collectors.toSet()); // If there are no new nor pending ranges we should finish the build if (newRanges.isEmpty() && pendingRanges.isEmpty()) { diff --git a/src/java/org/apache/cassandra/db/view/ViewManager.java b/src/java/org/apache/cassandra/db/view/ViewManager.java index 000477df7c..7e3ea1bcea 100644 --- a/src/java/org/apache/cassandra/db/view/ViewManager.java +++ b/src/java/org/apache/cassandra/db/view/ViewManager.java @@ -79,7 +79,7 @@ public class ViewManager { assert keyspace.getName().equals(update.metadata().keyspace); - if (coordinatorBatchlog && keyspace.getReplicationStrategy().getReplicationFactor() == 1) + if (coordinatorBatchlog && keyspace.getReplicationStrategy().getReplicationFactor().allReplicas == 1) continue; if (!forTable(update.metadata().id).updatedViews(update).isEmpty()) diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java index df16943263..ad10d9dcdf 100644 --- a/src/java/org/apache/cassandra/db/view/ViewUtils.java +++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java @@ -18,16 +18,17 @@ package org.apache.cassandra.db.view; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; +import java.util.function.Predicate; +import com.google.common.collect.Iterables; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.AbstractReplicationStrategy; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.utils.FBUtilities; public final class ViewUtils @@ -58,46 +59,51 @@ public final class ViewUtils * * @return Optional.empty() if this method is called using a base token which does not belong to this replica */ - public static Optional getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken) + public static Optional getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken) { AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspaceName).getReplicationStrategy(); String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); - List baseEndpoints = new ArrayList<>(); - List viewEndpoints = new ArrayList<>(); - for (InetAddressAndPort baseEndpoint : replicationStrategy.getNaturalEndpoints(baseToken)) - { - // An endpoint is local if we're not using Net - if (!(replicationStrategy instanceof NetworkTopologyStrategy) || - DatabaseDescriptor.getEndpointSnitch().getDatacenter(baseEndpoint).equals(localDataCenter)) - baseEndpoints.add(baseEndpoint); - } + EndpointsForToken naturalBaseReplicas = replicationStrategy.getNaturalReplicasForToken(baseToken); + EndpointsForToken naturalViewReplicas = replicationStrategy.getNaturalReplicasForToken(viewToken); - for (InetAddressAndPort viewEndpoint : replicationStrategy.getNaturalEndpoints(viewToken)) - { - // If we are a base endpoint which is also a view replica, we use ourselves as our view replica - if (viewEndpoint.equals(FBUtilities.getBroadcastAddressAndPort())) - return Optional.of(viewEndpoint); + Optional localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isLocal).toJavaUtil(); + if (localReplica.isPresent()) + return localReplica; - // We have to remove any endpoint which is shared between the base and the view, as it will select itself - // and throw off the counts otherwise. - if (baseEndpoints.contains(viewEndpoint)) - baseEndpoints.remove(viewEndpoint); - else if (!(replicationStrategy instanceof NetworkTopologyStrategy) || - DatabaseDescriptor.getEndpointSnitch().getDatacenter(viewEndpoint).equals(localDataCenter)) - viewEndpoints.add(viewEndpoint); - } + // We only select replicas from our own DC + // TODO: this is poor encapsulation, leaking implementation details of replication strategy + Predicate isLocalDC = r -> !(replicationStrategy instanceof NetworkTopologyStrategy) + || DatabaseDescriptor.getEndpointSnitch().getDatacenter(r).equals(localDataCenter); + + // We have to remove any endpoint which is shared between the base and the view, as it will select itself + // and throw off the counts otherwise. + EndpointsForToken baseReplicas = naturalBaseReplicas.filter( + r -> !naturalViewReplicas.endpoints().contains(r.endpoint()) && isLocalDC.test(r) + ); + EndpointsForToken viewReplicas = naturalViewReplicas.filter( + r -> !naturalBaseReplicas.endpoints().contains(r.endpoint()) && isLocalDC.test(r) + ); // The replication strategy will be the same for the base and the view, as they must belong to the same keyspace. // Since the same replication strategy is used, the same placement should be used and we should get the same // number of replicas for all of the tokens in the ring. - assert baseEndpoints.size() == viewEndpoints.size() : "Replication strategy should have the same number of endpoints for the base and the view"; - int baseIdx = baseEndpoints.indexOf(FBUtilities.getBroadcastAddressAndPort()); + assert baseReplicas.size() == viewReplicas.size() : "Replication strategy should have the same number of endpoints for the base and the view"; + + int baseIdx = -1; + for (int i=0; i> extends AbstractBounds implemen /** * Helper class to check if a token is contained within a given collection of ranges */ - public static class OrderedRangeContainmentChecker + public static class OrderedRangeContainmentChecker implements Predicate { private final Iterator> normalizedRangesIterator; private Token lastToken = null; @@ -550,7 +551,8 @@ public class Range> extends AbstractBounds implemen * @param t token to check, must be larger than or equal to the last token passed * @return true if the token is contained within the ranges given to the constructor. */ - public boolean contains(Token t) + @Override + public boolean test(Token t) { assert lastToken == null || lastToken.compareTo(t) <= 0; lastToken = t; @@ -567,4 +569,25 @@ public class Range> extends AbstractBounds implemen } } } + + public static > void assertNormalized(List> ranges) + { + Range lastRange = null; + for (Range range : ranges) + { + if (lastRange == null) + { + lastRange = range; + } + else if (lastRange.left.compareTo(range.left) >= 0 || lastRange.intersects(range)) + { + throw new AssertionError(String.format("Ranges aren't properly normalized. lastRange %s, range %s, compareTo %d, intersects %b, all ranges %s%n", + lastRange, + range, + lastRange.compareTo(range), + lastRange.intersects(range), + ranges)); + } + } + } } diff --git a/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java index b90bc96392..4b98b971a0 100644 --- a/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java +++ b/src/java/org/apache/cassandra/dht/RangeFetchMapCalculator.java @@ -19,25 +19,27 @@ package org.apache.cassandra.dht; import java.math.BigInteger; -import java.net.InetAddress; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; +import org.apache.cassandra.locator.EndpointsByRange; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.locator.Replica; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.locator.Replicas; import org.psjava.algo.graph.flownetwork.FordFulkersonAlgorithm; import org.psjava.algo.graph.flownetwork.MaximumFlowAlgorithm; import org.psjava.algo.graph.flownetwork.MaximumFlowAlgorithmResult; @@ -73,20 +75,20 @@ public class RangeFetchMapCalculator { private static final Logger logger = LoggerFactory.getLogger(RangeFetchMapCalculator.class); private static final long TRIVIAL_RANGE_LIMIT = 1000; - private final Multimap, InetAddressAndPort> rangesWithSources; - private final Collection sourceFilters; + private final EndpointsByRange rangesWithSources; + private final Predicate sourceFilters; private final String keyspace; //We need two Vertices to act as source and destination in the algorithm private final Vertex sourceVertex = OuterVertex.getSourceVertex(); private final Vertex destinationVertex = OuterVertex.getDestinationVertex(); private final Set> trivialRanges; - public RangeFetchMapCalculator(Multimap, InetAddressAndPort> rangesWithSources, - Collection sourceFilters, + public RangeFetchMapCalculator(EndpointsByRange rangesWithSources, + Collection> sourceFilters, String keyspace) { this.rangesWithSources = rangesWithSources; - this.sourceFilters = sourceFilters; + this.sourceFilters = Predicates.and(sourceFilters); this.keyspace = keyspace; this.trivialRanges = rangesWithSources.keySet() .stream() @@ -158,14 +160,15 @@ public class RangeFetchMapCalculator boolean localDCCheck = true; while (!added) { - List srcs = new ArrayList<>(rangesWithSources.get(trivialRange)); // sort with the endpoint having the least number of streams first: - srcs.sort(Comparator.comparingInt(o -> optimisedMap.get(o).size())); - for (InetAddressAndPort src : srcs) + EndpointsForRange replicas = rangesWithSources.get(trivialRange) + .sorted(Comparator.comparingInt(o -> optimisedMap.get(o.endpoint()).size())); + Replicas.temporaryAssertFull(replicas); + for (Replica replica : replicas) { - if (passFilters(src, localDCCheck)) + if (passFilters(replica, localDCCheck)) { - fetchMap.put(src, trivialRange); + fetchMap.put(replica.endpoint(), trivialRange); added = true; break; } @@ -347,15 +350,16 @@ public class RangeFetchMapCalculator private boolean addEndpoints(MutableCapacityGraph capacityGraph, RangeVertex rangeVertex, boolean localDCCheck) { boolean sourceFound = false; - for (InetAddressAndPort endpoint : rangesWithSources.get(rangeVertex.getRange())) + Replicas.temporaryAssertFull(rangesWithSources.get(rangeVertex.getRange())); + for (Replica replica : rangesWithSources.get(rangeVertex.getRange())) { - if (passFilters(endpoint, localDCCheck)) + if (passFilters(replica, localDCCheck)) { sourceFound = true; // if we pass filters, it means that we don't filter away localhost and we can count it as a source: - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) + if (replica.isLocal()) continue; // but don't add localhost to the graph to avoid streaming locally - final Vertex endpointVertex = new EndpointVertex(endpoint); + final Vertex endpointVertex = new EndpointVertex(replica.endpoint()); capacityGraph.insertVertex(rangeVertex); capacityGraph.insertVertex(endpointVertex); capacityGraph.addEdge(rangeVertex, endpointVertex, Integer.MAX_VALUE); @@ -364,26 +368,20 @@ public class RangeFetchMapCalculator return sourceFound; } - private boolean isInLocalDC(InetAddressAndPort endpoint) + private boolean isInLocalDC(Replica replica) { - return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(endpoint)); + return DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica)); } /** * - * @param endpoint Endpoint to check + * @param replica Replica to check * @param localDCCheck Allow endpoints with local DC * @return True if filters pass this endpoint */ - private boolean passFilters(final InetAddressAndPort endpoint, boolean localDCCheck) + private boolean passFilters(final Replica replica, boolean localDCCheck) { - for (RangeStreamer.ISourceFilter filter : sourceFilters) - { - if (!filter.shouldInclude(endpoint)) - return false; - } - - return !localDCCheck || isInLocalDC(endpoint); + return sourceFilters.apply(replica) && (!localDCCheck || isInLocalDC(replica)); } private static abstract class Vertex diff --git a/src/java/org/apache/cassandra/dht/RangeStreamer.java b/src/java/org/apache/cassandra/dht/RangeStreamer.java index 110fed639c..e8aa5d3e65 100644 --- a/src/java/org/apache/cassandra/dht/RangeStreamer.java +++ b/src/java/org/apache/cassandra/dht/RangeStreamer.java @@ -18,27 +18,40 @@ package org.apache.cassandra.dht; import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; +import com.google.common.base.Predicate; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.EndpointsByReplica; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.EndpointsByRange; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.gms.EndpointState; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.Replicas; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.PreviewKind; @@ -47,13 +60,25 @@ import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.utils.FBUtilities; +import static com.google.common.base.Predicates.and; +import static com.google.common.base.Predicates.not; +import static com.google.common.collect.Iterables.all; +import static com.google.common.collect.Iterables.any; +import static org.apache.cassandra.locator.Replica.fullReplica; + /** - * Assists in streaming ranges to a node. + * Assists in streaming ranges to this node. */ public class RangeStreamer { private static final Logger logger = LoggerFactory.getLogger(RangeStreamer.class); + public static Predicate ALIVE_PREDICATE = replica -> + (!Gossiper.instance.isEnabled() || + (Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()) == null || + Gossiper.instance.getEndpointStateForEndpoint(replica.endpoint()).isAlive())) && + FailureDetector.instance.isAlive(replica.endpoint()); + /* bootstrap tokens. can be null if replacing the node. */ private final Collection tokens; /* current token ring */ @@ -62,26 +87,59 @@ public class RangeStreamer private final InetAddressAndPort address; /* streaming description */ private final String description; - private final Multimap>>> toFetch = HashMultimap.create(); - private final Set sourceFilters = new HashSet<>(); + private final Multimap> toFetch = HashMultimap.create(); + private final Set> sourceFilters = new HashSet<>(); private final StreamPlan streamPlan; private final boolean useStrictConsistency; private final IEndpointSnitch snitch; private final StreamStateStore stateStore; - /** - * A filter applied to sources to stream from when constructing a fetch map. - */ - public static interface ISourceFilter + public static class FetchReplica { - public boolean shouldInclude(InetAddressAndPort endpoint); + public final Replica local; + public final Replica remote; + + public FetchReplica(Replica local, Replica remote) + { + Preconditions.checkNotNull(local); + Preconditions.checkNotNull(remote); + assert local.isLocal() && !remote.isLocal(); + this.local = local; + this.remote = remote; + } + + public String toString() + { + return "FetchReplica{" + + "local=" + local + + ", remote=" + remote + + '}'; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FetchReplica that = (FetchReplica) o; + + if (!local.equals(that.local)) return false; + return remote.equals(that.remote); + } + + public int hashCode() + { + int result = local.hashCode(); + result = 31 * result + remote.hashCode(); + return result; + } } /** * Source filter which excludes any endpoints that are not alive according to a * failure detector. */ - public static class FailureDetectorSourceFilter implements ISourceFilter + public static class FailureDetectorSourceFilter implements Predicate { private final IFailureDetector fd; @@ -90,16 +148,16 @@ public class RangeStreamer this.fd = fd; } - public boolean shouldInclude(InetAddressAndPort endpoint) + public boolean apply(Replica replica) { - return fd.isAlive(endpoint); + return fd.isAlive(replica.endpoint()); } } /** * Source filter which excludes any endpoints that are not in a specific data center. */ - public static class SingleDatacenterFilter implements ISourceFilter + public static class SingleDatacenterFilter implements Predicate { private final String sourceDc; private final IEndpointSnitch snitch; @@ -110,27 +168,27 @@ public class RangeStreamer this.snitch = snitch; } - public boolean shouldInclude(InetAddressAndPort endpoint) + public boolean apply(Replica replica) { - return snitch.getDatacenter(endpoint).equals(sourceDc); + return snitch.getDatacenter(replica).equals(sourceDc); } } /** * Source filter which excludes the current node from source calculations */ - public static class ExcludeLocalNodeFilter implements ISourceFilter + public static class ExcludeLocalNodeFilter implements Predicate { - public boolean shouldInclude(InetAddressAndPort endpoint) + public boolean apply(Replica replica) { - return !FBUtilities.getBroadcastAddressAndPort().equals(endpoint); + return !replica.isLocal(); } } /** * Source filter which only includes endpoints contained within a provided set. */ - public static class WhitelistedSourcesFilter implements ISourceFilter + public static class WhitelistedSourcesFilter implements Predicate { private final Set whitelistedSources; @@ -139,9 +197,9 @@ public class RangeStreamer this.whitelistedSources = whitelistedSources; } - public boolean shouldInclude(InetAddressAndPort endpoint) + public boolean apply(Replica replica) { - return whitelistedSources.contains(endpoint); + return whitelistedSources.contains(replica.endpoint()); } } @@ -167,7 +225,7 @@ public class RangeStreamer streamPlan.listeners(this.stateStore); } - public void addSourceFilter(ISourceFilter filter) + public void addSourceFilter(Predicate filter) { sourceFilters.add(filter); } @@ -176,80 +234,95 @@ public class RangeStreamer * Add ranges to be streamed for given keyspace. * * @param keyspaceName keyspace name - * @param ranges ranges to be streamed + * @param replicas ranges to be streamed */ - public void addRanges(String keyspaceName, Collection> ranges) + public void addRanges(String keyspaceName, ReplicaCollection replicas) { - if(Keyspace.open(keyspaceName).getReplicationStrategy() instanceof LocalStrategy) + Keyspace keyspace = Keyspace.open(keyspaceName); + AbstractReplicationStrategy strat = keyspace.getReplicationStrategy(); + if(strat instanceof LocalStrategy) { logger.info("Not adding ranges for Local Strategy keyspace={}", keyspaceName); return; } - boolean useStrictSource = useStrictSourcesForRanges(keyspaceName); - Multimap, InetAddressAndPort> rangesForKeyspace = useStrictSource - ? getAllRangesWithStrictSourcesFor(keyspaceName, ranges) : getAllRangesWithSourcesFor(keyspaceName, ranges); + boolean useStrictSource = useStrictSourcesForRanges(strat); + EndpointsByReplica fetchMap = calculateRangesToFetchWithPreferredEndpoints(replicas, keyspace, useStrictSource); - for (Map.Entry, InetAddressAndPort> entry : rangesForKeyspace.entries()) + for (Map.Entry entry : fetchMap.flattenEntries()) logger.info("{}: range {} exists on {} for keyspace {}", description, entry.getKey(), entry.getValue(), keyspaceName); - AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy(); - Multimap> rangeFetchMap = useStrictSource || strat == null || strat.getReplicationFactor() == 1 - ? getRangeFetchMap(rangesForKeyspace, sourceFilters, keyspaceName, useStrictConsistency) - : getOptimizedRangeFetchMap(rangesForKeyspace, sourceFilters, keyspaceName); - for (Map.Entry>> entry : rangeFetchMap.asMap().entrySet()) + Multimap workMap; + //Only use the optimized strategy if we don't care about strict sources, have a replication factor > 1, and no + //transient replicas. + if (useStrictSource || strat == null || strat.getReplicationFactor().allReplicas == 1 || strat.getReplicationFactor().hasTransientReplicas()) + { + workMap = convertPreferredEndpointsToWorkMap(fetchMap); + } + else + { + workMap = getOptimizedWorkMap(fetchMap, sourceFilters, keyspaceName); + } + + toFetch.put(keyspaceName, workMap); + for (Map.Entry> entry : workMap.asMap().entrySet()) { if (logger.isTraceEnabled()) { - for (Range r : entry.getValue()) - logger.trace("{}: range {} from source {} for keyspace {}", description, r, entry.getKey(), keyspaceName); + for (FetchReplica r : entry.getValue()) + logger.trace("{}: range source {} local range {} for keyspace {}", description, r.remote, r.local, keyspaceName); } - toFetch.put(keyspaceName, entry); } } /** - * @param keyspaceName keyspace name to check + * @param strat AbstractReplicationStrategy of keyspace to check * @return true when the node is bootstrapping, useStrictConsistency is true and # of nodes in the cluster is more than # of replica */ - private boolean useStrictSourcesForRanges(String keyspaceName) + private boolean useStrictSourcesForRanges(AbstractReplicationStrategy strat) { - AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy(); return useStrictConsistency && tokens != null - && metadata.getSizeOfAllEndpoints() != strat.getReplicationFactor(); + && metadata.getSizeOfAllEndpoints() != strat.getReplicationFactor().allReplicas; } /** - * Get a map of all ranges and their respective sources that are candidates for streaming the given ranges - * to us. For each range, the list of sources is sorted by proximity relative to the given destAddress. - * - * @throws java.lang.IllegalStateException when there is no source to get data streamed + * Wrapper method to assemble the arguments for invoking the implementation with RangeStreamer's parameters + * @param fetchRanges + * @param keyspace + * @param useStrictConsistency + * @return */ - private Multimap, InetAddressAndPort> getAllRangesWithSourcesFor(String keyspaceName, Collection> desiredRanges) + private EndpointsByReplica calculateRangesToFetchWithPreferredEndpoints(ReplicaCollection fetchRanges, Keyspace keyspace, boolean useStrictConsistency) { - AbstractReplicationStrategy strat = Keyspace.open(keyspaceName).getReplicationStrategy(); - Multimap, InetAddressAndPort> rangeAddresses = strat.getRangeAddresses(metadata.cloneOnlyTokenMap()); + AbstractReplicationStrategy strat = keyspace.getReplicationStrategy(); - Multimap, InetAddressAndPort> rangeSources = ArrayListMultimap.create(); - for (Range desiredRange : desiredRanges) + TokenMetadata tmd = metadata.cloneOnlyTokenMap(); + + TokenMetadata tmdAfter = null; + + if (tokens != null) { - for (Range range : rangeAddresses.keySet()) - { - if (range.contains(desiredRange)) - { - List preferred = snitch.getSortedListByProximity(address, rangeAddresses.get(range)); - rangeSources.putAll(desiredRange, preferred); - break; - } - } - - if (!rangeSources.keySet().contains(desiredRange)) - throw new IllegalStateException("No sources found for " + desiredRange); + // Pending ranges + tmdAfter = tmd.cloneOnlyTokenMap(); + tmdAfter.updateNormalTokens(tokens, address); + } + else if (useStrictConsistency) + { + throw new IllegalArgumentException("Can't ask for strict consistency and not supply tokens"); } - return rangeSources; + return RangeStreamer.calculateRangesToFetchWithPreferredEndpoints(snitch::sortedByProximity, + strat, + fetchRanges, + useStrictConsistency, + tmd, + tmdAfter, + ALIVE_PREDICATE, + keyspace.getName(), + sourceFilters); + } /** @@ -257,129 +330,234 @@ public class RangeStreamer * For each range, the list should only contain a single source. This allows us to consistently migrate data without violating * consistency. * - * @throws java.lang.IllegalStateException when there is no source to get data streamed, or more than 1 source found. - */ - private Multimap, InetAddressAndPort> getAllRangesWithStrictSourcesFor(String keyspace, Collection> desiredRanges) + **/ + public static EndpointsByReplica + calculateRangesToFetchWithPreferredEndpoints(BiFunction snitchGetSortedListByProximity, + AbstractReplicationStrategy strat, + ReplicaCollection fetchRanges, + boolean useStrictConsistency, + TokenMetadata tmdBefore, + TokenMetadata tmdAfter, + Predicate isAlive, + String keyspace, + Collection> sourceFilters) { - assert tokens != null; - AbstractReplicationStrategy strat = Keyspace.open(keyspace).getReplicationStrategy(); + EndpointsByRange rangeAddresses = strat.getRangeAddresses(tmdBefore); - // Active ranges - TokenMetadata metadataClone = metadata.cloneOnlyTokenMap(); - Multimap, InetAddressAndPort> addressRanges = strat.getRangeAddresses(metadataClone); + InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort(); + logger.debug ("Keyspace: {}", keyspace); + logger.debug("To fetch RN: {}", fetchRanges); + logger.debug("Fetch ranges: {}", rangeAddresses); - // Pending ranges - metadataClone.updateNormalTokens(tokens, address); - Multimap, InetAddressAndPort> pendingRangeAddresses = strat.getRangeAddresses(metadataClone); + Predicate testSourceFilters = and(sourceFilters); + Function sorted = + endpoints -> snitchGetSortedListByProximity.apply(localAddress, endpoints); - // Collects the source that will have its range moved to the new node - Multimap, InetAddressAndPort> rangeSources = ArrayListMultimap.create(); - - for (Range desiredRange : desiredRanges) + //This list of replicas is just candidates. With strict consistency it's going to be a narrow list. + EndpointsByReplica.Mutable rangesToFetchWithPreferredEndpoints = new EndpointsByReplica.Mutable(); + for (Replica toFetch : fetchRanges) { - for (Map.Entry, Collection> preEntry : addressRanges.asMap().entrySet()) - { - if (preEntry.getKey().contains(desiredRange)) - { - Set oldEndpoints = Sets.newHashSet(preEntry.getValue()); - Set newEndpoints = Sets.newHashSet(pendingRangeAddresses.get(desiredRange)); + //Replica that is sufficient to provide the data we need + //With strict consistency and transient replication we may end up with multiple types + //so this isn't used with strict consistency + Predicate isSufficient = r -> (toFetch.isTransient() || r.isFull()); + Predicate accept = r -> + isSufficient.test(r) // is sufficient + && !r.endpoint().equals(localAddress) // is not self + && isAlive.test(r); // is alive - // Due to CASSANDRA-5953 we can have a higher RF then we have endpoints. - // So we need to be careful to only be strict when endpoints == RF - if (oldEndpoints.size() == strat.getReplicationFactor()) + logger.debug("To fetch {}", toFetch); + for (Range range : rangeAddresses.keySet()) + { + if (range.contains(toFetch.range())) + { + EndpointsForRange oldEndpoints = rangeAddresses.get(range); + + //Ultimately we populate this with whatever is going to be fetched from to satisfy toFetch + //It could be multiple endpoints and we must fetch from all of them if they are there + //With transient replication and strict consistency this is to get the full data from a full replica and + //transient data from the transient replica losing data + EndpointsForRange sources; + if (useStrictConsistency) { - oldEndpoints.removeAll(newEndpoints); - assert oldEndpoints.size() == 1 : "Expected 1 endpoint but found " + oldEndpoints.size(); + //Start with two sets of who replicates the range before and who replicates it after + EndpointsForRange newEndpoints = strat.calculateNaturalReplicas(toFetch.range().right, tmdAfter); + logger.debug("Old endpoints {}", oldEndpoints); + logger.debug("New endpoints {}", newEndpoints); + + //Due to CASSANDRA-5953 we can have a higher RF then we have endpoints. + //So we need to be careful to only be strict when endpoints == RF + if (oldEndpoints.size() == strat.getReplicationFactor().allReplicas) + { + Set endpointsStillReplicated = newEndpoints.endpoints(); + // Remove new endpoints from old endpoints based on address + oldEndpoints = oldEndpoints.filter(r -> !endpointsStillReplicated.contains(r.endpoint())); + + if (!all(oldEndpoints, isAlive)) + throw new IllegalStateException("A node required to move the data consistently is down: " + + oldEndpoints.filter(not(isAlive))); + + if (oldEndpoints.size() > 1) + throw new AssertionError("Expected <= 1 endpoint but found " + oldEndpoints); + + //If we are transitioning from transient to full and and the set of replicas for the range is not changing + //we might end up with no endpoints to fetch from by address. In that case we can pick any full replica safely + //since we are already a transient replica and the existing replica remains. + //The old behavior where we might be asked to fetch ranges we don't need shouldn't occur anymore. + //So it's an error if we don't find what we need. + if (oldEndpoints.isEmpty() && toFetch.isTransient()) + { + throw new AssertionError("If there are no endpoints to fetch from then we must be transitioning from transient to full for range " + toFetch); + } + + if (!any(oldEndpoints, isSufficient)) + { + // need an additional replica + EndpointsForRange endpointsForRange = sorted.apply(rangeAddresses.get(range)); + // include all our filters, to ensure we include a matching node + Optional fullReplica = Iterables.tryFind(endpointsForRange, and(accept, testSourceFilters)).toJavaUtil(); + if (fullReplica.isPresent()) + oldEndpoints = Endpoints.concat(oldEndpoints, EndpointsForRange.of(fullReplica.get())); + else + throw new IllegalStateException("Couldn't find any matching sufficient replica out of " + endpointsForRange); + } + + //We have to check the source filters here to see if they will remove any replicas + //required for strict consistency + if (!all(oldEndpoints, testSourceFilters)) + throw new IllegalStateException("Necessary replicas for strict consistency were removed by source filters: " + oldEndpoints.filter(not(testSourceFilters))); + } + else + { + oldEndpoints = sorted.apply(oldEndpoints.filter(accept)); + } + + //Apply testSourceFilters that were given to us, and establish everything remaining is alive for the strict case + sources = oldEndpoints.filter(testSourceFilters); + } + else + { + //Without strict consistency we have given up on correctness so no point in fetching from + //a random full + transient replica since it's also likely to lose data + //Also apply testSourceFilters that were given to us so we can safely select a single source + sources = sorted.apply(rangeAddresses.get(range).filter(and(accept, testSourceFilters))); + //Limit it to just the first possible source, we don't need more than one and downstream + //will fetch from every source we supply + sources = sources.size() > 0 ? sources.subList(0, 1) : sources; } - rangeSources.put(desiredRange, oldEndpoints.iterator().next()); + // storing range and preferred endpoint set + rangesToFetchWithPreferredEndpoints.putAll(toFetch, sources, Conflict.NONE); + logger.debug("Endpoints to fetch for {} are {}", toFetch, sources); } } - // Validate - Collection addressList = rangeSources.get(desiredRange); - if (addressList == null || addressList.isEmpty()) - throw new IllegalStateException("No sources found for " + desiredRange); + EndpointsForRange addressList = rangesToFetchWithPreferredEndpoints.getIfPresent(toFetch); + if (addressList == null) + throw new IllegalStateException("Failed to find endpoints to fetch " + toFetch); - if (addressList.size() > 1) - throw new IllegalStateException("Multiple endpoints found for " + desiredRange); + /* + * When we move forwards (shrink our bucket) we are the one losing a range and no one else loses + * from that action (we also don't gain). When we move backwards there are two people losing a range. One is a full replica + * and the other is a transient replica. So we must need fetch from two places in that case for the full range we gain. + * For a transient range we only need to fetch from one. + */ + if (useStrictConsistency && addressList.size() > 1 && (addressList.filter(Replica::isFull).size() > 1 || addressList.filter(Replica::isTransient).size() > 1)) + throw new IllegalStateException(String.format("Multiple strict sources found for %s, sources: %s", toFetch, addressList)); - InetAddressAndPort sourceIp = addressList.iterator().next(); - EndpointState sourceState = Gossiper.instance.getEndpointStateForEndpoint(sourceIp); - if (Gossiper.instance.isEnabled() && (sourceState == null || !sourceState.isAlive())) - throw new RuntimeException("A node required to move the data consistently is down (" + sourceIp + "). " + - "If you wish to move the data from a potentially inconsistent replica, restart the node with -Dcassandra.consistent.rangemovement=false"); + //We must have enough stuff to fetch from + if ((toFetch.isFull() && !any(addressList, Replica::isFull)) || addressList.isEmpty()) + { + if (strat.getReplicationFactor().allReplicas == 1) + { + if (useStrictConsistency) + { + logger.warn("A node required to move the data consistently is down"); + throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace + " with RF=1. " + + "Ensure this keyspace contains replicas in the source datacenter."); + } + else + logger.warn("Unable to find sufficient sources for streaming range {} in keyspace {} with RF=1. " + + "Keyspace might be missing data.", toFetch, keyspace); + + } + else + { + if (useStrictConsistency) + logger.warn("A node required to move the data consistently is down"); + throw new IllegalStateException("Unable to find sufficient sources for streaming range " + toFetch + " in keyspace " + keyspace); + } + } } - - return rangeSources; + return rangesToFetchWithPreferredEndpoints.asImmutableView(); } /** - * @param rangesWithSources The ranges we want to fetch (key) and their potential sources (value) - * @param sourceFilters A (possibly empty) collection of source filters to apply. In addition to any filters given - * here, we always exclude ourselves. - * @param keyspace keyspace name - * @return Map of source endpoint to collection of ranges + * The preferred endpoint list is the wrong format because it is keyed by Replica (this node) rather than the source + * endpoint we will fetch from which streaming wants. + * @param preferredEndpoints + * @return */ - private static Multimap> getRangeFetchMap(Multimap, InetAddressAndPort> rangesWithSources, - Collection sourceFilters, String keyspace, - boolean useStrictConsistency) + public static Multimap convertPreferredEndpointsToWorkMap(EndpointsByReplica preferredEndpoints) { - Multimap> rangeFetchMapMap = HashMultimap.create(); - for (Range range : rangesWithSources.keySet()) + Multimap workMap = HashMultimap.create(); + for (Map.Entry e : preferredEndpoints.entrySet()) { - boolean foundSource = false; - - outer: - for (InetAddressAndPort address : rangesWithSources.get(range)) + for (Replica source : e.getValue()) { - for (ISourceFilter filter : sourceFilters) - { - if (!filter.shouldInclude(address)) - continue outer; - } - - if (address.equals(FBUtilities.getBroadcastAddressAndPort())) - { - // If localhost is a source, we have found one, but we don't add it to the map to avoid streaming locally - foundSource = true; - continue; - } - - rangeFetchMapMap.put(address, range); - foundSource = true; - break; // ensure we only stream from one other node for each range - } - - if (!foundSource) - { - AbstractReplicationStrategy strat = Keyspace.open(keyspace).getReplicationStrategy(); - if (strat != null && strat.getReplicationFactor() == 1) - { - if (useStrictConsistency) - throw new IllegalStateException("Unable to find sufficient sources for streaming range " + range + " in keyspace " + keyspace + " with RF=1. " + - "Ensure this keyspace contains replicas in the source datacenter."); - else - logger.warn("Unable to find sufficient sources for streaming range {} in keyspace {} with RF=1. " + - "Keyspace might be missing data.", range, keyspace); - } - else - throw new IllegalStateException("Unable to find sufficient sources for streaming range " + range + " in keyspace " + keyspace); + assert (e.getKey()).isLocal(); + assert !source.isLocal(); + workMap.put(source.endpoint(), new FetchReplica(e.getKey(), source)); } } - - return rangeFetchMapMap; + logger.debug("Work map {}", workMap); + return workMap; } - - private static Multimap> getOptimizedRangeFetchMap(Multimap, InetAddressAndPort> rangesWithSources, - Collection sourceFilters, String keyspace) + /** + * Optimized version that also outputs the final work map + */ + private static Multimap getOptimizedWorkMap(EndpointsByReplica rangesWithSources, + Collection> sourceFilters, String keyspace) { - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, sourceFilters, keyspace); + //For now we just aren't going to use the optimized range fetch map with transient replication to shrink + //the surface area to test and introduce bugs. + //In the future it's possible we could run it twice once for full ranges with only full replicas + //and once with transient ranges and all replicas. Then merge the result. + EndpointsByRange.Mutable unwrapped = new EndpointsByRange.Mutable(); + for (Map.Entry entry : rangesWithSources.flattenEntries()) + { + Replicas.temporaryAssertFull(entry.getValue()); + unwrapped.put(entry.getKey().range(), entry.getValue()); + } + + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(unwrapped.asImmutableView(), sourceFilters, keyspace); Multimap> rangeFetchMapMap = calculator.getRangeFetchMap(); logger.info("Output from RangeFetchMapCalculator for keyspace {}", keyspace); - validateRangeFetchMap(rangesWithSources, rangeFetchMapMap, keyspace); - return rangeFetchMapMap; + validateRangeFetchMap(unwrapped.asImmutableView(), rangeFetchMapMap, keyspace); + + //Need to rewrap as Replicas + Multimap wrapped = HashMultimap.create(); + for (Map.Entry> entry : rangeFetchMapMap.entries()) + { + Replica toFetch = null; + for (Replica r : rangesWithSources.keySet()) + { + if (r.range().equals(entry.getValue())) + { + if (toFetch != null) + throw new AssertionError(String.format("There shouldn't be multiple replicas for range %s, replica %s and %s here", r.range(), r, toFetch)); + toFetch = r; + } + } + if (toFetch == null) + throw new AssertionError("Shouldn't be possible for the Replica we fetch to be null here"); + //Committing the cardinal sin of synthesizing a Replica, but it's ok because we assert earlier all of them + //are full and optimized range fetch map doesn't support transient replication yet. + wrapped.put(entry.getKey(), new FetchReplica(toFetch, fullReplica(entry.getKey(), entry.getValue()))); + } + + return wrapped; } /** @@ -388,7 +566,7 @@ public class RangeStreamer * @param rangeFetchMapMap * @param keyspace */ - private static void validateRangeFetchMap(Multimap, InetAddressAndPort> rangesWithSources, Multimap> rangeFetchMapMap, String keyspace) + private static void validateRangeFetchMap(EndpointsByRange rangesWithSources, Multimap> rangeFetchMapMap, String keyspace) { for (Map.Entry> entry : rangeFetchMapMap.entries()) { @@ -398,7 +576,7 @@ public class RangeStreamer + " in keyspace " + keyspace); } - if (!rangesWithSources.get(entry.getValue()).contains(entry.getKey())) + if (!rangesWithSources.get(entry.getValue()).endpoints().contains(entry.getKey())) { throw new IllegalStateException("Trying to stream from wrong endpoint. Range: " + entry.getValue() + " in keyspace " + keyspace + " from endpoint: " + entry.getKey()); @@ -408,39 +586,70 @@ public class RangeStreamer } } - public static Multimap> getWorkMap(Multimap, InetAddressAndPort> rangesWithSourceTarget, String keyspace, - IFailureDetector fd, boolean useStrictConsistency) - { - return getRangeFetchMap(rangesWithSourceTarget, Collections.singleton(new FailureDetectorSourceFilter(fd)), keyspace, useStrictConsistency); - } - // For testing purposes @VisibleForTesting - Multimap>>> toFetch() + Multimap> toFetch() { return toFetch; } public StreamResultFuture fetchAsync() { - for (Map.Entry>>> entry : toFetch.entries()) - { - String keyspace = entry.getKey(); - InetAddressAndPort source = entry.getValue().getKey(); - Collection> ranges = entry.getValue().getValue(); + toFetch.forEach((keyspace, sources) -> { + logger.debug("Keyspace {} Sources {}", keyspace, sources); + sources.asMap().forEach((source, fetchReplicas) -> { - // filter out already streamed ranges - Set> availableRanges = stateStore.getAvailableRanges(keyspace, StorageService.instance.getTokenMetadata().partitioner); - if (ranges.removeAll(availableRanges)) - { - logger.info("Some ranges of {} are already available. Skipping streaming those ranges.", availableRanges); - } + // filter out already streamed ranges + RangesAtEndpoint available = stateStore.getAvailableRanges(keyspace, StorageService.instance.getTokenMetadata().partitioner); - if (logger.isTraceEnabled()) - logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(ranges, ", ")); - /* Send messages to respective folks to stream data over to me */ - streamPlan.requestRanges(source, keyspace, ranges); - } + Predicate isAvailable = fetch -> { + Replica availableRange = available.byRange().get(fetch.local.range()); + if (availableRange == null) + //Range is unavailable + return false; + if (fetch.local.isFull()) + //For full, pick only replicas with matching transientness + return availableRange.isFull() == fetch.remote.isFull(); + + // Any transient or full will do + return true; + }; + + List remaining = fetchReplicas.stream().filter(not(isAvailable)).collect(Collectors.toList()); + + if (remaining.size() < available.size()) + { + List skipped = fetchReplicas.stream().filter(isAvailable).collect(Collectors.toList()); + logger.info("Some ranges of {} are already available. Skipping streaming those ranges. Skipping {}. Fully available {} Transiently available {}", + fetchReplicas, skipped, available.filter(Replica::isFull).ranges(), available.filter(Replica::isTransient).ranges()); + } + + if (logger.isTraceEnabled()) + logger.trace("{}ing from {} ranges {}", description, source, StringUtils.join(remaining, ", ")); + + //At the other end the distinction between full and transient is ignored it just used the transient status + //of the Replica objects we send to determine what to send. The real reason we have this split down to + //StreamRequest is that on completion StreamRequest is used to write to the system table tracking + //what has already been streamed. At that point since we only have the local Replica instances so we don't + //know what we got from the remote. We preserve that here by splitting based on the remotes transient + //status. + InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); + RangesAtEndpoint full = remaining.stream() + .filter(pair -> pair.remote.isFull()) + .map(pair -> pair.local) + .collect(RangesAtEndpoint.collector(self)); + RangesAtEndpoint transientReplicas = remaining.stream() + .filter(pair -> pair.remote.isTransient()) + .map(pair -> pair.local) + .collect(RangesAtEndpoint.collector(self)); + + logger.debug("Source and our replicas {}", fetchReplicas); + logger.debug("Source {} Keyspace {} streaming full {} transient {}", source, keyspace, full, transientReplicas); + + /* Send messages to respective folks to stream data over to me */ + streamPlan.requestRanges(source, keyspace, full, transientReplicas); + }); + }); return streamPlan.execute(); } diff --git a/src/java/org/apache/cassandra/dht/Splitter.java b/src/java/org/apache/cassandra/dht/Splitter.java index c63fe91a73..8578448434 100644 --- a/src/java/org/apache/cassandra/dht/Splitter.java +++ b/src/java/org/apache/cassandra/dht/Splitter.java @@ -25,6 +25,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import com.google.common.annotations.VisibleForTesting; @@ -117,32 +118,31 @@ public abstract class Splitter return new BigDecimal(elapsedTokens(token, range)).divide(new BigDecimal(tokensInRange(range)), 3, BigDecimal.ROUND_HALF_EVEN).doubleValue(); } - public List splitOwnedRanges(int parts, List> localRanges, boolean dontSplitRanges) + public List splitOwnedRanges(int parts, List weightedRanges, boolean dontSplitRanges) { - if (localRanges.isEmpty() || parts == 1) + if (weightedRanges.isEmpty() || parts == 1) return Collections.singletonList(partitioner.getMaximumToken()); BigInteger totalTokens = BigInteger.ZERO; - for (Range r : localRanges) + for (WeightedRange weightedRange : weightedRanges) { - BigInteger right = valueForToken(token(r.right)); - totalTokens = totalTokens.add(right.subtract(valueForToken(r.left))); + totalTokens = totalTokens.add(weightedRange.totalTokens(this)); } + BigInteger perPart = totalTokens.divide(BigInteger.valueOf(parts)); // the range owned is so tiny we can't split it: if (perPart.equals(BigInteger.ZERO)) return Collections.singletonList(partitioner.getMaximumToken()); if (dontSplitRanges) - return splitOwnedRangesNoPartialRanges(localRanges, perPart, parts); + return splitOwnedRangesNoPartialRanges(weightedRanges, perPart, parts); List boundaries = new ArrayList<>(); BigInteger sum = BigInteger.ZERO; - for (Range r : localRanges) + for (WeightedRange weightedRange : weightedRanges) { - Token right = token(r.right); - BigInteger currentRangeWidth = valueForToken(right).subtract(valueForToken(r.left)).abs(); - BigInteger left = valueForToken(r.left); + BigInteger currentRangeWidth = weightedRange.totalTokens(this); + BigInteger left = valueForToken(weightedRange.left()); while (sum.add(currentRangeWidth).compareTo(perPart) >= 0) { BigInteger withinRangeBoundary = perPart.subtract(sum); @@ -155,26 +155,24 @@ public abstract class Splitter } boundaries.set(boundaries.size() - 1, partitioner.getMaximumToken()); - assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + localRanges; + assert boundaries.size() == parts : boundaries.size() + "!=" + parts + " " + boundaries + ":" + weightedRanges; return boundaries; } - private List splitOwnedRangesNoPartialRanges(List> localRanges, BigInteger perPart, int parts) + private List splitOwnedRangesNoPartialRanges(List weightedRanges, BigInteger perPart, int parts) { List boundaries = new ArrayList<>(parts); BigInteger sum = BigInteger.ZERO; int i = 0; - final int rangesCount = localRanges.size(); + final int rangesCount = weightedRanges.size(); while (boundaries.size() < parts - 1 && i < rangesCount - 1) { - Range r = localRanges.get(i); - Range nextRange = localRanges.get(i + 1); - Token right = token(r.right); - Token nextRight = token(nextRange.right); + WeightedRange r = weightedRanges.get(i); + WeightedRange nextRange = weightedRanges.get(i + 1); - BigInteger currentRangeWidth = valueForToken(right).subtract(valueForToken(r.left)); - BigInteger nextRangeWidth = valueForToken(nextRight).subtract(valueForToken(nextRange.left)); + BigInteger currentRangeWidth = r.totalTokens(this); + BigInteger nextRangeWidth = nextRange.totalTokens(this); sum = sum.add(currentRangeWidth); // does this or next range take us beyond the per part limit? @@ -187,7 +185,7 @@ public abstract class Splitter if (diffNext.compareTo(diffCurrent) >= 0) { sum = BigInteger.ZERO; - boundaries.add(right); + boundaries.add(token(r.right())); } } i++; @@ -256,4 +254,61 @@ public abstract class Splitter } return subranges; } + + public static class WeightedRange + { + private final double weight; + private final Range range; + + public WeightedRange(double weight, Range range) + { + this.weight = weight; + this.range = range; + } + + public BigInteger totalTokens(Splitter splitter) + { + BigInteger right = splitter.valueForToken(splitter.token(range.right)); + BigInteger left = splitter.valueForToken(range.left); + BigInteger factor = BigInteger.valueOf(Math.max(1, (long) (1 / weight))); + BigInteger size = right.subtract(left); + return size.abs().divide(factor); + } + + public Token left() + { + return range.left; + } + + public Token right() + { + return range.right; + } + + public Range range() + { + return range; + } + + public String toString() + { + return "WeightedRange{" + + "weight=" + weight + + ", range=" + range + + '}'; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (!(o instanceof WeightedRange)) return false; + WeightedRange that = (WeightedRange) o; + return Objects.equals(range, that.range); + } + + public int hashCode() + { + return Objects.hash(weight, range); + } + } } diff --git a/src/java/org/apache/cassandra/dht/StreamStateStore.java b/src/java/org/apache/cassandra/dht/StreamStateStore.java index e3ea8388f7..3144e819a5 100644 --- a/src/java/org/apache/cassandra/dht/StreamStateStore.java +++ b/src/java/org/apache/cassandra/dht/StreamStateStore.java @@ -19,38 +19,43 @@ package org.apache.cassandra.dht; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; import org.apache.cassandra.streaming.StreamRequest; import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.utils.Pair; /** * Store and update available ranges (data already received) to system keyspace. */ public class StreamStateStore implements StreamEventHandler { - public Set> getAvailableRanges(String keyspace, IPartitioner partitioner) + private static final Logger logger = LoggerFactory.getLogger(StreamStateStore.class); + + public RangesAtEndpoint getAvailableRanges(String keyspace, IPartitioner partitioner) { return SystemKeyspace.getAvailableRanges(keyspace, partitioner); } /** - * Check if given token's data is available in this node. + * Check if given token's data is available in this node. This doesn't handle transientness in a useful way + * so it's only used by a legacy test * * @param keyspace keyspace name * @param token token to check * @return true if given token in the keyspace is already streamed and ready to be served. */ + @VisibleForTesting public boolean isDataAvailable(String keyspace, Token token) { - Set> availableRanges = getAvailableRanges(keyspace, token.getPartitioner()); - for (Range range : availableRanges) - { - if (range.contains(token)) - return true; - } - return false; + RangesAtEndpoint availableRanges = getAvailableRanges(keyspace, token.getPartitioner()); + return availableRanges.ranges().stream().anyMatch(range -> range.contains(token)); } /** @@ -73,7 +78,7 @@ public class StreamStateStore implements StreamEventHandler } for (StreamRequest request : se.requests) { - SystemKeyspace.updateAvailableRanges(request.keyspace, request.ranges); + SystemKeyspace.updateAvailableRanges(request.keyspace, request.full.ranges(), request.transientReplicas.ranges()); } } } diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java index efd2766aa6..36fc8c22e0 100644 --- a/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java +++ b/src/java/org/apache/cassandra/dht/tokenallocator/ReplicationAwareTokenAllocator.java @@ -73,7 +73,7 @@ class ReplicationAwareTokenAllocator extends TokenAllocatorBase Map> unitInfos = createUnitInfos(groups); if (groups.size() < replicas) { - // We need at least replicas groups to do allocation correctly. If there aren't enough, + // We need at least replicas groups to do allocation correctly. If there aren't enough, // use random allocation. // This part of the code should only be reached via the RATATest. StrategyAdapter should disallow // token allocation in this case as the algorithm is not able to cover the behavior of NetworkTopologyStrategy. diff --git a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java index 61082df534..ef91fbb30c 100644 --- a/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java +++ b/src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java @@ -113,7 +113,7 @@ public class TokenAllocation { double size = current.size(next); Token representative = current.getPartitioner().midpoint(current, next); - for (InetAddressAndPort n : rs.calculateNaturalEndpoints(representative, tokenMetadata)) + for (InetAddressAndPort n : rs.calculateNaturalReplicas(representative, tokenMetadata).endpoints()) { Double v = ownership.get(n); ownership.put(n, v != null ? v + size : size); @@ -169,7 +169,7 @@ public class TokenAllocation static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final SimpleStrategy rs, final InetAddressAndPort endpoint) { - final int replicas = rs.getReplicationFactor(); + final int replicas = rs.getReplicationFactor().allReplicas; return new StrategyAdapter() { @@ -196,7 +196,7 @@ public class TokenAllocation static StrategyAdapter getStrategy(final TokenMetadata tokenMetadata, final NetworkTopologyStrategy rs, final IEndpointSnitch snitch, final InetAddressAndPort endpoint) { final String dc = snitch.getDatacenter(endpoint); - final int replicas = rs.getReplicationFactor(dc); + final int replicas = rs.getReplicationFactor(dc).allReplicas; if (replicas == 0 || replicas == 1) { diff --git a/src/java/org/apache/cassandra/exceptions/UnavailableException.java b/src/java/org/apache/cassandra/exceptions/UnavailableException.java index 7b4edd8d07..d6e8488e22 100644 --- a/src/java/org/apache/cassandra/exceptions/UnavailableException.java +++ b/src/java/org/apache/cassandra/exceptions/UnavailableException.java @@ -25,14 +25,26 @@ public class UnavailableException extends RequestExecutionException public final int required; public final int alive; - public UnavailableException(ConsistencyLevel consistency, int required, int alive) + public static UnavailableException create(ConsistencyLevel consistency, int required, int alive) { - this("Cannot achieve consistency level " + consistency, consistency, required, alive); + assert alive < required; + return create(consistency, required, 0, alive, 0); } - public UnavailableException(ConsistencyLevel consistency, String dc, int required, int alive) + public static UnavailableException create(ConsistencyLevel consistency, int required, int requiredFull, int alive, int aliveFull) { - this("Cannot achieve consistency level " + consistency + " in DC " + dc, consistency, required, alive); + if (required > alive) + return new UnavailableException("Cannot achieve consistency level " + consistency, consistency, required, alive); + assert requiredFull < aliveFull; + return new UnavailableException("Insufficient full replicas", consistency, required, alive); + } + + public static UnavailableException create(ConsistencyLevel consistency, String dc, int required, int requiredFull, int alive, int aliveFull) + { + if (required > alive) + return new UnavailableException("Cannot achieve consistency level " + consistency + " in DC " + dc, consistency, required, alive); + assert requiredFull < aliveFull; + return new UnavailableException("Insufficient full replicas in DC " + dc, consistency, required, alive); } public UnavailableException(String msg, ConsistencyLevel consistency, int required, int alive) diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java index 5646bf63fd..8546a70c17 100644 --- a/src/java/org/apache/cassandra/gms/EndpointState.java +++ b/src/java/org/apache/cassandra/gms/EndpointState.java @@ -144,6 +144,11 @@ public class EndpointState return rpcState != null && Boolean.parseBoolean(rpcState.value); } + public boolean isNormalState() + { + return getStatus().equals(VersionedValue.STATUS_NORMAL); + } + public String getStatus() { VersionedValue status = getApplicationState(ApplicationState.STATUS_WITH_PORT); diff --git a/src/java/org/apache/cassandra/hints/HintsService.java b/src/java/org/apache/cassandra/hints/HintsService.java index 0cd1278e5d..c6ad3d9b10 100644 --- a/src/java/org/apache/cassandra/hints/HintsService.java +++ b/src/java/org/apache/cassandra/hints/HintsService.java @@ -20,11 +20,14 @@ package org.apache.cassandra.hints; import java.io.File; import java.lang.management.ManagementFactory; import java.net.UnknownHostException; +import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import java.util.stream.Collectors; import javax.management.MBeanServer; import javax.management.ObjectName; @@ -39,6 +42,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.HintedHandoffMetrics; import org.apache.cassandra.metrics.StorageMetrics; @@ -46,9 +50,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; -import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.transform; -import static com.google.common.collect.Iterables.size; /** * A singleton-ish wrapper over various hints components: @@ -151,7 +153,7 @@ public final class HintsService implements HintsServiceMBean * @param hostIds host ids of the hint's target nodes * @param hint the hint to store */ - public void write(Iterable hostIds, Hint hint) + public void write(Collection hostIds, Hint hint) { if (isShutDown) throw new IllegalStateException("HintsService is shut down and can't accept new hints"); @@ -161,7 +163,7 @@ public final class HintsService implements HintsServiceMBean bufferPool.write(hostIds, hint); - StorageMetrics.totalHints.inc(size(hostIds)); + StorageMetrics.totalHints.inc(hostIds.size()); } /** @@ -183,9 +185,14 @@ public final class HintsService implements HintsServiceMBean String keyspaceName = hint.mutation.getKeyspaceName(); Token token = hint.mutation.key().getToken(); - Iterable hostIds = - transform(filter(StorageService.instance.getNaturalAndPendingEndpoints(keyspaceName, token), StorageProxy::shouldHint), - StorageService.instance::getHostIdForEndpoint); + EndpointsForToken replicas = StorageService.instance.getNaturalAndPendingReplicasForToken(keyspaceName, token); + + // judicious use of streams: eagerly materializing probably cheaper + // than performing filters / translations 2x extra via Iterables.filter/transform + List hostIds = replicas.stream() + .filter(StorageProxy::shouldHint) + .map(replica -> StorageService.instance.getHostIdForEndpoint(replica.endpoint())) + .collect(Collectors.toList()); write(hostIds, hint); } diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index 044a00b1b3..4eaf1fe64a 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -69,13 +69,14 @@ abstract class AbstractSSTableSimpleWriter implements Closeable SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS); if (makeRangeAware) - return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, formatType, 0, header); + return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, formatType, 0, header); return SSTableTxnWriter.create(metadata, createDescriptor(directory, metadata.keyspace, metadata.name, formatType), 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, + false, 0, header, Collections.emptySet()); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index f2605fb9d7..055bf2440e 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -23,6 +23,7 @@ import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.CopyOnWriteArraySet; +import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.Collections2; import com.google.common.collect.Sets; @@ -46,6 +47,9 @@ import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.memory.HeapAllocator; +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; + /** * This class is built on top of the SequenceFile. It stores * data on disk in sorted fashion. However the sorting is upto @@ -350,4 +354,13 @@ public abstract class SSTable { return AbstractBounds.bounds(first.getToken(), true, last.getToken(), true); } + + public static void validateRepairedMetadata(long repairedAt, UUID pendingRepair, boolean isTransient) + { + Preconditions.checkArgument((pendingRepair == NO_PENDING_REPAIR) || (repairedAt == UNREPAIRED_SSTABLE), + "pendingRepair cannot be set on a repaired sstable"); + Preconditions.checkArgument(!isTransient || (pendingRepair != NO_PENDING_REPAIR), + "isTransient can only be true for sstables pending repair"); + + } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java index 4ba05337bd..ec2a7006c5 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java @@ -126,7 +126,7 @@ public class SSTableLoader implements StreamEventHandler for (Map.Entry>> entry : ranges.entrySet()) { InetAddressAndPort endpoint = entry.getKey(); - Collection> tokenRanges = entry.getValue(); + List> tokenRanges = Range.normalize(entry.getValue()); List sstableSections = sstable.getPositionsForRanges(tokenRanges); long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java index 60b896273f..cfb1365649 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java @@ -99,10 +99,10 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem } @SuppressWarnings("resource") // log and writer closed during doPostCleanup - public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, int sstableLevel, SerializationHeader header) + public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header) { LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, sstableLevel, header, txn); + SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, txn); return new SSTableTxnWriter(txn, writer); } @@ -112,6 +112,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, SSTableFormat.Type type, int sstableLevel, SerializationHeader header) @@ -122,7 +123,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem SSTableMultiWriter writer; try { - writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, type, sstableLevel, 0, txn, header); + writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, sstableLevel, 0, txn, header); } catch (IOException e) { @@ -140,6 +141,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, int sstableLevel, SerializationHeader header, Collection indexes) @@ -147,12 +149,12 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem // if the column family store does not exist, we create a new default SSTableMultiWriter to use: LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel); - SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, metadata, collector, header, indexes, txn); + SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn); return new SSTableTxnWriter(txn, writer); } - public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor desc, long keyCount, long repairedAt, UUID pendingRepair, SerializationHeader header) + public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor desc, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, SerializationHeader header) { - return create(cfs, desc, keyCount, repairedAt, pendingRepair, 0, header); + return create(cfs, desc, keyCount, repairedAt, pendingRepair, isTransient, 0, header); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java index a40ec18b8b..eb5c5fee99 100644 --- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java @@ -111,13 +111,14 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { - SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, pendingRepair, metadata, metadataCollector, header, indexes, txn); + SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, indexes, txn); return new SimpleSSTableMultiWriter(writer, txn); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java index f289fe3912..29fa573b15 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java @@ -44,6 +44,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final long estimatedKeys; private final long repairedAt; private final UUID pendingRepair; + private final boolean isTransient; private final SSTableFormat.Type format; private final SerializationHeader header; private final LifecycleTransaction txn; @@ -53,7 +54,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final List finishedReaders = new ArrayList<>(); private SSTableMultiWriter currentWriter = null; - public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, UUID pendingRepair, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleTransaction txn, SerializationHeader header) throws IOException + public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, UUID pendingRepair, boolean isTransient, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleTransaction txn, SerializationHeader header) throws IOException { DiskBoundaries db = cfs.getDiskBoundaries(); directories = db.directories; @@ -62,6 +63,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter this.estimatedKeys = estimatedKeys / directories.size(); this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; + this.isTransient = isTransient; this.format = format; this.txn = txn; this.header = header; @@ -73,7 +75,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter throw new IOException(String.format("Insufficient disk space to store %s", FBUtilities.prettyPrintMemory(totalSize))); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, txn); } } @@ -95,7 +97,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter finishedWriters.add(currentWriter); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, txn); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 2fade21992..edb3afa844 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -1852,6 +1852,11 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted> ranges) { Bounds range = new Bounds<>(first.getToken(), last.getToken()); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 1e183e27df..cca59cff9f 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -55,6 +55,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional { protected long repairedAt; protected UUID pendingRepair; + protected boolean isTransient; protected long maxDataAge = -1; protected final long keyCount; protected final MetadataCollector metadataCollector; @@ -77,6 +78,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, @@ -86,6 +88,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional this.keyCount = keyCount; this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; + this.isTransient = isTransient; this.metadataCollector = metadataCollector; this.header = header; this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), descriptor.version, header); @@ -96,6 +99,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional Long keyCount, Long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, @@ -103,20 +107,21 @@ public abstract class SSTableWriter extends SSTable implements Transactional LifecycleTransaction txn) { Factory writerFactory = descriptor.getFormat().getWriterFactory(); - return writerFactory.open(descriptor, keyCount, repairedAt, pendingRepair, metadata, metadataCollector, header, observers(descriptor, indexes, txn.opType()), txn); + return writerFactory.open(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers(descriptor, indexes, txn.opType()), txn); } public static SSTableWriter create(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, int sstableLevel, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { TableMetadataRef metadata = Schema.instance.getTableMetadataRef(descriptor); - return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, sstableLevel, header, indexes, txn); + return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, indexes, txn); } public static SSTableWriter create(TableMetadataRef metadata, @@ -124,13 +129,14 @@ public abstract class SSTableWriter extends SSTable implements Transactional long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, int sstableLevel, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel); - return create(descriptor, keyCount, repairedAt, pendingRepair, metadata, collector, header, indexes, txn); + return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn); } @VisibleForTesting @@ -138,11 +144,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, SerializationHeader header, Collection indexes, LifecycleTransaction txn) { - return create(descriptor, keyCount, repairedAt, pendingRepair, 0, header, indexes, txn); + return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, 0, header, indexes, txn); } private static Set components(TableMetadata metadata) @@ -309,6 +316,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional metadata().params.bloomFilterFpChance, repairedAt, pendingRepair, + isTransient, header); } @@ -338,6 +346,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index 1d965ce681..9b82c14195 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -55,6 +55,8 @@ public abstract class Version public abstract boolean hasPendingRepair(); + public abstract boolean hasIsTransient(); + public abstract boolean hasMetadataChecksum(); /** diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index db73b4f0bd..8eb860344e 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -21,6 +21,9 @@ import java.util.Collection; import java.util.Set; import java.util.UUID; +import com.google.common.base.Preconditions; + +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.db.RowIndexEntry; @@ -85,13 +88,15 @@ public class BigFormat implements SSTableFormat long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, Collection observers, LifecycleTransaction txn) { - return new BigTableWriter(descriptor, keyCount, repairedAt, pendingRepair, metadata, metadataCollector, header, observers, txn); + SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient); + return new BigTableWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers, txn); } } @@ -120,7 +125,7 @@ public class BigFormat implements SSTableFormat // mb (3.0.7, 3.7): commit log lower bound included // mc (3.0.8, 3.9): commit log intervals included - // na (4.0.0): uncompressed chunks, pending repair session, checksummed sstable metadata file, new Bloomfilter format + // na (4.0.0): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format // // NOTE: when adding a new version, please add that to LegacySSTableTest, too. @@ -131,6 +136,7 @@ public class BigFormat implements SSTableFormat public final boolean hasMaxCompressedLength; private final boolean hasPendingRepair; private final boolean hasMetadataChecksum; + private final boolean hasIsTransient; /** * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped) * have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution. @@ -148,6 +154,7 @@ public class BigFormat implements SSTableFormat hasCommitLogIntervals = version.compareTo("mc") >= 0; hasMaxCompressedLength = version.compareTo("na") >= 0; hasPendingRepair = version.compareTo("na") >= 0; + hasIsTransient = version.compareTo("na") >= 0; hasMetadataChecksum = version.compareTo("na") >= 0; hasOldBfFormat = version.compareTo("na") < 0; } @@ -175,6 +182,12 @@ public class BigFormat implements SSTableFormat return hasPendingRepair; } + @Override + public boolean hasIsTransient() + { + return hasIsTransient; + } + @Override public int correspondingMessagingVersion() { diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index b5488ed346..7513e95455 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -68,13 +68,14 @@ public class BigTableWriter extends SSTableWriter long keyCount, long repairedAt, UUID pendingRepair, + boolean isTransient, TableMetadataRef metadata, MetadataCollector metadataCollector, SerializationHeader header, Collection observers, LifecycleTransaction txn) { - super(descriptor, keyCount, repairedAt, pendingRepair, metadata, metadataCollector, header, observers); + super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers); txn.trackNew(this); // must track before any files are created if (compression) diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java index 6a40d948a0..eb7b2c77fb 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java @@ -71,7 +71,7 @@ public interface IMetadataSerializer void mutateLevel(Descriptor descriptor, int newLevel) throws IOException; /** - * Mutate the repairedAt time and pendingRepair ID + * Mutate the repairedAt time, pendingRepair ID, and transient status */ - void mutateRepaired(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair) throws IOException; + public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair, boolean isTransient) throws IOException; } diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index 9d9c1a89e5..36c218b023 100755 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@ -83,7 +83,8 @@ public class MetadataCollector implements PartitionStatisticsCollector ActiveRepairService.UNREPAIRED_SSTABLE, -1, -1, - null); + null, + false); } protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram(); @@ -272,7 +273,7 @@ public class MetadataCollector implements PartitionStatisticsCollector this.hasLegacyCounterShards = this.hasLegacyCounterShards || hasLegacyCounterShards; } - public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, UUID pendingRepair, SerializationHeader header) + public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, UUID pendingRepair, boolean isTransient, SerializationHeader header) { Map components = new EnumMap<>(MetadataType.class); components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance)); @@ -294,7 +295,8 @@ public class MetadataCollector implements PartitionStatisticsCollector repairedAt, totalColumnsSet, totalRows, - pendingRepair)); + pendingRepair, + isTransient)); components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality)); components.put(MetadataType.HEADER, header.toComponent()); return components; diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index 74923a0fb7..f76db2d533 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -230,7 +230,7 @@ public class MetadataSerializer implements IMetadataSerializer rewriteSSTableMetadata(descriptor, currentComponents); } - public void mutateRepaired(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair) throws IOException + public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, UUID newPendingRepair, boolean isTransient) throws IOException { if (logger.isTraceEnabled()) logger.trace("Mutating {} to repairedAt time {} and pendingRepair {}", @@ -238,7 +238,7 @@ public class MetadataSerializer implements IMetadataSerializer Map currentComponents = deserialize(descriptor, EnumSet.allOf(MetadataType.class)); StatsMetadata stats = (StatsMetadata) currentComponents.remove(MetadataType.STATS); // mutate time & id - currentComponents.put(MetadataType.STATS, stats.mutateRepairedAt(newRepairedAt).mutatePendingRepair(newPendingRepair)); + currentComponents.put(MetadataType.STATS, stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair, isTransient)); rewriteSSTableMetadata(descriptor, currentComponents); } diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index 2b8ebef096..f14fb5d3bf 100755 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -64,6 +64,7 @@ public class StatsMetadata extends MetadataComponent public final long totalColumnsSet; public final long totalRows; public final UUID pendingRepair; + public final boolean isTransient; public StatsMetadata(EstimatedHistogram estimatedPartitionSize, EstimatedHistogram estimatedColumnCount, @@ -83,7 +84,8 @@ public class StatsMetadata extends MetadataComponent long repairedAt, long totalColumnsSet, long totalRows, - UUID pendingRepair) + UUID pendingRepair, + boolean isTransient) { this.estimatedPartitionSize = estimatedPartitionSize; this.estimatedColumnCount = estimatedColumnCount; @@ -104,6 +106,7 @@ public class StatsMetadata extends MetadataComponent this.totalColumnsSet = totalColumnsSet; this.totalRows = totalRows; this.pendingRepair = pendingRepair; + this.isTransient = isTransient; } public MetadataType getType() @@ -155,10 +158,11 @@ public class StatsMetadata extends MetadataComponent repairedAt, totalColumnsSet, totalRows, - pendingRepair); + pendingRepair, + isTransient); } - public StatsMetadata mutateRepairedAt(long newRepairedAt) + public StatsMetadata mutateRepairedMetadata(long newRepairedAt, UUID newPendingRepair, boolean newIsTransient) { return new StatsMetadata(estimatedPartitionSize, estimatedColumnCount, @@ -178,30 +182,8 @@ public class StatsMetadata extends MetadataComponent newRepairedAt, totalColumnsSet, totalRows, - pendingRepair); - } - - public StatsMetadata mutatePendingRepair(UUID newPendingRepair) - { - return new StatsMetadata(estimatedPartitionSize, - estimatedColumnCount, - commitLogIntervals, - minTimestamp, - maxTimestamp, - minLocalDeletionTime, - maxLocalDeletionTime, - minTTL, - maxTTL, - compressionRatio, - estimatedTombstoneDropTime, - sstableLevel, - minClusteringValues, - maxClusteringValues, - hasLegacyCounterShards, - repairedAt, - totalColumnsSet, - totalRows, - newPendingRepair); + newPendingRepair, + newIsTransient); } @Override @@ -292,6 +274,12 @@ public class StatsMetadata extends MetadataComponent if (component.pendingRepair != null) size += UUIDSerializer.serializer.serializedSize(component.pendingRepair, 0); } + + if (version.hasIsTransient()) + { + size += TypeSizes.sizeof(component.isTransient); + } + return size; } @@ -338,6 +326,11 @@ public class StatsMetadata extends MetadataComponent out.writeByte(0); } } + + if (version.hasIsTransient()) + { + out.writeBoolean(component.isTransient); + } } public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException @@ -386,6 +379,8 @@ public class StatsMetadata extends MetadataComponent pendingRepair = UUIDSerializer.serializer.deserialize(in, 0); } + boolean isTransient = version.hasIsTransient() && in.readBoolean(); + return new StatsMetadata(partitionSizes, columnCounts, commitLogIntervals, @@ -404,7 +399,8 @@ public class StatsMetadata extends MetadataComponent repairedAt, totalColumnsSet, totalRows, - pendingRepair); + pendingRepair, + isTransient); } } } diff --git a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java index 2ee8eea6b1..2e7408be20 100644 --- a/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/AbstractEndpointSnitch.java @@ -17,13 +17,12 @@ */ package org.apache.cassandra.locator; -import java.util.*; - +import com.google.common.collect.Iterables; import org.apache.cassandra.config.DatabaseDescriptor; public abstract class AbstractEndpointSnitch implements IEndpointSnitch { - public abstract int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2); + public abstract int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2); /** * Sorts the Collection of node addresses by proximity to the given address @@ -31,27 +30,9 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch * @param unsortedAddress the nodes to sort * @return a new sorted List */ - public List getSortedListByProximity(InetAddressAndPort address, Collection unsortedAddress) + public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddress) { - List preferred = new ArrayList<>(unsortedAddress); - sortByProximity(address, preferred); - return preferred; - } - - /** - * Sorts the List of node addresses, in-place, by proximity to the given address - * @param address the address to sort the proximity by - * @param addresses the nodes to sort - */ - public void sortByProximity(final InetAddressAndPort address, List addresses) - { - Collections.sort(addresses, new Comparator() - { - public int compare(InetAddressAndPort a1, InetAddressAndPort a2) - { - return compareEndpoints(address, a1, a2); - } - }); + return unsortedAddress.sorted((r1, r2) -> compareEndpoints(address, r1, r2)); } public void gossiperStarting() @@ -59,7 +40,7 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch // noop by default } - public boolean isWorthMergingForRangeQuery(List merged, List l1, List l2) + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) { // Querying remote DC is likely to be an order of magnitude slower than // querying locally, so 2 queries to local nodes is likely to still be @@ -70,14 +51,9 @@ public abstract class AbstractEndpointSnitch implements IEndpointSnitch : true; } - private boolean hasRemoteNode(List l) + private boolean hasRemoteNode(ReplicaCollection l) { String localDc = DatabaseDescriptor.getLocalDataCenter(); - for (InetAddressAndPort ep : l) - { - if (!localDc.equals(getDatacenter(ep))) - return true; - } - return false; + return Iterables.any(l, replica -> !localDc.equals(getDatacenter(replica))); } } diff --git a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java index e91f6ac207..08c41f097f 100644 --- a/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java +++ b/src/java/org/apache/cassandra/locator/AbstractNetworkTopologySnitch.java @@ -37,8 +37,11 @@ public abstract class AbstractNetworkTopologySnitch extends AbstractEndpointSnit */ abstract public String getDatacenter(InetAddressAndPort endpoint); - public int compareEndpoints(InetAddressAndPort address, InetAddressAndPort a1, InetAddressAndPort a2) + @Override + public int compareEndpoints(InetAddressAndPort address, Replica r1, Replica r2) { + InetAddressAndPort a1 = r1.endpoint(); + InetAddressAndPort a2 = r2.endpoint(); if (address.equals(a1) && !address.equals(a2)) return -1; if (address.equals(a2) && !address.equals(a1)) diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java new file mode 100644 index 0000000000..ecf1296c8d --- /dev/null +++ b/src/java/org/apache/cassandra/locator/AbstractReplicaCollection.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collector; +import java.util.stream.Stream; + +/** + * A collection like class for Replica objects. Since the Replica class contains inetaddress, range, and + * transient replication status, basic contains and remove methods can be ambiguous. Replicas forces you + * to be explicit about what you're checking the container for, or removing from it. + */ +public abstract class AbstractReplicaCollection> implements ReplicaCollection +{ + protected static final List EMPTY_LIST = new ArrayList<>(); // since immutable, can safely return this to avoid megamorphic callsites + + public static , B extends Builder> Collector collector(Set characteristics, Supplier supplier) + { + return new Collector() + { + private final BiConsumer accumulator = Builder::add; + private final BinaryOperator combiner = (a, b) -> { a.addAll(b.mutable); return a; }; + private final Function finisher = Builder::build; + public Supplier supplier() { return supplier; } + public BiConsumer accumulator() { return accumulator; } + public BinaryOperator combiner() { return combiner; } + public Function finisher() { return finisher; } + public Set characteristics() { return characteristics; } + }; + } + + protected final List list; + protected final boolean isSnapshot; + protected AbstractReplicaCollection(List list, boolean isSnapshot) + { + this.list = list; + this.isSnapshot = isSnapshot; + } + + // if subList == null, should return self (or a clone thereof) + protected abstract C snapshot(List subList); + protected abstract C self(); + /** + * construct a new Mutable of our own type, so that we can concatenate + * TODO: this isn't terribly pretty, but we need sometimes to select / merge two Endpoints of unknown type; + */ + public abstract Mutable newMutable(int initialCapacity); + + + public C snapshot() + { + return isSnapshot ? self() + : snapshot(list.isEmpty() ? EMPTY_LIST + : new ArrayList<>(list)); + } + + public final C subList(int start, int end) + { + List subList; + if (isSnapshot) + { + if (start == 0 && end == size()) return self(); + else if (start == end) subList = EMPTY_LIST; + else subList = list.subList(start, end); + } + else + { + if (start == end) subList = EMPTY_LIST; + else subList = new ArrayList<>(list.subList(start, end)); // TODO: we could take a subList here, but comodification checks stop us + } + return snapshot(subList); + } + + public final C filter(Predicate predicate) + { + return filter(predicate, Integer.MAX_VALUE); + } + + public final C filter(Predicate predicate, int limit) + { + if (isEmpty()) + return snapshot(); + + List copy = null; + int beginRun = -1, endRun = -1; + int i = 0; + for (; i < list.size() ; ++i) + { + Replica replica = list.get(i); + if (predicate.test(replica)) + { + if (copy != null) + copy.add(replica); + else if (beginRun < 0) + beginRun = i; + else if (endRun > 0) + { + copy = new ArrayList<>(Math.min(limit, (list.size() - i) + (endRun - beginRun))); + for (int j = beginRun ; j < endRun ; ++j) + copy.add(list.get(j)); + copy.add(list.get(i)); + } + if (--limit == 0) + { + ++i; + break; + } + } + else if (beginRun >= 0 && endRun < 0) + endRun = i; + } + + if (beginRun < 0) + beginRun = endRun = 0; + if (endRun < 0) + endRun = i; + if (copy == null) + return subList(beginRun, endRun); + return snapshot(copy); + } + + public final class Select + { + private final List result; + public Select(int expectedSize) + { + this.result = new ArrayList<>(expectedSize); + } + + /** + * Add matching replica to the result; this predicate should be mutually exclusive with all prior predicates. + * Stop once we have targetSize replicas in total, including preceding calls + */ + public Select add(Predicate predicate, int targetSize) + { + assert !Iterables.any(result, predicate::test); + for (int i = 0 ; result.size() < targetSize && i < list.size() ; ++i) + if (predicate.test(list.get(i))) + result.add(list.get(i)); + return this; + } + public Select add(Predicate predicate) + { + return add(predicate, Integer.MAX_VALUE); + } + public C get() + { + return snapshot(result); + } + } + + /** + * An efficient method for selecting a subset of replica via a sequence of filters. + * + * Example: select().add(filter1).add(filter2, 3).get(); + * + * @return a Select object + */ + public final Select select() + { + return select(list.size()); + } + public final Select select(int expectedSize) + { + return new Select(expectedSize); + } + + public final C sorted(Comparator comparator) + { + List copy = new ArrayList<>(list); + copy.sort(comparator); + return snapshot(copy); + } + + public final Replica get(int i) + { + return list.get(i); + } + + public final int size() + { + return list.size(); + } + + public final boolean isEmpty() + { + return list.isEmpty(); + } + + public final Iterator iterator() + { + return list.iterator(); + } + + public final Stream stream() { return list.stream(); } + + public final boolean equals(Object o) + { + if (this == o) return true; + if (!(o instanceof AbstractReplicaCollection)) + { + if (!(o instanceof ReplicaCollection)) + return false; + + ReplicaCollection that = (ReplicaCollection) o; + return Iterables.elementsEqual(this, that); + } + AbstractReplicaCollection that = (AbstractReplicaCollection) o; + return Objects.equals(list, that.list); + } + + public final int hashCode() + { + return list.hashCode(); + } + + @Override + public final String toString() + { + return list.toString(); + } + + static > C concat(C replicas, C extraReplicas, Mutable.Conflict ignoreConflicts) + { + if (extraReplicas.isEmpty()) + return replicas; + if (replicas.isEmpty()) + return extraReplicas; + Mutable mutable = replicas.newMutable(replicas.size() + extraReplicas.size()); + mutable.addAll(replicas); + mutable.addAll(extraReplicas, ignoreConflicts); + return mutable.asImmutableView(); + } + +} diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java index 3e9b5bb26a..0ddc0a4392 100644 --- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java +++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java @@ -22,8 +22,8 @@ import java.lang.reflect.InvocationTargetException; import java.util.*; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; +import com.google.common.base.Preconditions; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,9 +73,9 @@ public abstract class AbstractReplicationStrategy // lazy-initialize keyspace itself since we don't create them until after the replication strategies } - private final Map> cachedEndpoints = new NonBlockingHashMap>(); + private final Map cachedReplicas = new NonBlockingHashMap<>(); - public ArrayList getCachedEndpoints(Token t) + public EndpointsForRange getCachedReplicas(Token t) { long lastVersion = tokenMetadata.getRingVersion(); @@ -86,13 +86,13 @@ public abstract class AbstractReplicationStrategy if (lastVersion > lastInvalidatedVersion) { logger.trace("clearing cached endpoints"); - cachedEndpoints.clear(); + cachedReplicas.clear(); lastInvalidatedVersion = lastVersion; } } } - return cachedEndpoints.get(t); + return cachedReplicas.get(t); } /** @@ -102,64 +102,65 @@ public abstract class AbstractReplicationStrategy * @param searchPosition the position the natural endpoints are requested for * @return a copy of the natural endpoints for the given token */ - public ArrayList getNaturalEndpoints(RingPosition searchPosition) + public EndpointsForToken getNaturalReplicasForToken(RingPosition searchPosition) + { + return getNaturalReplicas(searchPosition).forToken(searchPosition.getToken()); + } + + public EndpointsForRange getNaturalReplicas(RingPosition searchPosition) { Token searchToken = searchPosition.getToken(); Token keyToken = TokenMetadata.firstToken(tokenMetadata.sortedTokens(), searchToken); - ArrayList endpoints = getCachedEndpoints(keyToken); + EndpointsForRange endpoints = getCachedReplicas(keyToken); if (endpoints == null) { TokenMetadata tm = tokenMetadata.cachedOnlyTokenMap(); // if our cache got invalidated, it's possible there is a new token to account for too keyToken = TokenMetadata.firstToken(tm.sortedTokens(), searchToken); - endpoints = new ArrayList(calculateNaturalEndpoints(searchToken, tm)); - cachedEndpoints.put(keyToken, endpoints); + endpoints = calculateNaturalReplicas(searchToken, tm); + cachedReplicas.put(keyToken, endpoints); } - return new ArrayList(endpoints); + return endpoints; } /** * calculate the natural endpoints for the given token * - * @see #getNaturalEndpoints(org.apache.cassandra.dht.RingPosition) + * @see #getNaturalReplicasForToken(org.apache.cassandra.dht.RingPosition) * * @param searchToken the token the natural endpoints are requested for * @return a copy of the natural endpoints for the given token */ - public abstract List calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata); + public abstract EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata); - public AbstractWriteResponseHandler getWriteResponseHandler(Collection naturalEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistency_level, + public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime) { - return getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, callback, writeType, queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel()); + return getWriteResponseHandler(replicaLayout, callback, writeType, queryStartNanoTime, DatabaseDescriptor.getIdealConsistencyLevel()); } - public AbstractWriteResponseHandler getWriteResponseHandler(Collection naturalEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistency_level, + public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime, ConsistencyLevel idealConsistencyLevel) { AbstractWriteResponseHandler resultResponseHandler; - if (consistency_level.isDatacenterLocal()) + if (replicaLayout.consistencyLevel.isDatacenterLocal()) { // block for in this context will be localnodes block. - resultResponseHandler = new DatacenterWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime); + resultResponseHandler = new DatacenterWriteResponseHandler(replicaLayout, callback, writeType, queryStartNanoTime); } - else if (consistency_level == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy)) + else if (replicaLayout.consistencyLevel == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy)) { - resultResponseHandler = new DatacenterSyncWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime); + resultResponseHandler = new DatacenterSyncWriteResponseHandler(replicaLayout, callback, writeType, queryStartNanoTime); } else { - resultResponseHandler = new WriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, getKeyspace(), callback, writeType, queryStartNanoTime); + resultResponseHandler = new WriteResponseHandler(replicaLayout, callback, writeType, queryStartNanoTime); } //Check if tracking the ideal consistency level is configured @@ -168,16 +169,14 @@ public abstract class AbstractReplicationStrategy //If ideal and requested are the same just use this handler to track the ideal consistency level //This is also used so that the ideal consistency level handler when constructed knows it is the ideal //one for tracking purposes - if (idealConsistencyLevel == consistency_level) + if (idealConsistencyLevel == replicaLayout.consistencyLevel) { resultResponseHandler.setIdealCLResponseHandler(resultResponseHandler); } else { //Construct a delegate response handler to use to track the ideal consistency level - AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(naturalEndpoints, - pendingEndpoints, - idealConsistencyLevel, + AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(replicaLayout.withConsistencyLevel(idealConsistencyLevel), callback, writeType, queryStartNanoTime, @@ -202,7 +201,12 @@ public abstract class AbstractReplicationStrategy * * @return the replication factor */ - public abstract int getReplicationFactor(); + public abstract ReplicationFactor getReplicationFactor(); + + public boolean hasTransientReplicas() + { + return getReplicationFactor().hasTransientReplicas(); + } /* * NOTE: this is pretty inefficient. also the inverse (getRangeAddresses) below. @@ -210,53 +214,81 @@ public abstract class AbstractReplicationStrategy * (fixing this would probably require merging tokenmetadata into replicationstrategy, * so we could cache/invalidate cleanly.) */ - public Multimap> getAddressRanges(TokenMetadata metadata) + public RangesByEndpoint getAddressReplicas(TokenMetadata metadata) { - Multimap> map = HashMultimap.create(); + RangesByEndpoint.Mutable map = new RangesByEndpoint.Mutable(); for (Token token : metadata.sortedTokens()) { Range range = metadata.getPrimaryRangeFor(token); - for (InetAddressAndPort ep : calculateNaturalEndpoints(token, metadata)) + for (Replica replica : calculateNaturalReplicas(token, metadata)) { - map.put(ep, range); + // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here + Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy); + map.put(replica.endpoint(), replica); } } - return map; + return map.asImmutableView(); } - public Multimap, InetAddressAndPort> getRangeAddresses(TokenMetadata metadata) + public RangesAtEndpoint getAddressReplicas(TokenMetadata metadata, InetAddressAndPort endpoint) { - Multimap, InetAddressAndPort> map = HashMultimap.create(); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint); + for (Token token : metadata.sortedTokens()) + { + Range range = metadata.getPrimaryRangeFor(token); + Replica replica = calculateNaturalReplicas(token, metadata) + .byEndpoint().get(endpoint); + if (replica != null) + { + // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here + Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy); + builder.add(replica, Conflict.DUPLICATE); + } + } + return builder.build(); + } + + + public EndpointsByRange getRangeAddresses(TokenMetadata metadata) + { + EndpointsByRange.Mutable map = new EndpointsByRange.Mutable(); for (Token token : metadata.sortedTokens()) { Range range = metadata.getPrimaryRangeFor(token); - for (InetAddressAndPort ep : calculateNaturalEndpoints(token, metadata)) + for (Replica replica : calculateNaturalReplicas(token, metadata)) { - map.put(range, ep); + // LocalStrategy always returns (min, min] ranges for it's replicas, so we skip the check here + Preconditions.checkState(range.equals(replica.range()) || this instanceof LocalStrategy); + map.put(range, replica); } } - return map; + return map.asImmutableView(); } - public Multimap> getAddressRanges() + public RangesByEndpoint getAddressReplicas() { - return getAddressRanges(tokenMetadata.cloneOnlyTokenMap()); + return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap()); } - public Collection> getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddressAndPort pendingAddress) + public RangesAtEndpoint getAddressReplicas(InetAddressAndPort endpoint) { - return getPendingAddressRanges(metadata, Arrays.asList(pendingToken), pendingAddress); + return getAddressReplicas(tokenMetadata.cloneOnlyTokenMap(), endpoint); } - public Collection> getPendingAddressRanges(TokenMetadata metadata, Collection pendingTokens, InetAddressAndPort pendingAddress) + public RangesAtEndpoint getPendingAddressRanges(TokenMetadata metadata, Token pendingToken, InetAddressAndPort pendingAddress) + { + return getPendingAddressRanges(metadata, Collections.singleton(pendingToken), pendingAddress); + } + + public RangesAtEndpoint getPendingAddressRanges(TokenMetadata metadata, Collection pendingTokens, InetAddressAndPort pendingAddress) { TokenMetadata temp = metadata.cloneOnlyTokenMap(); temp.updateNormalTokens(pendingTokens, pendingAddress); - return getAddressRanges(temp).get(pendingAddress); + return getAddressReplicas(temp, pendingAddress); } public abstract void validateOptions() throws ConfigurationException; @@ -329,6 +361,10 @@ public abstract class AbstractReplicationStrategy AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, tokenMetadata, snitch, strategyOptions); strategy.validateExpectedOptions(); strategy.validateOptions(); + if (strategy.hasTransientReplicas() && !DatabaseDescriptor.isTransientReplicationEnabled()) + { + throw new ConfigurationException("Transient replication is disabled. Enable in cassandra.yaml to use."); + } } public static Class getClass(String cls) throws ConfigurationException @@ -344,21 +380,23 @@ public abstract class AbstractReplicationStrategy public boolean hasSameSettings(AbstractReplicationStrategy other) { - return getClass().equals(other.getClass()) && getReplicationFactor() == other.getReplicationFactor(); + return getClass().equals(other.getClass()) && getReplicationFactor().equals(other.getReplicationFactor()); } - protected void validateReplicationFactor(String rf) throws ConfigurationException + protected void validateReplicationFactor(String s) throws ConfigurationException { try { - if (Integer.parseInt(rf) < 0) + ReplicationFactor rf = ReplicationFactor.fromString(s); + if (rf.hasTransientReplicas()) { - throw new ConfigurationException("Replication factor must be non-negative; found " + rf); + if (DatabaseDescriptor.getNumTokens() > 1) + throw new ConfigurationException(String.format("Transient replication is not supported with vnodes yet")); } } - catch (NumberFormatException e2) + catch (IllegalArgumentException e) { - throw new ConfigurationException("Replication factor must be numeric; found " + rf); + throw new ConfigurationException(e.getMessage()); } } diff --git a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java index 010c8921fa..d35f1fb202 100644 --- a/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/DynamicEndpointSnitch.java @@ -42,7 +42,6 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; - /** * A dynamic snitch that sorts endpoints by latency with an adapted phi failure detector */ @@ -185,55 +184,38 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa return subsnitch.getDatacenter(endpoint); } - public List getSortedListByProximity(final InetAddressAndPort address, Collection addresses) - { - List list = new ArrayList<>(addresses); - sortByProximity(address, list); - return list; - } - @Override - public void sortByProximity(final InetAddressAndPort address, List addresses) + public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddresses) { assert address.equals(FBUtilities.getBroadcastAddressAndPort()); // we only know about ourself - if (dynamicBadnessThreshold == 0) - { - sortByProximityWithScore(address, addresses); - } - else - { - sortByProximityWithBadness(address, addresses); - } + return dynamicBadnessThreshold == 0 + ? sortedByProximityWithScore(address, unsortedAddresses) + : sortedByProximityWithBadness(address, unsortedAddresses); } - private void sortByProximityWithScore(final InetAddressAndPort address, List addresses) + private > C sortedByProximityWithScore(final InetAddressAndPort address, C unsortedAddresses) { // Scores can change concurrently from a call to this method. But Collections.sort() expects // its comparator to be "stable", that is 2 endpoint should compare the same way for the duration // of the sort() call. As we copy the scores map on write, it is thus enough to alias the current // version of it during this call. final HashMap scores = this.scores; - Collections.sort(addresses, new Comparator() - { - public int compare(InetAddressAndPort a1, InetAddressAndPort a2) - { - return compareEndpoints(address, a1, a2, scores); - } - }); + return unsortedAddresses.sorted((r1, r2) -> compareEndpoints(address, r1, r2, scores)); } - private void sortByProximityWithBadness(final InetAddressAndPort address, List addresses) + private > C sortedByProximityWithBadness(final InetAddressAndPort address, C replicas) { - if (addresses.size() < 2) - return; + if (replicas.size() < 2) + return replicas; - subsnitch.sortByProximity(address, addresses); + // TODO: avoid copy + replicas = subsnitch.sortedByProximity(address, replicas); HashMap scores = this.scores; // Make sure the score don't change in the middle of the loop below // (which wouldn't really matter here but its cleaner that way). - ArrayList subsnitchOrderedScores = new ArrayList<>(addresses.size()); - for (InetAddressAndPort inet : addresses) + ArrayList subsnitchOrderedScores = new ArrayList<>(replicas.size()); + for (Replica replica : replicas) { - Double score = scores.get(inet); + Double score = scores.get(replica.endpoint()); if (score == null) score = 0.0; subsnitchOrderedScores.add(score); @@ -250,17 +232,18 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa { if (subsnitchScore > (sortedScoreIterator.next() * (1.0 + dynamicBadnessThreshold))) { - sortByProximityWithScore(address, addresses); - return; + return sortedByProximityWithScore(address, replicas); } } + + return replicas; } // Compare endpoints given an immutable snapshot of the scores - private int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2, Map scores) + private int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2, Map scores) { - Double scored1 = scores.get(a1); - Double scored2 = scores.get(a2); + Double scored1 = scores.get(a1.endpoint()); + Double scored2 = scores.get(a2.endpoint()); if (scored1 == null) { @@ -280,7 +263,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa return 1; } - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { // That function is fundamentally unsafe because the scores can change at any time and so the result of that // method is not stable for identical arguments. This is why we don't rely on super.sortByProximity() in @@ -414,7 +397,7 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa return getSeverity(FBUtilities.getBroadcastAddressAndPort()); } - public boolean isWorthMergingForRangeQuery(List merged, List l1, List l2) + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) { if (!subsnitch.isWorthMergingForRangeQuery(merged, l1, l2)) return false; @@ -434,12 +417,12 @@ public class DynamicEndpointSnitch extends AbstractEndpointSnitch implements ILa } // Return the max score for the endpoint in the provided list, or -1.0 if no node have a score. - private double maxScore(List endpoints) + private double maxScore(ReplicaCollection endpoints) { double maxScore = -1.0; - for (InetAddressAndPort endpoint : endpoints) + for (Replica replica : endpoints) { - Double score = scores.get(endpoint); + Double score = scores.get(replica.endpoint()); if (score == null) continue; diff --git a/src/java/org/apache/cassandra/locator/Ec2Snitch.java b/src/java/org/apache/cassandra/locator/Ec2Snitch.java index b6aafd3b95..d0474e4369 100644 --- a/src/java/org/apache/cassandra/locator/Ec2Snitch.java +++ b/src/java/org/apache/cassandra/locator/Ec2Snitch.java @@ -68,7 +68,7 @@ public class Ec2Snitch extends AbstractNetworkTopologySnitch { String az = awsApiCall(ZONE_NAME_QUERY_URL); - // if using the full naming scheme, region name is created by removing letters from the + // if using the full naming scheme, region name is created by removing letters from the // end of the availability zone and zone is the full zone name usingLegacyNaming = isUsingLegacyNaming(props); String region; diff --git a/src/java/org/apache/cassandra/locator/Endpoints.java b/src/java/org/apache/cassandra/locator/Endpoints.java new file mode 100644 index 0000000000..3d5faa48fa --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Endpoints.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; +import org.apache.cassandra.utils.FBUtilities; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public abstract class Endpoints> extends AbstractReplicaCollection +{ + static final Map EMPTY_MAP = Collections.unmodifiableMap(new LinkedHashMap<>()); + + volatile Map byEndpoint; + + Endpoints(List list, boolean isSnapshot, Map byEndpoint) + { + super(list, isSnapshot); + this.byEndpoint = byEndpoint; + } + + @Override + public Set endpoints() + { + return byEndpoint().keySet(); + } + + public Map byEndpoint() + { + Map map = byEndpoint; + if (map == null) + byEndpoint = map = buildByEndpoint(list); + return map; + } + + public boolean contains(InetAddressAndPort endpoint, boolean isFull) + { + Replica replica = byEndpoint().get(endpoint); + return replica != null && replica.isFull() == isFull; + } + + @Override + public boolean contains(Replica replica) + { + return replica != null + && Objects.equals( + byEndpoint().get(replica.endpoint()), + replica); + } + + private static Map buildByEndpoint(List list) + { + // TODO: implement a delegating map that uses our superclass' list, and is immutable + Map byEndpoint = new LinkedHashMap<>(list.size()); + for (Replica replica : list) + { + Replica prev = byEndpoint.put(replica.endpoint(), replica); + assert prev == null : "duplicate endpoint in EndpointsForRange: " + prev + " and " + replica; + } + + return Collections.unmodifiableMap(byEndpoint); + } + + public E withoutSelf() + { + InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); + return filter(r -> !self.equals(r.endpoint())); + } + + public E without(Set remove) + { + return filter(r -> !remove.contains(r.endpoint())); + } + + public E keep(Set keep) + { + return filter(r -> keep.contains(r.endpoint())); + } + + public E keep(Iterable endpoints) + { + ReplicaCollection.Mutable copy = newMutable( + endpoints instanceof Collection + ? ((Collection) endpoints).size() + : size() + ); + Map byEndpoint = byEndpoint(); + for (InetAddressAndPort endpoint : endpoints) + { + Replica keep = byEndpoint.get(endpoint); + if (keep == null) + continue; + copy.add(keep, ReplicaCollection.Mutable.Conflict.DUPLICATE); + } + return copy.asSnapshot(); + } + + /** + * Care must be taken to ensure no conflicting ranges occur in pending and natural. + * Conflicts can occur for two reasons: + * 1) due to lack of isolation when reading pending/natural + * 2) because a movement that changes the type of replication from transient to full must be handled + * differently for reads and writes (with the reader treating it as transient, and writer as full) + * + * The method haveConflicts() below, and resolveConflictsInX, are used to detect and resolve any issues + */ + public static > E concat(E natural, E pending) + { + return AbstractReplicaCollection.concat(natural, pending, Conflict.NONE); + } + + public static > boolean haveConflicts(E natural, E pending) + { + Set naturalEndpoints = natural.endpoints(); + for (InetAddressAndPort pendingEndpoint : pending.endpoints()) + { + if (naturalEndpoints.contains(pendingEndpoint)) + return true; + } + return false; + } + + // must apply first + public static > E resolveConflictsInNatural(E natural, E pending) + { + return natural.filter(r -> !r.isTransient() || !pending.contains(r.endpoint(), true)); + } + + // must apply second + public static > E resolveConflictsInPending(E natural, E pending) + { + return pending.without(natural.endpoints()); + } + +} diff --git a/src/java/org/apache/cassandra/locator/EndpointsByRange.java b/src/java/org/apache/cassandra/locator/EndpointsByRange.java new file mode 100644 index 0000000000..cdc8a68e37 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/EndpointsByRange.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; + +import java.util.Collections; +import java.util.Map; + +public class EndpointsByRange extends ReplicaMultimap, EndpointsForRange> +{ + public EndpointsByRange(Map, EndpointsForRange> map) + { + super(map); + } + + public EndpointsForRange get(Range range) + { + Preconditions.checkNotNull(range); + return map.getOrDefault(range, EndpointsForRange.empty(range)); + } + + public static class Mutable extends ReplicaMultimap.Mutable, EndpointsForRange.Mutable> + { + @Override + protected EndpointsForRange.Mutable newMutable(Range range) + { + return new EndpointsForRange.Mutable(range); + } + + // TODO: consider all ignoreDuplicates cases + public void putAll(Range range, EndpointsForRange replicas, Conflict ignoreConflicts) + { + get(range).addAll(replicas, ignoreConflicts); + } + + public EndpointsByRange asImmutableView() + { + return new EndpointsByRange(Collections.unmodifiableMap(Maps.transformValues(map, EndpointsForRange.Mutable::asImmutableView))); + } + } + +} diff --git a/src/java/org/apache/cassandra/locator/EndpointsByReplica.java b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java new file mode 100644 index 0000000000..ceea2d1183 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; + +import java.util.Collections; +import java.util.Map; + +public class EndpointsByReplica extends ReplicaMultimap +{ + public EndpointsByReplica(Map map) + { + super(map); + } + + public EndpointsForRange get(Replica range) + { + Preconditions.checkNotNull(range); + return map.getOrDefault(range, EndpointsForRange.empty(range.range())); + } + + public static class Mutable extends ReplicaMultimap.Mutable + { + @Override + protected EndpointsForRange.Mutable newMutable(Replica replica) + { + return new EndpointsForRange.Mutable(replica.range()); + } + + // TODO: consider all ignoreDuplicates cases + public void putAll(Replica range, EndpointsForRange replicas, Conflict ignoreConflicts) + { + map.computeIfAbsent(range, r -> newMutable(r)).addAll(replicas, ignoreConflicts); + } + + public EndpointsByReplica asImmutableView() + { + return new EndpointsByReplica(Collections.unmodifiableMap(Maps.transformValues(map, EndpointsForRange.Mutable::asImmutableView))); + } + } + +} diff --git a/src/java/org/apache/cassandra/locator/EndpointsForRange.java b/src/java/org/apache/cassandra/locator/EndpointsForRange.java new file mode 100644 index 0000000000..c2d82329e5 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/EndpointsForRange.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Preconditions; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.Iterables.all; + +/** + * A ReplicaCollection where all Replica are required to cover a range that fully contains the range() defined in the builder(). + * Endpoints are guaranteed to be unique; on construction, this is enforced unless optionally silenced (in which case + * only the first occurrence makes the cut). + */ +public class EndpointsForRange extends Endpoints +{ + private final Range range; + private EndpointsForRange(Range range, List list, boolean isSnapshot) + { + this(range, list, isSnapshot, null); + } + private EndpointsForRange(Range range, List list, boolean isSnapshot, Map byEndpoint) + { + super(list, isSnapshot, byEndpoint); + this.range = range; + assert range != null; + } + + public Range range() + { + return range; + } + + @Override + public Mutable newMutable(int initialCapacity) + { + return new Mutable(range, initialCapacity); + } + + public EndpointsForToken forToken(Token token) + { + if (!range.contains(token)) + throw new IllegalArgumentException(token + " is not contained within " + range); + return new EndpointsForToken(token, list, isSnapshot, byEndpoint); + } + + @Override + public EndpointsForRange self() + { + return this; + } + + @Override + protected EndpointsForRange snapshot(List snapshot) + { + if (snapshot.isEmpty()) return empty(range); + return new EndpointsForRange(range, snapshot, true); + } + + public static class Mutable extends EndpointsForRange implements ReplicaCollection.Mutable + { + boolean hasSnapshot; + public Mutable(Range range) { this(range, 0); } + public Mutable(Range range, int capacity) { super(range, new ArrayList<>(capacity), false, new LinkedHashMap<>()); } + + public void add(Replica replica, Conflict ignoreConflict) + { + if (hasSnapshot) throw new IllegalStateException(); + Preconditions.checkNotNull(replica); + if (!replica.range().contains(super.range)) + throw new IllegalArgumentException("Replica " + replica + " does not contain " + super.range); + + Replica prev = super.byEndpoint.put(replica.endpoint(), replica); + if (prev != null) + { + super.byEndpoint.put(replica.endpoint(), prev); // restore prev + switch (ignoreConflict) + { + case DUPLICATE: + if (prev.equals(replica)) + break; + case NONE: + throw new IllegalArgumentException("Conflicting replica added (expected unique endpoints): " + replica + "; existing: " + prev); + case ALL: + } + return; + } + + list.add(replica); + } + + @Override + public Map byEndpoint() + { + // our internal map is modifiable, but it is unsafe to modify the map externally + // it would be possible to implement a safe modifiable map, but it is probably not valuable + return Collections.unmodifiableMap(super.byEndpoint()); + } + + private EndpointsForRange get(boolean isSnapshot) + { + return new EndpointsForRange(super.range, super.list, isSnapshot, Collections.unmodifiableMap(super.byEndpoint)); + } + + public EndpointsForRange asImmutableView() + { + return get(false); + } + + public EndpointsForRange asSnapshot() + { + hasSnapshot = true; + return get(true); + } + } + + public static class Builder extends ReplicaCollection.Builder + { + public Builder(Range range) { this(range, 0); } + public Builder(Range range, int capacity) { super (new Mutable(range, capacity)); } + public boolean containsEndpoint(InetAddressAndPort endpoint) + { + return mutable.asImmutableView().byEndpoint.containsKey(endpoint); + } + } + + public static Builder builder(Range range) + { + return new Builder(range); + } + public static Builder builder(Range range, int capacity) + { + return new Builder(range, capacity); + } + + public static EndpointsForRange empty(Range range) + { + return new EndpointsForRange(range, EMPTY_LIST, true, EMPTY_MAP); + } + + public static EndpointsForRange of(Replica replica) + { + // we only use ArrayList or ArrayList.SubList, to ensure callsites are bimorphic + ArrayList one = new ArrayList<>(1); + one.add(replica); + // we can safely use singletonMap, as we only otherwise use LinkedHashMap + return new EndpointsForRange(replica.range(), one, true, Collections.unmodifiableMap(Collections.singletonMap(replica.endpoint(), replica))); + } + + public static EndpointsForRange of(Replica ... replicas) + { + return copyOf(Arrays.asList(replicas)); + } + + public static EndpointsForRange copyOf(Collection replicas) + { + if (replicas.isEmpty()) + throw new IllegalArgumentException("Collection must be non-empty to copy"); + Range range = replicas.iterator().next().range(); + assert all(replicas, r -> range.equals(r.range())); + return builder(range, replicas.size()).addAll(replicas).build(); + } +} diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java new file mode 100644 index 0000000000..f24c6155a4 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Preconditions; +import org.apache.cassandra.dht.Token; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A ReplicaCollection where all Replica are required to cover a range that fully contains the token() defined in the builder(). + * Endpoints are guaranteed to be unique; on construction, this is enforced unless optionally silenced (in which case + * only the first occurrence makes the cut). + */ +public class EndpointsForToken extends Endpoints +{ + private final Token token; + private EndpointsForToken(Token token, List list, boolean isSnapshot) + { + this(token, list, isSnapshot, null); + } + + EndpointsForToken(Token token, List list, boolean isSnapshot, Map byEndpoint) + { + super(list, isSnapshot, byEndpoint); + this.token = token; + assert token != null; + } + + public Token token() + { + return token; + } + + @Override + public Mutable newMutable(int initialCapacity) + { + return new Mutable(token, initialCapacity); + } + + @Override + public EndpointsForToken self() + { + return this; + } + + @Override + protected EndpointsForToken snapshot(List subList) + { + if (subList.isEmpty()) return empty(token); + return new EndpointsForToken(token, subList, true); + } + + public static class Mutable extends EndpointsForToken implements ReplicaCollection.Mutable + { + boolean hasSnapshot; + public Mutable(Token token) { this(token, 0); } + public Mutable(Token token, int capacity) { super(token, new ArrayList<>(capacity), false, new LinkedHashMap<>()); } + + public void add(Replica replica, Conflict ignoreConflict) + { + if (hasSnapshot) throw new IllegalStateException(); + Preconditions.checkNotNull(replica); + if (!replica.range().contains(super.token)) + throw new IllegalArgumentException("Replica " + replica + " does not contain " + super.token); + + Replica prev = super.byEndpoint.put(replica.endpoint(), replica); + if (prev != null) + { + super.byEndpoint.put(replica.endpoint(), prev); // restore prev + switch (ignoreConflict) + { + case DUPLICATE: + if (prev.equals(replica)) + break; + case NONE: + throw new IllegalArgumentException("Conflicting replica added (expected unique endpoints): " + replica + "; existing: " + prev); + case ALL: + } + return; + } + + list.add(replica); + } + + @Override + public Map byEndpoint() + { + // our internal map is modifiable, but it is unsafe to modify the map externally + // it would be possible to implement a safe modifiable map, but it is probably not valuable + return Collections.unmodifiableMap(super.byEndpoint()); + } + + private EndpointsForToken get(boolean isSnapshot) + { + return new EndpointsForToken(super.token, super.list, isSnapshot, Collections.unmodifiableMap(super.byEndpoint)); + } + + public EndpointsForToken asImmutableView() + { + return get(false); + } + + public EndpointsForToken asSnapshot() + { + hasSnapshot = true; + return get(true); + } + } + + public static class Builder extends ReplicaCollection.Builder + { + public Builder(Token token) { this(token, 0); } + public Builder(Token token, int capacity) { super (new Mutable(token, capacity)); } + } + + public static Builder builder(Token token) + { + return new Builder(token); + } + public static Builder builder(Token token, int capacity) + { + return new Builder(token, capacity); + } + + public static EndpointsForToken empty(Token token) + { + return new EndpointsForToken(token, EMPTY_LIST, true, EMPTY_MAP); + } + + public static EndpointsForToken of(Token token, Replica replica) + { + // we only use ArrayList or ArrayList.SubList, to ensure callsites are bimorphic + ArrayList one = new ArrayList<>(1); + one.add(replica); + // we can safely use singletonMap, as we only otherwise use LinkedHashMap + return new EndpointsForToken(token, one, true, Collections.unmodifiableMap(Collections.singletonMap(replica.endpoint(), replica))); + } + + public static EndpointsForToken of(Token token, Replica ... replicas) + { + return copyOf(token, Arrays.asList(replicas)); + } + + public static EndpointsForToken copyOf(Token token, Collection replicas) + { + if (replicas.isEmpty()) return empty(token); + return builder(token, replicas.size()).addAll(replicas).build(); + } +} diff --git a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java index 63d333b3a3..b7797b0b14 100644 --- a/src/java/org/apache/cassandra/locator/IEndpointSnitch.java +++ b/src/java/org/apache/cassandra/locator/IEndpointSnitch.java @@ -17,8 +17,6 @@ */ package org.apache.cassandra.locator; -import java.util.Collection; -import java.util.List; import java.util.Set; /** @@ -39,20 +37,20 @@ public interface IEndpointSnitch */ public String getDatacenter(InetAddressAndPort endpoint); + default public String getDatacenter(Replica replica) + { + return getDatacenter(replica.endpoint()); + } + /** * returns a new List sorted by proximity to the given endpoint */ - public List getSortedListByProximity(InetAddressAndPort address, Collection unsortedAddress); - - /** - * This method will sort the List by proximity to the given address. - */ - public void sortByProximity(InetAddressAndPort address, List addresses); + public > C sortedByProximity(final InetAddressAndPort address, C addresses); /** * compares two endpoints in relation to the target endpoint, returning as Comparator.compare would */ - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2); + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2); /** * called after Gossiper instance exists immediately before it starts gossiping @@ -63,7 +61,7 @@ public interface IEndpointSnitch * Returns whether for a range query doing a query against merged is likely * to be faster than 2 sequential queries, one against l1 followed by one against l2. */ - public boolean isWorthMergingForRangeQuery(List merged, List l1, List l2); + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2); /** * Determine if the datacenter or rack values in the current node's snitch conflict with those passed in parameters. diff --git a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java index 38a1a49ec8..a47c72a716 100644 --- a/src/java/org/apache/cassandra/locator/InetAddressAndPort.java +++ b/src/java/org/apache/cassandra/locator/InetAddressAndPort.java @@ -25,6 +25,7 @@ import java.net.UnknownHostException; import com.google.common.base.Preconditions; import com.google.common.net.HostAndPort; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.FastByteOperations; /** @@ -191,9 +192,9 @@ public final class InetAddressAndPort implements Comparable, return InetAddressAndPort.getByAddress(InetAddress.getLoopbackAddress()); } - public static InetAddressAndPort getLocalHost() throws UnknownHostException + public static InetAddressAndPort getLocalHost() { - return InetAddressAndPort.getByAddress(InetAddress.getLocalHost()); + return FBUtilities.getLocalAddressAndPort(); } public static void initializeDefaultPort(int port) diff --git a/src/java/org/apache/cassandra/locator/LocalStrategy.java b/src/java/org/apache/cassandra/locator/LocalStrategy.java index a76fe966ac..41cc9b07db 100644 --- a/src/java/org/apache/cassandra/locator/LocalStrategy.java +++ b/src/java/org/apache/cassandra/locator/LocalStrategy.java @@ -17,12 +17,11 @@ */ package org.apache.cassandra.locator; -import java.util.ArrayList; import java.util.Collections; import java.util.Collection; -import java.util.List; import java.util.Map; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.RingPosition; import org.apache.cassandra.dht.Token; @@ -30,32 +29,40 @@ import org.apache.cassandra.utils.FBUtilities; public class LocalStrategy extends AbstractReplicationStrategy { + private static final ReplicationFactor RF = ReplicationFactor.fullOnly(1); + private final EndpointsForRange replicas; + public LocalStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map configOptions) { super(keyspaceName, tokenMetadata, snitch, configOptions); + replicas = EndpointsForRange.of( + new Replica(FBUtilities.getBroadcastAddressAndPort(), + DatabaseDescriptor.getPartitioner().getMinimumToken(), + DatabaseDescriptor.getPartitioner().getMinimumToken(), + true + ) + ); } /** - * We need to override this even if we override calculateNaturalEndpoints, + * We need to override this even if we override calculateNaturalReplicas, * because the default implementation depends on token calculations but * LocalStrategy may be used before tokens are set up. */ @Override - public ArrayList getNaturalEndpoints(RingPosition searchPosition) + public EndpointsForRange getNaturalReplicas(RingPosition searchPosition) { - ArrayList l = new ArrayList(1); - l.add(FBUtilities.getBroadcastAddressAndPort()); - return l; + return replicas; } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - return Collections.singletonList(FBUtilities.getBroadcastAddressAndPort()); + return replicas; } - public int getReplicationFactor() + public ReplicationFactor getReplicationFactor() { - return 1; + return RF; } public void validateOptions() throws ConfigurationException diff --git a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java index cb2ea465f2..c63f4f3af8 100644 --- a/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java +++ b/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java @@ -20,10 +20,12 @@ package org.apache.cassandra.locator; import java.util.*; import java.util.Map.Entry; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.dht.Datacenters; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.TokenMetadata.Topology; @@ -49,14 +51,17 @@ import com.google.common.collect.Multimap; */ public class NetworkTopologyStrategy extends AbstractReplicationStrategy { - private final Map datacenters; + private final Map datacenters; + private final ReplicationFactor aggregateRf; private static final Logger logger = LoggerFactory.getLogger(NetworkTopologyStrategy.class); public NetworkTopologyStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map configOptions) throws ConfigurationException { super(keyspaceName, tokenMetadata, snitch, configOptions); - Map newDatacenters = new HashMap(); + int replicas = 0; + int trans = 0; + Map newDatacenters = new HashMap<>(); if (configOptions != null) { for (Entry entry : configOptions.entrySet()) @@ -64,12 +69,15 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy String dc = entry.getKey(); if (dc.equalsIgnoreCase("replication_factor")) throw new ConfigurationException("replication_factor is an option for SimpleStrategy, not NetworkTopologyStrategy"); - Integer replicas = Integer.valueOf(entry.getValue()); - newDatacenters.put(dc, replicas); + ReplicationFactor rf = ReplicationFactor.fromString(entry.getValue()); + replicas += rf.allReplicas; + trans += rf.transientReplicas(); + newDatacenters.put(dc, rf); } } datacenters = Collections.unmodifiableMap(newDatacenters); + aggregateRf = ReplicationFactor.withTransient(replicas, trans); logger.info("Configured datacenter replicas are {}", FBUtilities.toString(datacenters)); } @@ -79,7 +87,8 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy private static final class DatacenterEndpoints { /** List accepted endpoints get pushed into. */ - Set endpoints; + EndpointsForRange.Mutable replicas; + /** * Racks encountered so far. Replicas are put into separate racks while possible. * For efficiency the set is shared between the instances, using the location pair (dc, rack) to make sure @@ -90,41 +99,51 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy /** Number of replicas left to fill from this DC. */ int rfLeft; int acceptableRackRepeats; + int transients; - DatacenterEndpoints(int rf, int rackCount, int nodeCount, Set endpoints, Set> racks) + DatacenterEndpoints(ReplicationFactor rf, int rackCount, int nodeCount, EndpointsForRange.Mutable replicas, Set> racks) { - this.endpoints = endpoints; + this.replicas = replicas; this.racks = racks; // If there aren't enough nodes in this DC to fill the RF, the number of nodes is the effective RF. - this.rfLeft = Math.min(rf, nodeCount); + this.rfLeft = Math.min(rf.allReplicas, nodeCount); // If there aren't enough racks in this DC to fill the RF, we'll still use at least one node from each rack, // and the difference is to be filled by the first encountered nodes. - acceptableRackRepeats = rf - rackCount; + acceptableRackRepeats = rf.allReplicas - rackCount; + + // if we have fewer replicas than rf calls for, reduce transients accordingly + int reduceTransients = rf.allReplicas - this.rfLeft; + transients = Math.max(rf.transientReplicas() - reduceTransients, 0); + ReplicationFactor.validate(rfLeft, transients); } /** - * Attempts to add an endpoint to the replicas for this datacenter, adding to the endpoints set if successful. + * Attempts to add an endpoint to the replicas for this datacenter, adding to the replicas set if successful. * Returns true if the endpoint was added, and this datacenter does not require further replicas. */ - boolean addEndpointAndCheckIfDone(InetAddressAndPort ep, Pair location) + boolean addEndpointAndCheckIfDone(InetAddressAndPort ep, Pair location, Range replicatedRange) { if (done()) return false; + if (replicas.endpoints().contains(ep)) + // Cannot repeat a node. + return false; + + Replica replica = new Replica(ep, replicatedRange, rfLeft > transients); + if (racks.add(location)) { // New rack. --rfLeft; - boolean added = endpoints.add(ep); - assert added; + replicas.add(replica, Conflict.NONE); return done(); } if (acceptableRackRepeats <= 0) // There must be rfLeft distinct racks left, do not add any more rack repeats. return false; - if (!endpoints.add(ep)) - // Cannot repeat a node. - return false; + + replicas.add(replica, Conflict.NONE); // Added a node that is from an already met rack to match RF when there aren't enough racks. --acceptableRackRepeats; --rfLeft; @@ -141,10 +160,15 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy /** * calculate endpoints in one pass through the tokens by tracking our progress in each DC. */ - public List calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata) + public EndpointsForRange calculateNaturalReplicas(Token searchToken, TokenMetadata tokenMetadata) { // we want to preserve insertion order so that the first added endpoint becomes primary - Set replicas = new LinkedHashSet<>(); + ArrayList sortedTokens = tokenMetadata.sortedTokens(); + Token replicaEnd = TokenMetadata.firstToken(sortedTokens, searchToken); + Token replicaStart = tokenMetadata.getPredecessor(replicaEnd); + Range replicatedRange = new Range<>(replicaStart, replicaEnd); + + EndpointsForRange.Mutable builder = new EndpointsForRange.Mutable(replicatedRange); Set> seenRacks = new HashSet<>(); Topology topology = tokenMetadata.getTopology(); @@ -158,31 +182,31 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy Map dcs = new HashMap<>(datacenters.size() * 2); // Create a DatacenterEndpoints object for each non-empty DC. - for (Map.Entry en : datacenters.entrySet()) + for (Map.Entry en : datacenters.entrySet()) { String dc = en.getKey(); - int rf = en.getValue(); + ReplicationFactor rf = en.getValue(); int nodeCount = sizeOrZero(allEndpoints.get(dc)); - if (rf <= 0 || nodeCount <= 0) + if (rf.allReplicas <= 0 || nodeCount <= 0) continue; - DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, replicas, seenRacks); + DatacenterEndpoints dcEndpoints = new DatacenterEndpoints(rf, sizeOrZero(racks.get(dc)), nodeCount, builder, seenRacks); dcs.put(dc, dcEndpoints); ++dcsToFill; } - Iterator tokenIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), searchToken, false); + Iterator tokenIter = TokenMetadata.ringIterator(sortedTokens, searchToken, false); while (dcsToFill > 0 && tokenIter.hasNext()) { Token next = tokenIter.next(); InetAddressAndPort ep = tokenMetadata.getEndpoint(next); Pair location = topology.getLocation(ep); DatacenterEndpoints dcEndpoints = dcs.get(location.left); - if (dcEndpoints != null && dcEndpoints.addEndpointAndCheckIfDone(ep, location)) + if (dcEndpoints != null && dcEndpoints.addEndpointAndCheckIfDone(ep, location, replicatedRange)) --dcsToFill; } - return new ArrayList<>(replicas); + return builder.asImmutableView(); } private int sizeOrZero(Multimap collection) @@ -195,18 +219,15 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy return collection != null ? collection.size() : 0; } - public int getReplicationFactor() + public ReplicationFactor getReplicationFactor() { - int total = 0; - for (int repFactor : datacenters.values()) - total += repFactor; - return total; + return aggregateRf; } - public int getReplicationFactor(String dc) + public ReplicationFactor getReplicationFactor(String dc) { - Integer replicas = datacenters.get(dc); - return replicas == null ? 0 : replicas; + ReplicationFactor replicas = datacenters.get(dc); + return replicas == null ? ReplicationFactor.ZERO : replicas; } public Set getDatacenters() diff --git a/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java b/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java index 93e629ed6e..449c51e28e 100644 --- a/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java +++ b/src/java/org/apache/cassandra/locator/OldNetworkTopologyStrategy.java @@ -21,9 +21,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Collection; import java.util.Iterator; -import java.util.List; import java.util.Map; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.Token; @@ -36,27 +36,32 @@ import org.apache.cassandra.dht.Token; */ public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy { + private final ReplicationFactor rf; public OldNetworkTopologyStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map configOptions) { super(keyspaceName, tokenMetadata, snitch, configOptions); + this.rf = ReplicationFactor.fromString(this.configOptions.get("replication_factor")); } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - int replicas = getReplicationFactor(); - List endpoints = new ArrayList<>(replicas); ArrayList tokens = metadata.sortedTokens(); - if (tokens.isEmpty()) - return endpoints; + return EndpointsForRange.empty(new Range<>(metadata.partitioner.getMinimumToken(), metadata.partitioner.getMinimumToken())); Iterator iter = TokenMetadata.ringIterator(tokens, token, false); Token primaryToken = iter.next(); - endpoints.add(metadata.getEndpoint(primaryToken)); + Token previousToken = metadata.getPredecessor(primaryToken); + Range tokenRange = new Range<>(previousToken, primaryToken); + + EndpointsForRange.Builder replicas = EndpointsForRange.builder(tokenRange, rf.allReplicas); + + assert !rf.hasTransientReplicas() : "support transient replicas"; + replicas.add(new Replica(metadata.getEndpoint(primaryToken), previousToken, primaryToken, true)); boolean bDataCenter = false; boolean bOtherRack = false; - while (endpoints.size() < replicas && iter.hasNext()) + while (replicas.size() < rf.allReplicas && iter.hasNext()) { // First try to find one in a different data center Token t = iter.next(); @@ -65,7 +70,7 @@ public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy // If we have already found something in a diff datacenter no need to find another if (!bDataCenter) { - endpoints.add(metadata.getEndpoint(t)); + replicas.add(new Replica(metadata.getEndpoint(t), previousToken, primaryToken, true)); bDataCenter = true; } continue; @@ -77,7 +82,7 @@ public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy // If we have already found something in a diff rack no need to find another if (!bOtherRack) { - endpoints.add(metadata.getEndpoint(t)); + replicas.add(new Replica(metadata.getEndpoint(t), previousToken, primaryToken, true)); bOtherRack = true; } } @@ -86,23 +91,24 @@ public class OldNetworkTopologyStrategy extends AbstractReplicationStrategy // If we found N number of nodes we are good. This loop wil just exit. Otherwise just // loop through the list and add until we have N nodes. - if (endpoints.size() < replicas) + if (replicas.size() < rf.allReplicas) { iter = TokenMetadata.ringIterator(tokens, token, false); - while (endpoints.size() < replicas && iter.hasNext()) + while (replicas.size() < rf.allReplicas && iter.hasNext()) { Token t = iter.next(); - if (!endpoints.contains(metadata.getEndpoint(t))) - endpoints.add(metadata.getEndpoint(t)); + Replica replica = new Replica(metadata.getEndpoint(t), previousToken, primaryToken, true); + if (!replicas.containsEndpoint(replica.endpoint())) + replicas.add(replica); } } - return endpoints; + return replicas.build(); } - public int getReplicationFactor() + public ReplicationFactor getReplicationFactor() { - return Integer.parseInt(this.configOptions.get("replication_factor")); + return rf; } public void validateOptions() throws ConfigurationException diff --git a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java index 92307a3980..b8b7bc60a0 100644 --- a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java +++ b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java @@ -23,166 +23,147 @@ package org.apache.cassandra.locator; import com.google.common.collect.Iterators; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import java.util.*; -public class PendingRangeMaps implements Iterable, List>> +public class PendingRangeMaps implements Iterable, EndpointsForRange.Mutable>> { - private static final Logger logger = LoggerFactory.getLogger(PendingRangeMaps.class); - /** * We have for NavigableMap to be able to search for ranges containing a token efficiently. * * First two are for non-wrap-around ranges, and the last two are for wrap-around ranges. */ // ascendingMap will sort the ranges by the ascending order of right token - final NavigableMap, List> ascendingMap; + private final NavigableMap, EndpointsForRange.Mutable> ascendingMap; + /** * sorting end ascending, if ends are same, sorting begin descending, so that token (end, end) will * come before (begin, end] with the same end, and (begin, end) will be selected in the tailMap. */ - static final Comparator> ascendingComparator = new Comparator>() - { - @Override - public int compare(Range o1, Range o2) - { - int res = o1.right.compareTo(o2.right); - if (res != 0) - return res; + private static final Comparator> ascendingComparator = (o1, o2) -> { + int res = o1.right.compareTo(o2.right); + if (res != 0) + return res; - return o2.left.compareTo(o1.left); - } - }; + return o2.left.compareTo(o1.left); + }; // ascendingMap will sort the ranges by the descending order of left token - final NavigableMap, List> descendingMap; + private final NavigableMap, EndpointsForRange.Mutable> descendingMap; + /** * sorting begin descending, if begins are same, sorting end descending, so that token (begin, begin) will * come after (begin, end] with the same begin, and (begin, end) won't be selected in the tailMap. */ - static final Comparator> descendingComparator = new Comparator>() - { - @Override - public int compare(Range o1, Range o2) - { - int res = o2.left.compareTo(o1.left); - if (res != 0) - return res; + private static final Comparator> descendingComparator = (o1, o2) -> { + int res = o2.left.compareTo(o1.left); + if (res != 0) + return res; - // if left tokens are same, sort by the descending of the right tokens. - return o2.right.compareTo(o1.right); - } - }; + // if left tokens are same, sort by the descending of the right tokens. + return o2.right.compareTo(o1.right); + }; // these two maps are for warp around ranges. - final NavigableMap, List> ascendingMapForWrapAround; + private final NavigableMap, EndpointsForRange.Mutable> ascendingMapForWrapAround; + /** * for wrap around range (begin, end], which begin > end. * Sorting end ascending, if ends are same, sorting begin ascending, * so that token (end, end) will come before (begin, end] with the same end, and (begin, end] will be selected in * the tailMap. */ - static final Comparator> ascendingComparatorForWrapAround = new Comparator>() - { - @Override - public int compare(Range o1, Range o2) - { - int res = o1.right.compareTo(o2.right); - if (res != 0) - return res; + private static final Comparator> ascendingComparatorForWrapAround = (o1, o2) -> { + int res = o1.right.compareTo(o2.right); + if (res != 0) + return res; - return o1.left.compareTo(o2.left); - } + return o1.left.compareTo(o2.left); }; - final NavigableMap, List> descendingMapForWrapAround; + private final NavigableMap, EndpointsForRange.Mutable> descendingMapForWrapAround; + /** * for wrap around ranges, which begin > end. * Sorting end ascending, so that token (begin, begin) will come after (begin, end] with the same begin, * and (begin, end) won't be selected in the tailMap. */ - static final Comparator> descendingComparatorForWrapAround = new Comparator>() - { - @Override - public int compare(Range o1, Range o2) - { - int res = o2.left.compareTo(o1.left); - if (res != 0) - return res; - return o1.right.compareTo(o2.right); - } + private static final Comparator> descendingComparatorForWrapAround = (o1, o2) -> { + int res = o2.left.compareTo(o1.left); + if (res != 0) + return res; + return o1.right.compareTo(o2.right); }; public PendingRangeMaps() { - this.ascendingMap = new TreeMap, List>(ascendingComparator); - this.descendingMap = new TreeMap, List>(descendingComparator); - this.ascendingMapForWrapAround = new TreeMap, List>(ascendingComparatorForWrapAround); - this.descendingMapForWrapAround = new TreeMap, List>(descendingComparatorForWrapAround); + this.ascendingMap = new TreeMap<>(ascendingComparator); + this.descendingMap = new TreeMap<>(descendingComparator); + this.ascendingMapForWrapAround = new TreeMap<>(ascendingComparatorForWrapAround); + this.descendingMapForWrapAround = new TreeMap<>(descendingComparatorForWrapAround); } static final void addToMap(Range range, - InetAddressAndPort address, - NavigableMap, List> ascendingMap, - NavigableMap, List> descendingMap) + Replica replica, + NavigableMap, EndpointsForRange.Mutable> ascendingMap, + NavigableMap, EndpointsForRange.Mutable> descendingMap) { - List addresses = ascendingMap.get(range); - if (addresses == null) + EndpointsForRange.Mutable replicas = ascendingMap.get(range); + if (replicas == null) { - addresses = new ArrayList<>(1); - ascendingMap.put(range, addresses); - descendingMap.put(range, addresses); + replicas = new EndpointsForRange.Mutable(range,1); + ascendingMap.put(range, replicas); + descendingMap.put(range, replicas); } - addresses.add(address); + replicas.add(replica, Conflict.DUPLICATE); } - public void addPendingRange(Range range, InetAddressAndPort address) + public void addPendingRange(Range range, Replica replica) { if (Range.isWrapAround(range.left, range.right)) { - addToMap(range, address, ascendingMapForWrapAround, descendingMapForWrapAround); + addToMap(range, replica, ascendingMapForWrapAround, descendingMapForWrapAround); } else { - addToMap(range, address, ascendingMap, descendingMap); + addToMap(range, replica, ascendingMap, descendingMap); } } - static final void addIntersections(Set endpointsToAdd, - NavigableMap, List> smallerMap, - NavigableMap, List> biggerMap) + static final void addIntersections(EndpointsForToken.Builder replicasToAdd, + NavigableMap, EndpointsForRange.Mutable> smallerMap, + NavigableMap, EndpointsForRange.Mutable> biggerMap) { // find the intersection of two sets for (Range range : smallerMap.keySet()) { - List addresses = biggerMap.get(range); - if (addresses != null) + EndpointsForRange.Mutable replicas = biggerMap.get(range); + if (replicas != null) { - endpointsToAdd.addAll(addresses); + replicasToAdd.addAll(replicas); } } } - public Collection pendingEndpointsFor(Token token) + public EndpointsForToken pendingEndpointsFor(Token token) { - Set endpoints = new HashSet<>(); + EndpointsForToken.Builder replicas = EndpointsForToken.builder(token); - Range searchRange = new Range(token, token); + Range searchRange = new Range<>(token, token); // search for non-wrap-around maps - NavigableMap, List> ascendingTailMap = ascendingMap.tailMap(searchRange, true); - NavigableMap, List> descendingTailMap = descendingMap.tailMap(searchRange, false); + NavigableMap, EndpointsForRange.Mutable> ascendingTailMap = ascendingMap.tailMap(searchRange, true); + NavigableMap, EndpointsForRange.Mutable> descendingTailMap = descendingMap.tailMap(searchRange, false); // add intersections of two maps if (ascendingTailMap.size() < descendingTailMap.size()) { - addIntersections(endpoints, ascendingTailMap, descendingTailMap); + addIntersections(replicas, ascendingTailMap, descendingTailMap); } else { - addIntersections(endpoints, descendingTailMap, ascendingTailMap); + addIntersections(replicas, descendingTailMap, ascendingTailMap); } // search for wrap-around sets @@ -190,29 +171,29 @@ public class PendingRangeMaps implements Iterable, List, List> entry : ascendingTailMap.entrySet()) + for (Map.Entry, EndpointsForRange.Mutable> entry : ascendingTailMap.entrySet()) { - endpoints.addAll(entry.getValue()); + replicas.addAll(entry.getValue()); } - for (Map.Entry, List> entry : descendingTailMap.entrySet()) + for (Map.Entry, EndpointsForRange.Mutable> entry : descendingTailMap.entrySet()) { - endpoints.addAll(entry.getValue()); + replicas.addAll(entry.getValue()); } - return endpoints; + return replicas.build(); } public String printPendingRanges() { StringBuilder sb = new StringBuilder(); - for (Map.Entry, List> entry : this) + for (Map.Entry, EndpointsForRange.Mutable> entry : this) { Range range = entry.getKey(); - for (InetAddressAndPort address : entry.getValue()) + for (Replica replica : entry.getValue()) { - sb.append(address).append(':').append(range); + sb.append(replica).append(':').append(range); sb.append(System.getProperty("line.separator")); } } @@ -221,7 +202,7 @@ public class PendingRangeMaps implements Iterable, List, List>> iterator() + public Iterator, EndpointsForRange.Mutable>> iterator() { return Iterators.concat(ascendingMap.entrySet().iterator(), ascendingMapForWrapAround.entrySet().iterator()); } diff --git a/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java b/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java new file mode 100644 index 0000000000..74828adc68 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/RangesAtEndpoint.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +import static org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict.*; + +/** + * A ReplicaCollection for Ranges occurring at an endpoint. All Replica will be for the same endpoint, + * and must be unique Ranges (though overlapping ranges are presently permitted, these should probably not be permitted to occur) + */ +public class RangesAtEndpoint extends AbstractReplicaCollection +{ + private static final Map, Replica> EMPTY_MAP = Collections.unmodifiableMap(new LinkedHashMap<>()); + + private final InetAddressAndPort endpoint; + private volatile Map, Replica> byRange; + private volatile RangesAtEndpoint fullRanges; + private volatile RangesAtEndpoint transRanges; + + private RangesAtEndpoint(InetAddressAndPort endpoint, List list, boolean isSnapshot) + { + this(endpoint, list, isSnapshot, null); + } + private RangesAtEndpoint(InetAddressAndPort endpoint, List list, boolean isSnapshot, Map, Replica> byRange) + { + super(list, isSnapshot); + this.endpoint = endpoint; + this.byRange = byRange; + assert endpoint != null; + } + + public InetAddressAndPort endpoint() + { + return endpoint; + } + + @Override + public Set endpoints() + { + return Collections.unmodifiableSet(list.isEmpty() + ? Collections.emptySet() + : Collections.singleton(endpoint) + ); + } + + public Set> ranges() + { + return byRange().keySet(); + } + + public Map, Replica> byRange() + { + Map, Replica> map = byRange; + if (map == null) + byRange = map = buildByRange(list); + return map; + } + + @Override + protected RangesAtEndpoint snapshot(List subList) + { + if (subList.isEmpty()) return empty(endpoint); + return new RangesAtEndpoint(endpoint, subList, true); + } + + @Override + public RangesAtEndpoint self() + { + return this; + } + + @Override + public ReplicaCollection.Mutable newMutable(int initialCapacity) + { + return new Mutable(endpoint, initialCapacity); + } + + @Override + public boolean contains(Replica replica) + { + return replica != null + && Objects.equals( + byRange().get(replica.range()), + replica); + } + + public RangesAtEndpoint full() + { + RangesAtEndpoint coll = fullRanges; + if (fullRanges == null) + fullRanges = coll = filter(Replica::isFull); + return coll; + } + + public RangesAtEndpoint trans() + { + RangesAtEndpoint coll = transRanges; + if (transRanges == null) + transRanges = coll = filter(Replica::isTransient); + return coll; + } + + public Collection> fullRanges() + { + return full().ranges(); + } + + public Collection> transientRanges() + { + return trans().ranges(); + } + + public boolean contains(Range range, boolean isFull) + { + Replica replica = byRange().get(range); + return replica != null && replica.isFull() == isFull; + } + + private static Map, Replica> buildByRange(List list) + { + // TODO: implement a delegating map that uses our superclass' list, and is immutable + Map, Replica> byRange = new LinkedHashMap<>(list.size()); + for (Replica replica : list) + { + Replica prev = byRange.put(replica.range(), replica); + assert prev == null : "duplicate range in RangesAtEndpoint: " + prev + " and " + replica; + } + + return Collections.unmodifiableMap(byRange); + } + + public static Collector collector(InetAddressAndPort endpoint) + { + return collector(ImmutableSet.of(), () -> new Builder(endpoint)); + } + + public static class Mutable extends RangesAtEndpoint implements ReplicaCollection.Mutable + { + boolean hasSnapshot; + public Mutable(InetAddressAndPort endpoint) { this(endpoint, 0); } + public Mutable(InetAddressAndPort endpoint, int capacity) { super(endpoint, new ArrayList<>(capacity), false, new LinkedHashMap<>()); } + + public void add(Replica replica, Conflict ignoreConflict) + { + if (hasSnapshot) throw new IllegalStateException(); + Preconditions.checkNotNull(replica); + if (!Objects.equals(super.endpoint, replica.endpoint())) + throw new IllegalArgumentException("Replica " + replica + " has incorrect endpoint (expected " + super.endpoint + ")"); + + Replica prev = super.byRange.put(replica.range(), replica); + if (prev != null) + { + super.byRange.put(replica.range(), prev); // restore prev + switch (ignoreConflict) + { + case DUPLICATE: + if (prev.equals(replica)) + break; + case NONE: + throw new IllegalArgumentException("Conflicting replica added (expected unique ranges): " + replica + "; existing: " + prev); + case ALL: + } + return; + } + + list.add(replica); + } + + @Override + public Map, Replica> byRange() + { + // our internal map is modifiable, but it is unsafe to modify the map externally + // it would be possible to implement a safe modifiable map, but it is probably not valuable + return Collections.unmodifiableMap(super.byRange()); + } + + public RangesAtEndpoint get(boolean isSnapshot) + { + return new RangesAtEndpoint(super.endpoint, super.list, isSnapshot, Collections.unmodifiableMap(super.byRange)); + } + + public RangesAtEndpoint asImmutableView() + { + return get(false); + } + + public RangesAtEndpoint asSnapshot() + { + hasSnapshot = true; + return get(true); + } + } + + public static class Builder extends ReplicaCollection.Builder + { + public Builder(InetAddressAndPort endpoint) { this(endpoint, 0); } + public Builder(InetAddressAndPort endpoint, int capacity) { super(new Mutable(endpoint, capacity)); } + } + + public static RangesAtEndpoint.Builder builder(InetAddressAndPort endpoint) + { + return new RangesAtEndpoint.Builder(endpoint); + } + + public static RangesAtEndpoint.Builder builder(InetAddressAndPort endpoint, int capacity) + { + return new RangesAtEndpoint.Builder(endpoint, capacity); + } + + public static RangesAtEndpoint empty(InetAddressAndPort endpoint) + { + return new RangesAtEndpoint(endpoint, EMPTY_LIST, true, EMPTY_MAP); + } + + public static RangesAtEndpoint of(Replica replica) + { + ArrayList one = new ArrayList<>(1); + one.add(replica); + return new RangesAtEndpoint(replica.endpoint(), one, true, Collections.unmodifiableMap(Collections.singletonMap(replica.range(), replica))); + } + + public static RangesAtEndpoint of(Replica ... replicas) + { + return copyOf(Arrays.asList(replicas)); + } + + public static RangesAtEndpoint copyOf(List replicas) + { + if (replicas.isEmpty()) + throw new IllegalArgumentException("Must specify a non-empty collection of replicas"); + return builder(replicas.get(0).endpoint(), replicas.size()).addAll(replicas).build(); + } + + + /** + * Use of this method to synthesize Replicas is almost always wrong. In repair it turns out the concerns of transient + * vs non-transient are handled at a higher level, but eventually repair needs to ask streaming to actually move + * the data and at that point it doesn't have a great handle on what the replicas are and it doesn't really matter. + * + * Streaming expects to be given Replicas with each replica indicating what type of data (transient or not transient) + * should be sent. + * + * So in this one instance we can lie to streaming and pretend all the replicas are full and use a dummy address + * and it doesn't matter because streaming doesn't rely on the address for anything other than debugging and full + * is a valid value for transientness because streaming is selecting candidate tables from the repair/unrepaired + * set already. + * @param ranges + * @return + */ + @VisibleForTesting + public static RangesAtEndpoint toDummyList(Collection> ranges) + { + InetAddressAndPort dummy; + try + { + dummy = InetAddressAndPort.getByNameOverrideDefaults("0.0.0.0", 0); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + + //For repair we are less concerned with full vs transient since repair is already dealing with those concerns. + //Always say full and then if the repair is incremental or not will determine what is streamed. + return ranges.stream() + .map(range -> new Replica(dummy, range, true)) + .collect(collector(dummy)); + } + + /** + * @return concatenate two DISJOINT collections together + */ + public static RangesAtEndpoint concat(RangesAtEndpoint replicas, RangesAtEndpoint extraReplicas) + { + return AbstractReplicaCollection.concat(replicas, extraReplicas, NONE); + } + +} diff --git a/src/java/org/apache/cassandra/locator/RangesByEndpoint.java b/src/java/org/apache/cassandra/locator/RangesByEndpoint.java new file mode 100644 index 0000000000..698b1334d3 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/RangesByEndpoint.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; + +import java.util.Collections; +import java.util.Map; + +public class RangesByEndpoint extends ReplicaMultimap +{ + public RangesByEndpoint(Map map) + { + super(map); + } + + public RangesAtEndpoint get(InetAddressAndPort endpoint) + { + Preconditions.checkNotNull(endpoint); + return map.getOrDefault(endpoint, RangesAtEndpoint.empty(endpoint)); + } + + public static class Mutable extends ReplicaMultimap.Mutable + { + @Override + protected RangesAtEndpoint.Mutable newMutable(InetAddressAndPort endpoint) + { + return new RangesAtEndpoint.Mutable(endpoint); + } + + public RangesByEndpoint asImmutableView() + { + return new RangesByEndpoint(Collections.unmodifiableMap(Maps.transformValues(map, RangesAtEndpoint.Mutable::asImmutableView))); + } + } + +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/locator/Replica.java b/src/java/org/apache/cassandra/locator/Replica.java new file mode 100644 index 0000000000..37b6050b8e --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Replica.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.Objects; +import java.util.Set; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.utils.FBUtilities; + +/** + * A Replica represents an owning node for a copy of a portion of the token ring. + * + * It consists of: + * - the logical token range that is being replicated (i.e. for the first logical replica only, this will be equal + * to one of its owned portions of the token ring; all other replicas will have this token range also) + * - an endpoint (IP and port) + * - whether the range is replicated in full, or transiently (CASSANDRA-14404) + * + * In general, it is preferred to use a Replica to a Range<Token>, particularly when users of the concept depend on + * knowledge of the full/transient status of the copy. + * + * That means you should avoid unwrapping and rewrapping these things and think hard about subtraction + * and such and what the result is WRT to transientness. Definitely avoid creating fake Replicas with misinformation + * about endpoints, ranges, or transientness. + */ +public final class Replica implements Comparable +{ + private final Range range; + private final InetAddressAndPort endpoint; + private final boolean full; + + public Replica(InetAddressAndPort endpoint, Range range, boolean full) + { + Preconditions.checkNotNull(endpoint); + Preconditions.checkNotNull(range); + this.endpoint = endpoint; + this.range = range; + this.full = full; + } + + public Replica(InetAddressAndPort endpoint, Token start, Token end, boolean full) + { + this(endpoint, new Range<>(start, end), full); + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Replica replica = (Replica) o; + return full == replica.full && + Objects.equals(endpoint, replica.endpoint) && + Objects.equals(range, replica.range); + } + + @Override + public int compareTo(Replica o) + { + int c = range.compareTo(o.range); + if (c == 0) + c = endpoint.compareTo(o.endpoint); + if (c == 0) + c = Boolean.compare(full, o.full); + return c; + } + + public int hashCode() + { + return Objects.hash(endpoint, range, full); + } + + @Override + public String toString() + { + return (full ? "Full" : "Transient") + '(' + endpoint() + ',' + range + ')'; + } + + public final InetAddressAndPort endpoint() + { + return endpoint; + } + + public boolean isLocal() + { + return endpoint.equals(FBUtilities.getBroadcastAddressAndPort()); + } + + public Range range() + { + return range; + } + + public boolean isFull() + { + return full; + } + + public final boolean isTransient() + { + return !isFull(); + } + + /** + * This is used exclusively in TokenMetadata to check if a portion of a range is already replicated + * by an endpoint so that we only mark as pending the portion that is either not replicated sufficiently (transient + * when we need full) or at all. + * + * If it's not replicated at all it needs to be pending because there is no data. + * If it's replicated but only transiently and we need to replicate it fully it must be marked as pending until it + * is available fully otherwise a read might treat this replica as full and not read from a full replica that has + * the data. + */ + public RangesAtEndpoint subtractSameReplication(RangesAtEndpoint toSubtract) + { + Set> subtractedRanges = range().subtractAll(toSubtract.filter(r -> r.isFull() == isFull()).ranges()); + RangesAtEndpoint.Builder result = RangesAtEndpoint.builder(endpoint, subtractedRanges.size()); + for (Range range : subtractedRanges) + { + result.add(decorateSubrange(range)); + } + return result.build(); + } + + /** + * Don't use this method and ignore transient status unless you are explicitly handling it outside this method. + * + * This helper method is used by StorageService.calculateStreamAndFetchRanges to perform subtraction. + * It ignores transient status because it's already being handled in calculateStreamAndFetchRanges. + */ + public RangesAtEndpoint subtractIgnoreTransientStatus(Range subtract) + { + Set> ranges = this.range.subtract(subtract); + RangesAtEndpoint.Builder result = RangesAtEndpoint.builder(endpoint, ranges.size()); + for (Range subrange : ranges) + result.add(decorateSubrange(subrange)); + return result.build(); + } + + public boolean contains(Range that) + { + return range().contains(that); + } + + public boolean intersectsOnRange(Replica replica) + { + return range().intersects(replica.range()); + } + + public Replica decorateSubrange(Range subrange) + { + Preconditions.checkArgument(range.contains(subrange)); + return new Replica(endpoint(), subrange, isFull()); + } + + public static Replica fullReplica(InetAddressAndPort endpoint, Range range) + { + return new Replica(endpoint, range, true); + } + + public static Replica fullReplica(InetAddressAndPort endpoint, Token start, Token end) + { + return fullReplica(endpoint, new Range<>(start, end)); + } + + public static Replica transientReplica(InetAddressAndPort endpoint, Range range) + { + return new Replica(endpoint, range, false); + } + + public static Replica transientReplica(InetAddressAndPort endpoint, Token start, Token end) + { + return transientReplica(endpoint, new Range<>(start, end)); + } + +} + diff --git a/src/java/org/apache/cassandra/locator/ReplicaCollection.java b/src/java/org/apache/cassandra/locator/ReplicaCollection.java new file mode 100644 index 0000000000..6833f4b992 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/ReplicaCollection.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Stream; + +/** + * A collection like class for Replica objects. Represents both a well defined order on the contained Replica objects, + * and efficient methods for accessing the contained Replicas, directly and as a projection onto their endpoints and ranges. + */ +public interface ReplicaCollection> extends Iterable +{ + /** + * @return a Set of the endpoints of the contained Replicas. + * Iteration order is maintained where there is a 1:1 relationship between endpoint and Replica + * Typically this collection offers O(1) access methods, and this is true for all but ReplicaList. + */ + public abstract Set endpoints(); + + /** + * @param i a value in the range [0..size()) + * @return the i'th Replica, in our iteration order + */ + public abstract Replica get(int i); + + /** + * @return the number of Replica contained + */ + public abstract int size(); + + /** + * @return true iff size() == 0 + */ + public abstract boolean isEmpty(); + + /** + * @return true iff a Replica in this collection is equal to the provided Replica. + * Typically this method is expected to take O(1) time, and this is true for all but ReplicaList. + */ + public abstract boolean contains(Replica replica); + + /** + * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate. + * An effort will be made to either return ourself, or a subList, where possible. + * It is guaranteed that no changes to any upstream Mutable will affect the state of the result. + */ + public abstract C filter(Predicate predicate); + + /** + * @return a *eagerly constructed* copy of this collection containing the Replica that match the provided predicate. + * An effort will be made to either return ourself, or a subList, where possible. + * It is guaranteed that no changes to any upstream Mutable will affect the state of the result. + * Only the first maxSize items will be returned. + */ + public abstract C filter(Predicate predicate, int maxSize); + + /** + * @return an *eagerly constructed* copy of this collection containing the Replica at positions [start..end); + * An effort will be made to either return ourself, or a subList, where possible. + * It is guaranteed that no changes to any upstream Mutable will affect the state of the result. + */ + public abstract C subList(int start, int end); + + /** + * @return an *eagerly constructed* copy of this collection containing the Replica re-ordered according to this comparator + * It is guaranteed that no changes to any upstream Mutable will affect the state of the result. + */ + public abstract C sorted(Comparator comparator); + + public abstract Iterator iterator(); + public abstract Stream stream(); + + public abstract boolean equals(Object o); + public abstract int hashCode(); + public abstract String toString(); + + /** + * A mutable extension of a ReplicaCollection. This is append-only, so it is safe to select a subList, + * or at any time take an asImmutableView() snapshot. + */ + public interface Mutable> extends ReplicaCollection + { + /** + * @return an Immutable clone that mirrors any modifications to this Mutable instance. + */ + C asImmutableView(); + + /** + * @return an Immutable clone that assumes this Mutable will never be modified again. + * If this is not true, behaviour is undefined. + */ + C asSnapshot(); + + enum Conflict { NONE, DUPLICATE, ALL} + + /** + * @param replica add this replica to the end of the collection + * @param ignoreConflict if false, fail on any conflicting additions (as defined by C's semantics) + */ + void add(Replica replica, Conflict ignoreConflict); + + default public void add(Replica replica) + { + add(replica, Conflict.NONE); + } + + default public void addAll(Iterable replicas, Conflict ignoreConflicts) + { + for (Replica replica : replicas) + add(replica, ignoreConflicts); + } + + default public void addAll(Iterable replicas) + { + addAll(replicas, Conflict.NONE); + } + } + + public static class Builder, M extends Mutable, B extends Builder> + { + Mutable mutable; + public Builder(Mutable mutable) { this.mutable = mutable; } + + public int size() { return mutable.size(); } + public B add(Replica replica) { mutable.add(replica); return (B) this; } + public B add(Replica replica, Conflict ignoreConflict) { mutable.add(replica, ignoreConflict); return (B) this; } + public B addAll(Iterable replica) { mutable.addAll(replica); return (B) this; } + public B addAll(Iterable replica, Conflict ignoreConflict) { mutable.addAll(replica, ignoreConflict); return (B) this; } + + public C build() + { + C result = mutable.asSnapshot(); + mutable = null; + return result; + } + } + +} diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java new file mode 100644 index 0000000000..946a7f8f9f --- /dev/null +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Predicates; +import com.google.common.collect.Iterables; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.net.IAsyncCallback; +import org.apache.cassandra.service.StorageProxy; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.utils.FBUtilities; + +import static com.google.common.collect.Iterables.any; + +/** + * Encapsulates knowledge about the ring necessary for performing a specific operation, with static accessors + * for building the relevant layout. + * + * Constitutes: + * - the 'natural' replicas replicating the range or token relevant for the operation + * - if for performing a write, any 'pending' replicas that are taking ownership of the range, and must receive updates + * - the 'selected' replicas, those that should be targeted for any operation + * - 'all' replicas represents natural+pending + * + * @param the type of Endpoints this ReplayLayout holds (either EndpointsForToken or EndpointsForRange) + * @param the type of itself, including its type parameters, for return type of modifying methods + */ +public abstract class ReplicaLayout, L extends ReplicaLayout> +{ + private volatile E all; + protected final E natural; + protected final E pending; + protected final E selected; + + protected final Keyspace keyspace; + protected final ConsistencyLevel consistencyLevel; + + private ReplicaLayout(Keyspace keyspace, ConsistencyLevel consistencyLevel, E natural, E pending, E selected) + { + this(keyspace, consistencyLevel, natural, pending, selected, null); + } + + private ReplicaLayout(Keyspace keyspace, ConsistencyLevel consistencyLevel, E natural, E pending, E selected, E all) + { + assert selected != null; + assert pending == null || !Endpoints.haveConflicts(natural, pending); + this.keyspace = keyspace; + this.consistencyLevel = consistencyLevel; + this.natural = natural; + this.pending = pending; + this.selected = selected; + // if we logically have no pending endpoints (they are null), then 'all' our endpoints are natural + if (all == null && pending == null) + all = natural; + this.all = all; + } + + public Replica getReplicaFor(InetAddressAndPort endpoint) + { + return natural.byEndpoint().get(endpoint); + } + + public E natural() + { + return natural; + } + + public E all() + { + E result = all; + if (result == null) + all = result = Endpoints.concat(natural, pending); + return result; + } + + public E selected() + { + return selected; + } + + /** + * @return the pending replicas - will be null for read layouts + * TODO: ideally we would enforce at compile time that read layouts have no pending to access + */ + public E pending() + { + return pending; + } + + public int blockFor() + { + return pending == null + ? consistencyLevel.blockFor(keyspace) + : consistencyLevel.blockForWrite(keyspace, pending); + } + + public Keyspace keyspace() + { + return keyspace; + } + + public ConsistencyLevel consistencyLevel() + { + return consistencyLevel; + } + + abstract public L withSelected(E replicas); + + abstract public L withConsistencyLevel(ConsistencyLevel cl); + + public L forNaturalUncontacted() + { + E more; + if (consistencyLevel.isDatacenterLocal() && keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) + { + IEndpointSnitch snitch = keyspace.getReplicationStrategy().snitch; + String localDC = DatabaseDescriptor.getLocalDataCenter(); + + more = natural.filter(replica -> !selected.contains(replica) && + snitch.getDatacenter(replica).equals(localDC)); + } else + { + more = natural.filter(replica -> !selected.contains(replica)); + } + + return withSelected(more); + } + + public static class ForRange extends ReplicaLayout + { + public final AbstractBounds range; + + @VisibleForTesting + public ForRange(Keyspace keyspace, ConsistencyLevel consistencyLevel, AbstractBounds range, EndpointsForRange natural, EndpointsForRange selected) + { + // Range queries do not contact pending replicas + super(keyspace, consistencyLevel, natural, null, selected); + this.range = range; + } + + @Override + public ForRange withSelected(EndpointsForRange newSelected) + { + return new ForRange(keyspace, consistencyLevel, range, natural, newSelected); + } + + @Override + public ForRange withConsistencyLevel(ConsistencyLevel cl) + { + return new ForRange(keyspace, cl, range, natural, selected); + } + } + + public static class ForToken extends ReplicaLayout + { + public final Token token; + + @VisibleForTesting + public ForToken(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, EndpointsForToken natural, EndpointsForToken pending, EndpointsForToken selected) + { + super(keyspace, consistencyLevel, natural, pending, selected); + this.token = token; + } + + public ForToken(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, EndpointsForToken natural, EndpointsForToken pending, EndpointsForToken selected, EndpointsForToken all) + { + super(keyspace, consistencyLevel, natural, pending, selected, all); + this.token = token; + } + + public ForToken withSelected(EndpointsForToken newSelected) + { + return new ForToken(keyspace, consistencyLevel, token, natural, pending, newSelected); + } + + @Override + public ForToken withConsistencyLevel(ConsistencyLevel cl) + { + return new ForToken(keyspace, cl, token, natural, pending, selected); + } + } + + public static class ForPaxos extends ForToken + { + private final int requiredParticipants; + + private ForPaxos(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, int requiredParticipants, EndpointsForToken natural, EndpointsForToken pending, EndpointsForToken selected, EndpointsForToken all) + { + super(keyspace, consistencyLevel, token, natural, pending, selected, all); + this.requiredParticipants = requiredParticipants; + } + + public int getRequiredParticipants() + { + return requiredParticipants; + } + } + + public static ForToken forSingleReplica(Keyspace keyspace, Token token, Replica replica) + { + EndpointsForToken singleReplica = EndpointsForToken.of(token, replica); + return new ForToken(keyspace, ConsistencyLevel.ONE, token, singleReplica, EndpointsForToken.empty(token), singleReplica, singleReplica); + } + + public static ForRange forSingleReplica(Keyspace keyspace, AbstractBounds range, Replica replica) + { + EndpointsForRange singleReplica = EndpointsForRange.of(replica); + return new ForRange(keyspace, ConsistencyLevel.ONE, range, singleReplica, singleReplica); + } + + public static ForToken forCounterWrite(Keyspace keyspace, Token token, Replica replica) + { + return forSingleReplica(keyspace, token, replica); + } + + public static ForToken forBatchlogWrite(Keyspace keyspace, Collection endpoints) throws UnavailableException + { + // A single case we write not for range or token, but multiple mutations to many tokens + Token token = DatabaseDescriptor.getPartitioner().getMinimumToken(); + EndpointsForToken natural = EndpointsForToken.copyOf(token, SystemReplicas.getSystemReplicas(endpoints)); + EndpointsForToken pending = EndpointsForToken.empty(token); + ConsistencyLevel consistencyLevel = natural.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO; + + return forWriteWithDownNodes(keyspace, consistencyLevel, token, natural, pending); + } + + public static ForToken forWriteWithDownNodes(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token) throws UnavailableException + { + return forWrite(keyspace, consistencyLevel, token, Predicates.alwaysTrue()); + } + + public static ForToken forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, Predicate isAlive) throws UnavailableException + { + EndpointsForToken natural = StorageService.getNaturalReplicasForToken(keyspace.getName(), token); + EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(token, keyspace.getName()); + return forWrite(keyspace, consistencyLevel, token, natural, pending, isAlive); + } + + public static ForToken forWriteWithDownNodes(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, EndpointsForToken natural, EndpointsForToken pending) throws UnavailableException + { + return forWrite(keyspace, consistencyLevel, token, natural, pending, Predicates.alwaysTrue()); + } + + public static ForToken forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, EndpointsForToken natural, EndpointsForToken pending, Predicate isAlive) throws UnavailableException + { + if (Endpoints.haveConflicts(natural, pending)) + { + natural = Endpoints.resolveConflictsInNatural(natural, pending); + pending = Endpoints.resolveConflictsInPending(natural, pending); + } + + if (!any(natural, Replica::isTransient) && !any(pending, Replica::isTransient)) + { + EndpointsForToken selected = Endpoints.concat(natural, pending).filter(r -> isAlive.test(r.endpoint())); + return new ForToken(keyspace, consistencyLevel, token, natural, pending, selected); + } + + return forWrite(keyspace, consistencyLevel, token, consistencyLevel.blockForWrite(keyspace, pending), natural, pending, isAlive); + } + + public static ReplicaLayout.ForPaxos forPaxos(Keyspace keyspace, DecoratedKey key, ConsistencyLevel consistencyForPaxos) throws UnavailableException + { + Token tk = key.getToken(); + EndpointsForToken natural = StorageService.getNaturalReplicasForToken(keyspace.getName(), tk); + EndpointsForToken pending = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(tk, keyspace.getName()); + if (Endpoints.haveConflicts(natural, pending)) + { + natural = Endpoints.resolveConflictsInNatural(natural, pending); + pending = Endpoints.resolveConflictsInPending(natural, pending); + } + + // TODO CASSANDRA-14547 + Replicas.temporaryAssertFull(natural); + Replicas.temporaryAssertFull(pending); + + if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL) + { + // Restrict natural and pending to node in the local DC only + String localDc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); + IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); + Predicate isLocalDc = replica -> localDc.equals(snitch.getDatacenter(replica)); + + natural = natural.filter(isLocalDc); + pending = pending.filter(isLocalDc); + } + + int participants = pending.size() + natural.size(); + int requiredParticipants = participants / 2 + 1; // See CASSANDRA-8346, CASSANDRA-833 + + EndpointsForToken all = Endpoints.concat(natural, pending); + EndpointsForToken selected = all.filter(IAsyncCallback.isReplicaAlive); + if (selected.size() < requiredParticipants) + throw UnavailableException.create(consistencyForPaxos, requiredParticipants, selected.size()); + + // We cannot allow CAS operations with 2 or more pending endpoints, see #8346. + // Note that we fake an impossible number of required nodes in the unavailable exception + // to nail home the point that it's an impossible operation no matter how many nodes are live. + if (pending.size() > 1) + throw new UnavailableException(String.format("Cannot perform LWT operation as there is more than one (%d) pending range movement", pending.size()), + consistencyForPaxos, + participants + 1, + selected.size()); + + return new ReplicaLayout.ForPaxos(keyspace, consistencyForPaxos, key.getToken(), requiredParticipants, natural, pending, selected, all); + } + + /** + * We want to send mutations to as many full replicas as we can, and just as many transient replicas + * as we need to meet blockFor. + */ + @VisibleForTesting + public static ForToken forWrite(Keyspace keyspace, ConsistencyLevel consistencyLevel, Token token, int blockFor, EndpointsForToken natural, EndpointsForToken pending, Predicate livePredicate) throws UnavailableException + { + EndpointsForToken all = Endpoints.concat(natural, pending); + EndpointsForToken selected = all + .select() + .add(r -> r.isFull() && livePredicate.test(r.endpoint())) + .add(r -> r.isTransient() && livePredicate.test(r.endpoint()), blockFor) + .get(); + + consistencyLevel.assureSufficientLiveNodesForWrite(keyspace, selected, pending); + + return new ForToken(keyspace, consistencyLevel, token, natural, pending, selected, all); + } + + public static ForToken forRead(Keyspace keyspace, Token token, ConsistencyLevel consistencyLevel, SpeculativeRetryPolicy retry) + { + EndpointsForToken natural = StorageProxy.getLiveSortedReplicasForToken(keyspace, token); + EndpointsForToken selected = consistencyLevel.filterForQuery(keyspace, natural, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE)); + + // Throw UAE early if we don't have enough replicas. + consistencyLevel.assureSufficientLiveNodesForRead(keyspace, selected); + + return new ForToken(keyspace, consistencyLevel, token, natural, null, selected); + } + + public static ForRange forRangeRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, AbstractBounds range, EndpointsForRange natural, EndpointsForRange selected) + { + return new ForRange(keyspace, consistencyLevel, range, natural, selected); + } + + public String toString() + { + return "ReplicaLayout [ CL: " + consistencyLevel + " keyspace: " + keyspace + " natural: " + natural + "pending: " + pending + " selected: " + selected + " ]"; + } +} + diff --git a/src/java/org/apache/cassandra/locator/ReplicaMultimap.java b/src/java/org/apache/cassandra/locator/ReplicaMultimap.java new file mode 100644 index 0000000000..3e3fcb4735 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/ReplicaMultimap.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; + +public abstract class ReplicaMultimap> +{ + final Map map; + ReplicaMultimap(Map map) + { + this.map = map; + } + + public abstract C get(K key); + public C getIfPresent(K key) { return map.get(key); } + + public static abstract class Mutable + > + extends ReplicaMultimap + { + protected abstract MutableCollection newMutable(K key); + + Mutable() + { + super(new HashMap<>()); + } + + public MutableCollection get(K key) + { + Preconditions.checkNotNull(key); + return map.computeIfAbsent(key, k -> newMutable(key)); + } + + public void put(K key, Replica replica) + { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(replica); + get(key).add(replica); + } + } + + public Iterable flattenValues() + { + return Iterables.concat(map.values()); + } + + public Iterable> flattenEntries() + { + return () -> { + Stream> s = map.entrySet() + .stream() + .flatMap(entry -> entry.getValue() + .stream() + .map(replica -> (Map.Entry)new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), replica))); + return s.iterator(); + }; + } + + public boolean isEmpty() + { + return map.isEmpty(); + } + + public boolean containsKey(Object key) + { + return map.containsKey(key); + } + + public Set keySet() + { + return map.keySet(); + } + + public Set> entrySet() + { + return map.entrySet(); + } + + public Map asMap() + { + return map; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReplicaMultimap that = (ReplicaMultimap) o; + return Objects.equals(map, that.map); + } + + public int hashCode() + { + return map.hashCode(); + } + + @Override + public String toString() + { + return map.toString(); + } +} diff --git a/src/java/org/apache/cassandra/locator/Replicas.java b/src/java/org/apache/cassandra/locator/Replicas.java new file mode 100644 index 0000000000..299e6ec31a --- /dev/null +++ b/src/java/org/apache/cassandra/locator/Replicas.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.Iterables; + +import static com.google.common.collect.Iterables.all; + +public class Replicas +{ + + public static int countFull(ReplicaCollection liveReplicas) + { + int count = 0; + for (Replica replica : liveReplicas) + if (replica.isFull()) + ++count; + return count; + } + + /** + * A placeholder for areas of the code that cannot yet handle transient replicas, but should do so in future + */ + public static void temporaryAssertFull(Replica replica) + { + if (!replica.isFull()) + { + throw new UnsupportedOperationException("transient replicas are currently unsupported: " + replica); + } + } + + /** + * A placeholder for areas of the code that cannot yet handle transient replicas, but should do so in future + */ + public static void temporaryAssertFull(Iterable replicas) + { + if (!all(replicas, Replica::isFull)) + { + throw new UnsupportedOperationException("transient replicas are currently unsupported: " + Iterables.toString(replicas)); + } + } + + /** + * For areas of the code that should never see a transient replica + */ + public static void assertFull(Iterable replicas) + { + if (!all(replicas, Replica::isFull)) + { + throw new UnsupportedOperationException("transient replicas are currently unsupported: " + Iterables.toString(replicas)); + } + } + + public static List stringify(ReplicaCollection replicas, boolean withPort) + { + List stringEndpoints = new ArrayList<>(replicas.size()); + for (Replica replica: replicas) + { + stringEndpoints.add(replica.endpoint().getHostAddress(withPort)); + } + return stringEndpoints; + } + +} diff --git a/src/java/org/apache/cassandra/locator/ReplicationFactor.java b/src/java/org/apache/cassandra/locator/ReplicationFactor.java new file mode 100644 index 0000000000..c0ed31f095 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/ReplicationFactor.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.utils.FBUtilities; + +public class ReplicationFactor +{ + public static final ReplicationFactor ZERO = new ReplicationFactor(0); + + public final int allReplicas; + public final int fullReplicas; + + private ReplicationFactor(int allReplicas, int transientReplicas) + { + validate(allReplicas, transientReplicas); + this.allReplicas = allReplicas; + this.fullReplicas = allReplicas - transientReplicas; + } + + public int transientReplicas() + { + return allReplicas - fullReplicas; + } + + public boolean hasTransientReplicas() + { + return allReplicas != fullReplicas; + } + + private ReplicationFactor(int allReplicas) + { + this(allReplicas, 0); + } + + static void validate(int totalRF, int transientRF) + { + Preconditions.checkArgument(transientRF == 0 || DatabaseDescriptor.isTransientReplicationEnabled(), + "Transient replication is not enabled on this node"); + Preconditions.checkArgument(totalRF >= 0, + "Replication factor must be non-negative, found %s", totalRF); + Preconditions.checkArgument(transientRF == 0 || transientRF < totalRF, + "Transient replicas must be zero, or less than total replication factor. For %s/%s", totalRF, transientRF); + if (transientRF > 0) + { + Preconditions.checkArgument(DatabaseDescriptor.getNumTokens() == 1, + "Transient nodes are not allowed with multiple tokens"); + Stream endpoints = Stream.concat(Gossiper.instance.getLiveMembers().stream(), Gossiper.instance.getUnreachableMembers().stream()); + List badVersionEndpoints = endpoints.filter(Predicates.not(FBUtilities.getBroadcastAddressAndPort()::equals)) + .filter(endpoint -> Gossiper.instance.getReleaseVersion(endpoint) != null && Gossiper.instance.getReleaseVersion(endpoint).major < 4) + .collect(Collectors.toList()); + if (!badVersionEndpoints.isEmpty()) + throw new AssertionError("Transient replication is not supported in mixed version clusters with nodes < 4.0. Bad nodes: " + badVersionEndpoints); + } + else if (transientRF < 0) + { + throw new AssertionError(String.format("Amount of transient nodes should be strictly positive, but was: '%d'", transientRF)); + } + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ReplicationFactor that = (ReplicationFactor) o; + return allReplicas == that.allReplicas && fullReplicas == that.fullReplicas; + } + + public int hashCode() + { + return Objects.hash(allReplicas, fullReplicas); + } + + public static ReplicationFactor fullOnly(int totalReplicas) + { + return new ReplicationFactor(totalReplicas); + } + + public static ReplicationFactor withTransient(int totalReplicas, int transientReplicas) + { + return new ReplicationFactor(totalReplicas, transientReplicas); + } + + public static ReplicationFactor fromString(String s) + { + if (s.contains("/")) + { + String[] parts = s.split("/"); + Preconditions.checkArgument(parts.length == 2, + "Replication factor format is or /"); + return new ReplicationFactor(Integer.valueOf(parts[0]), Integer.valueOf(parts[1])); + } + else + { + return new ReplicationFactor(Integer.valueOf(s), 0); + } + } + + @Override + public String toString() + { + return "rf(" + allReplicas + (hasTransientReplicas() ? '/' + transientReplicas() : "") + ')'; + } +} diff --git a/src/java/org/apache/cassandra/locator/SimpleSnitch.java b/src/java/org/apache/cassandra/locator/SimpleSnitch.java index e31fc6b153..d605b6e9dd 100644 --- a/src/java/org/apache/cassandra/locator/SimpleSnitch.java +++ b/src/java/org/apache/cassandra/locator/SimpleSnitch.java @@ -17,8 +17,6 @@ */ package org.apache.cassandra.locator; -import java.util.List; - /** * A simple endpoint snitch implementation that treats Strategy order as proximity, * allowing non-read-repaired reads to prefer a single endpoint, which improves @@ -37,12 +35,14 @@ public class SimpleSnitch extends AbstractEndpointSnitch } @Override - public void sortByProximity(final InetAddressAndPort address, List addresses) + public > C sortedByProximity(final InetAddressAndPort address, C unsortedAddress) { // Optimization to avoid walking the list + return unsortedAddress; } - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + @Override + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) { // Making all endpoints equal ensures we won't change the original ordering (since // Collections.sort is guaranteed to be stable) diff --git a/src/java/org/apache/cassandra/locator/SimpleStrategy.java b/src/java/org/apache/cassandra/locator/SimpleStrategy.java index 545ad28e0c..7a000b77bf 100644 --- a/src/java/org/apache/cassandra/locator/SimpleStrategy.java +++ b/src/java/org/apache/cassandra/locator/SimpleStrategy.java @@ -21,9 +21,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Collection; import java.util.Iterator; -import java.util.List; import java.util.Map; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.dht.Token; @@ -36,34 +36,41 @@ import org.apache.cassandra.dht.Token; */ public class SimpleStrategy extends AbstractReplicationStrategy { + private final ReplicationFactor rf; + public SimpleStrategy(String keyspaceName, TokenMetadata tokenMetadata, IEndpointSnitch snitch, Map configOptions) { super(keyspaceName, tokenMetadata, snitch, configOptions); + this.rf = ReplicationFactor.fromString(this.configOptions.get("replication_factor")); } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - int replicas = getReplicationFactor(); - ArrayList tokens = metadata.sortedTokens(); - List endpoints = new ArrayList(replicas); + ArrayList ring = metadata.sortedTokens(); + if (ring.isEmpty()) + return EndpointsForRange.empty(new Range<>(metadata.partitioner.getMinimumToken(), metadata.partitioner.getMinimumToken())); - if (tokens.isEmpty()) - return endpoints; + Token replicaEnd = TokenMetadata.firstToken(ring, token); + Token replicaStart = metadata.getPredecessor(replicaEnd); + Range replicaRange = new Range<>(replicaStart, replicaEnd); + Iterator iter = TokenMetadata.ringIterator(ring, token, false); + + EndpointsForRange.Builder replicas = EndpointsForRange.builder(replicaRange, rf.allReplicas); // Add the token at the index by default - Iterator iter = TokenMetadata.ringIterator(tokens, token, false); - while (endpoints.size() < replicas && iter.hasNext()) + while (replicas.size() < rf.allReplicas && iter.hasNext()) { - InetAddressAndPort ep = metadata.getEndpoint(iter.next()); - if (!endpoints.contains(ep)) - endpoints.add(ep); + Token tk = iter.next(); + InetAddressAndPort ep = metadata.getEndpoint(tk); + if (!replicas.containsEndpoint(ep)) + replicas.add(new Replica(ep, replicaRange, replicas.size() < rf.fullReplicas)); } - return endpoints; + return replicas.build(); } - public int getReplicationFactor() + public ReplicationFactor getReplicationFactor() { - return Integer.parseInt(this.configOptions.get("replication_factor")); + return rf; } public void validateOptions() throws ConfigurationException diff --git a/src/java/org/apache/cassandra/locator/SystemReplicas.java b/src/java/org/apache/cassandra/locator/SystemReplicas.java new file mode 100644 index 0000000000..13a9d743b0 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/SystemReplicas.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +public class SystemReplicas +{ + private static final Map systemReplicas = new ConcurrentHashMap<>(); + public static final Range FULL_RANGE = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), + DatabaseDescriptor.getPartitioner().getMinimumToken()); + + private static Replica createSystemReplica(InetAddressAndPort endpoint) + { + return new Replica(endpoint, FULL_RANGE, true); + } + + /** + * There are a few places where a system function borrows write path functionality, but doesn't otherwise + * fit into normal replication strategies (ie: hints and batchlog). So here we provide a replica instance + * @param endpoint + * @return + */ + public static Replica getSystemReplica(InetAddressAndPort endpoint) + { + return systemReplicas.computeIfAbsent(endpoint, SystemReplicas::createSystemReplica); + } + + public static Collection getSystemReplicas(Collection endpoints) + { + List replicas = new ArrayList<>(endpoints.size()); + for (InetAddressAndPort endpoint: endpoints) + { + replicas.add(getSystemReplica(endpoint)); + } + return replicas; + } +} diff --git a/src/java/org/apache/cassandra/locator/TokenMetadata.java b/src/java/org/apache/cassandra/locator/TokenMetadata.java index 46c191f60c..4ab34db1f5 100644 --- a/src/java/org/apache/cassandra/locator/TokenMetadata.java +++ b/src/java/org/apache/cassandra/locator/TokenMetadata.java @@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.*; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +88,7 @@ public class TokenMetadata // (don't need to record Token here since it's still part of tokenToEndpointMap until it's done leaving) private final Set leavingEndpoints = new HashSet<>(); // this is a cache of the calculation from {tokenToEndpointMap, bootstrapTokens, leavingEndpoints} + // NOTE: this may contain ranges that conflict with the those implied by sortedTokens when a range is changing its transient status private final ConcurrentMap pendingRanges = new ConcurrentHashMap(); // nodes which are migrating to the new tokens in the ring @@ -733,24 +735,20 @@ public class TokenMetadata return sortedTokens; } - public Multimap, InetAddressAndPort> getPendingRangesMM(String keyspaceName) + public EndpointsByRange getPendingRangesMM(String keyspaceName) { - Multimap, InetAddressAndPort> map = HashMultimap.create(); + EndpointsByRange.Mutable byRange = new EndpointsByRange.Mutable(); PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName); if (pendingRangeMaps != null) { - for (Map.Entry, List> entry : pendingRangeMaps) + for (Map.Entry, EndpointsForRange.Mutable> entry : pendingRangeMaps) { - Range range = entry.getKey(); - for (InetAddressAndPort address : entry.getValue()) - { - map.put(range, address); - } + byRange.putAll(entry.getKey(), entry.getValue(), Conflict.ALL); } } - return map; + return byRange.asImmutableView(); } /** a mutable map may be returned but caller should not modify it */ @@ -759,17 +757,18 @@ public class TokenMetadata return this.pendingRanges.get(keyspaceName); } - public List> getPendingRanges(String keyspaceName, InetAddressAndPort endpoint) + public RangesAtEndpoint getPendingRanges(String keyspaceName, InetAddressAndPort endpoint) { - List> ranges = new ArrayList<>(); - for (Map.Entry, InetAddressAndPort> entry : getPendingRangesMM(keyspaceName).entries()) + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint); + for (Map.Entry, Replica> entry : getPendingRangesMM(keyspaceName).flattenEntries()) { - if (entry.getValue().equals(endpoint)) + Replica replica = entry.getValue(); + if (replica.endpoint().equals(endpoint)) { - ranges.add(entry.getKey()); + builder.add(replica); } } - return ranges; + return builder.build(); } /** @@ -858,25 +857,27 @@ public class TokenMetadata { PendingRangeMaps newPendingRanges = new PendingRangeMaps(); - Multimap> addressRanges = strategy.getAddressRanges(metadata); + RangesByEndpoint addressRanges = strategy.getAddressReplicas(metadata); // Copy of metadata reflecting the situation after all leave operations are finished. TokenMetadata allLeftMetadata = removeEndpoints(metadata.cloneOnlyTokenMap(), leavingEndpoints); // get all ranges that will be affected by leaving nodes - Set> affectedRanges = new HashSet>(); + Set> removeAffectedRanges = new HashSet<>(); for (InetAddressAndPort endpoint : leavingEndpoints) - affectedRanges.addAll(addressRanges.get(endpoint)); + removeAffectedRanges.addAll(addressRanges.get(endpoint).ranges()); // for each of those ranges, find what new nodes will be responsible for the range when // all leaving nodes are gone. - for (Range range : affectedRanges) + for (Range range : removeAffectedRanges) { - Set currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata)); - Set newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata)); - for (InetAddressAndPort address : Sets.difference(newEndpoints, currentEndpoints)) + EndpointsForRange currentReplicas = strategy.calculateNaturalReplicas(range.right, metadata); + EndpointsForRange newReplicas = strategy.calculateNaturalReplicas(range.right, allLeftMetadata); + for (Replica replica : newReplicas) { - newPendingRanges.addPendingRange(range, address); + if (currentReplicas.endpoints().contains(replica.endpoint())) + continue; + newPendingRanges.addPendingRange(range, replica); } } @@ -891,9 +892,9 @@ public class TokenMetadata Collection tokens = bootstrapAddresses.get(endpoint); allLeftMetadata.updateNormalTokens(tokens, endpoint); - for (Range range : strategy.getAddressRanges(allLeftMetadata).get(endpoint)) + for (Replica replica : strategy.getAddressReplicas(allLeftMetadata, endpoint)) { - newPendingRanges.addPendingRange(range, endpoint); + newPendingRanges.addPendingRange(replica.range(), replica); } allLeftMetadata.removeEndpoint(endpoint); } @@ -906,38 +907,43 @@ public class TokenMetadata for (Pair moving : movingEndpoints) { //Calculate all the ranges which will could be affected. This will include the ranges before and after the move. - Set> moveAffectedRanges = new HashSet<>(); + Set moveAffectedReplicas = new HashSet<>(); InetAddressAndPort endpoint = moving.right; // address of the moving node //Add ranges before the move - for (Range range : strategy.getAddressRanges(allLeftMetadata).get(endpoint)) + for (Replica replica : strategy.getAddressReplicas(allLeftMetadata, endpoint)) { - moveAffectedRanges.add(range); + moveAffectedReplicas.add(replica); } allLeftMetadata.updateNormalToken(moving.left, endpoint); //Add ranges after the move - for (Range range : strategy.getAddressRanges(allLeftMetadata).get(endpoint)) + for (Replica replica : strategy.getAddressReplicas(allLeftMetadata, endpoint)) { - moveAffectedRanges.add(range); + moveAffectedReplicas.add(replica); } - for(Range range : moveAffectedRanges) + for (Replica replica : moveAffectedReplicas) { - Set currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, metadata)); - Set newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(range.right, allLeftMetadata)); + Set currentEndpoints = strategy.calculateNaturalReplicas(replica.range().right, metadata).endpoints(); + Set newEndpoints = strategy.calculateNaturalReplicas(replica.range().right, allLeftMetadata).endpoints(); Set difference = Sets.difference(newEndpoints, currentEndpoints); - for(final InetAddressAndPort address : difference) + for (final InetAddressAndPort address : difference) { - Collection> newRanges = strategy.getAddressRanges(allLeftMetadata).get(address); - Collection> oldRanges = strategy.getAddressRanges(metadata).get(address); - //We want to get rid of any ranges which the node is currently getting. - newRanges.removeAll(oldRanges); + RangesAtEndpoint newReplicas = strategy.getAddressReplicas(allLeftMetadata, address); + RangesAtEndpoint oldReplicas = strategy.getAddressReplicas(metadata, address); - for(Range newRange : newRanges) + // Filter out the things that are already replicated + newReplicas = newReplicas.filter(r -> !oldReplicas.contains(r)); + for (Replica newReplica : newReplicas) { - for(Range pendingRange : newRange.subtractAll(oldRanges)) + // for correctness on write, we need to treat ranges that are becoming full differently + // to those that are presently transient; however reads must continue to use the current view + // for ranges that are becoming transient. We could choose to ignore them here, but it's probably + // cleaner to ensure this is dealt with at point of use, where we can make a conscious decision + // about which to use + for (Replica pendingReplica : newReplica.subtractSameReplication(oldReplicas)) { - newPendingRanges.addPendingRange(pendingRange, address); + newPendingRanges.addPendingRange(pendingReplica.range(), pendingReplica); } } } @@ -1206,11 +1212,11 @@ public class TokenMetadata return sb.toString(); } - public Collection pendingEndpointsFor(Token token, String keyspaceName) + public EndpointsForToken pendingEndpointsForToken(Token token, String keyspaceName) { PendingRangeMaps pendingRangeMaps = this.pendingRanges.get(keyspaceName); if (pendingRangeMaps == null) - return Collections.emptyList(); + return EndpointsForToken.empty(token); return pendingRangeMaps.pendingEndpointsFor(token); } @@ -1218,9 +1224,15 @@ public class TokenMetadata /** * @deprecated retained for benefit of old tests */ - public Collection getWriteEndpoints(Token token, String keyspaceName, Collection naturalEndpoints) + public EndpointsForToken getWriteEndpoints(Token token, String keyspaceName, EndpointsForToken natural) { - return ImmutableList.copyOf(Iterables.concat(naturalEndpoints, pendingEndpointsFor(token, keyspaceName))); + EndpointsForToken pending = pendingEndpointsForToken(token, keyspaceName); + if (Endpoints.haveConflicts(natural, pending)) + { + natural = Endpoints.resolveConflictsInNatural(natural, pending); + pending = Endpoints.resolveConflictsInPending(natural, pending); + } + return Endpoints.concat(natural, pending); } /** @return an endpoint to token multimap representation of tokenToEndpointMap (a copy) */ diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 9e8d54209e..5a90804d72 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -102,6 +102,8 @@ public class KeyspaceMetrics public final Counter speculativeFailedRetries; /** Needed to speculate, but didn't have enough replicas **/ public final Counter speculativeInsufficientReplicas; + /** Needed to write to a transient replica to satisfy quorum **/ + public final Counter speculativeWrites; /** Number of started repairs as coordinator on this keyspace */ public final Counter repairsStarted; /** Number of completed repairs as coordinator on this keyspace */ @@ -268,41 +270,12 @@ public class KeyspaceMetrics writeFailedIdealCL = Metrics.counter(factory.createMetricName("WriteFailedIdealCL")); idealCLWriteLatency = new LatencyMetrics(factory, "IdealCLWrite"); - speculativeRetries = createKeyspaceCounter("SpeculativeRetries", new MetricValue() - { - public Long getValue(TableMetrics metric) - { - return metric.speculativeRetries.getCount(); - } - }); - speculativeFailedRetries = createKeyspaceCounter("SpeculativeFailedRetries", new MetricValue() - { - public Long getValue(TableMetrics metric) - { - return metric.speculativeFailedRetries.getCount(); - } - }); - speculativeInsufficientReplicas = createKeyspaceCounter("SpeculativeInsufficientReplicas", new MetricValue() - { - public Long getValue(TableMetrics metric) - { - return metric.speculativeInsufficientReplicas.getCount(); - } - }); - repairsStarted = createKeyspaceCounter("RepairJobsStarted", new MetricValue() - { - public Long getValue(TableMetrics metric) - { - return metric.repairsStarted.getCount(); - } - }); - repairsCompleted = createKeyspaceCounter("RepairJobsCompleted", new MetricValue() - { - public Long getValue(TableMetrics metric) - { - return metric.repairsCompleted.getCount(); - } - }); + speculativeRetries = createKeyspaceCounter("SpeculativeRetries", metric -> metric.speculativeRetries.getCount()); + speculativeFailedRetries = createKeyspaceCounter("SpeculativeFailedRetries", metric -> metric.speculativeFailedRetries.getCount()); + speculativeInsufficientReplicas = createKeyspaceCounter("SpeculativeInsufficientReplicas", metric -> metric.speculativeInsufficientReplicas.getCount()); + speculativeWrites = createKeyspaceCounter("SpeculativeWrites", metric -> metric.speculativeWrites.getCount()); + repairsStarted = createKeyspaceCounter("RepairJobsStarted", metric -> metric.repairsStarted.getCount()); + repairsCompleted = createKeyspaceCounter("RepairJobsCompleted", metric -> metric.repairsCompleted.getCount()); repairTime = Metrics.timer(factory.createMetricName("RepairTime")); repairPrepareTime = Metrics.timer(factory.createMetricName("RepairPrepareTime")); anticompactionTime = Metrics.timer(factory.createMetricName("AntiCompactionTime")); diff --git a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java index fe7673de15..3d00b12489 100644 --- a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java @@ -37,6 +37,7 @@ public class ReadRepairMetrics @Deprecated public static final Meter attempted = Metrics.meter(factory.createMetricName("Attempted")); + // Incremented when additional requests were sent during blocking read repair due to unavailable or slow nodes public static final Meter speculatedRead = Metrics.meter(factory.createMetricName("SpeculatedRead")); public static final Meter speculatedWrite = Metrics.meter(factory.createMetricName("SpeculatedWrite")); diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 49603bae08..53ebcb09b3 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -214,6 +214,9 @@ public class TableMetrics public final Counter speculativeInsufficientReplicas; public final Gauge speculativeSampleLatencyNanos; + public final Counter speculativeWrites; + public final Gauge speculativeWriteLatencyNanos; + public final static LatencyMetrics globalReadLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Read"); public final static LatencyMetrics globalWriteLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Write"); public final static LatencyMetrics globalRangeLatency = new LatencyMetrics(globalFactory, globalAliasFactory, "Range"); @@ -239,7 +242,7 @@ public class TableMetrics Keyspace k = Schema.instance.getKeyspaceInstance(keyspace); if (SchemaConstants.DISTRIBUTED_KEYSPACE_NAME.equals(k.getName())) continue; - if (k.getReplicationStrategy().getReplicationFactor() < 2) + if (k.getReplicationStrategy().getReplicationFactor().allReplicas < 2) continue; for (ColumnFamilyStore cf : k.getColumnFamilyStores()) @@ -825,13 +828,11 @@ public class TableMetrics speculativeRetries = createTableCounter("SpeculativeRetries"); speculativeFailedRetries = createTableCounter("SpeculativeFailedRetries"); speculativeInsufficientReplicas = createTableCounter("SpeculativeInsufficientReplicas"); - speculativeSampleLatencyNanos = createTableGauge("SpeculativeSampleLatencyNanos", new Gauge() - { - public Long getValue() - { - return cfs.sampleLatencyNanos; - } - }); + speculativeSampleLatencyNanos = createTableGauge("SpeculativeSampleLatencyNanos", () -> cfs.sampleReadLatencyNanos); + + speculativeWrites = createTableCounter("SpeculativeWrites"); + speculativeWriteLatencyNanos = createTableGauge("SpeculativeWriteLatencyNanos", () -> cfs.transientWriteLatencyNanos); + keyCacheHitRate = Metrics.register(factory.createMetricName("KeyCacheHitRate"), aliasFactory.createMetricName("KeyCacheHitRate"), new RatioGauge() diff --git a/src/java/org/apache/cassandra/net/IAsyncCallback.java b/src/java/org/apache/cassandra/net/IAsyncCallback.java index 251d263baa..253b41231c 100644 --- a/src/java/org/apache/cassandra/net/IAsyncCallback.java +++ b/src/java/org/apache/cassandra/net/IAsyncCallback.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; /** * implementors of IAsyncCallback need to make sure that any public methods @@ -30,13 +31,9 @@ import org.apache.cassandra.locator.InetAddressAndPort; */ public interface IAsyncCallback { - Predicate isAlive = new Predicate() - { - public boolean apply(InetAddressAndPort endpoint) - { - return FailureDetector.instance.isAlive(endpoint); - } - }; + final Predicate isAlive = FailureDetector.instance::isAlive; + + final Predicate isReplicaAlive = replica -> isAlive.apply(replica.endpoint()); /** * @param msg response received. diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index c8fe3b7c81..bd290a1d95 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -90,6 +90,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.ILatencySubscriber; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.ConnectionMetrics; import org.apache.cassandra.metrics.DroppedMessageMetrics; @@ -604,8 +605,9 @@ public final class MessagingService implements MessagingServiceMBean if (expiredCallbackInfo.shouldHint()) { - Mutation mutation = ((WriteCallbackInfo) expiredCallbackInfo).mutation(); - return StorageProxy.submitHint(mutation, expiredCallbackInfo.target, null); + WriteCallbackInfo writeCallbackInfo = ((WriteCallbackInfo) expiredCallbackInfo); + Mutation mutation = writeCallbackInfo.mutation(); + return StorageProxy.submitHint(mutation, writeCallbackInfo.getReplica(), null); } return null; @@ -961,7 +963,7 @@ public final class MessagingService implements MessagingServiceMBean return verbHandlers.get(type); } - public int addCallback(IAsyncCallback cb, MessageOut message, InetAddressAndPort to, long timeout, boolean failureCallback) + public int addWriteCallback(IAsyncCallback cb, MessageOut message, InetAddressAndPort to, long timeout, boolean failureCallback) { assert message.verb != Verb.MUTATION; // mutations need to call the overload with a ConsistencyLevel int messageId = nextId(); @@ -970,12 +972,12 @@ public final class MessagingService implements MessagingServiceMBean return messageId; } - public int addCallback(IAsyncCallback cb, - MessageOut message, - InetAddressAndPort to, - long timeout, - ConsistencyLevel consistencyLevel, - boolean allowHints) + public int addWriteCallback(IAsyncCallback cb, + MessageOut message, + Replica to, + long timeout, + ConsistencyLevel consistencyLevel, + boolean allowHints) { assert message.verb == Verb.MUTATION || message.verb == Verb.COUNTER_MUTATION @@ -1024,7 +1026,7 @@ public final class MessagingService implements MessagingServiceMBean */ public int sendRR(MessageOut message, InetAddressAndPort to, IAsyncCallback cb, long timeout, boolean failureCallback) { - int id = addCallback(cb, message, to, timeout, failureCallback); + int id = addWriteCallback(cb, message, to, timeout, failureCallback); updateBackPressureOnSend(to, cb, message); sendOneWay(failureCallback ? message.withParameter(ParameterType.FAILURE_CALLBACK, ONE_BYTE) : message, id, to); return id; @@ -1042,14 +1044,14 @@ public final class MessagingService implements MessagingServiceMBean * suggest that a timeout occurred to the invoker of the send(). * @return an reference to message id used to match with the result */ - public int sendRR(MessageOut message, - InetAddressAndPort to, - AbstractWriteResponseHandler handler, - boolean allowHints) + public int sendWriteRR(MessageOut message, + Replica to, + AbstractWriteResponseHandler handler, + boolean allowHints) { - int id = addCallback(handler, message, to, message.getTimeout(), handler.consistencyLevel, allowHints); - updateBackPressureOnSend(to, handler, message); - sendOneWay(message.withParameter(ParameterType.FAILURE_CALLBACK, ONE_BYTE), id, to); + int id = addWriteCallback(handler, message, to, message.getTimeout(), handler.consistencyLevel(), allowHints); + updateBackPressureOnSend(to.endpoint(), handler, message); + sendOneWay(message.withParameter(ParameterType.FAILURE_CALLBACK, ONE_BYTE), id, to.endpoint()); return id; } diff --git a/src/java/org/apache/cassandra/net/WriteCallbackInfo.java b/src/java/org/apache/cassandra/net/WriteCallbackInfo.java index 41ac31b8de..c54e7dcba9 100644 --- a/src/java/org/apache/cassandra/net/WriteCallbackInfo.java +++ b/src/java/org/apache/cassandra/net/WriteCallbackInfo.java @@ -21,7 +21,7 @@ package org.apache.cassandra.net; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.utils.FBUtilities; @@ -30,24 +30,31 @@ public class WriteCallbackInfo extends CallbackInfo { // either a Mutation, or a Paxos Commit (MessageOut) private final Object mutation; + private final Replica replica; - public WriteCallbackInfo(InetAddressAndPort target, + public WriteCallbackInfo(Replica replica, IAsyncCallback callback, MessageOut message, IVersionedSerializer serializer, ConsistencyLevel consistencyLevel, boolean allowHints) { - super(target, callback, serializer, true); + super(replica.endpoint(), callback, serializer, true); assert message != null; this.mutation = shouldHint(allowHints, message, consistencyLevel); //Local writes shouldn't go through messaging service (https://issues.apache.org/jira/browse/CASSANDRA-10477) assert (!target.equals(FBUtilities.getBroadcastAddressAndPort())); + this.replica = replica; } public boolean shouldHint() { - return mutation != null && StorageProxy.shouldHint(target); + return mutation != null && StorageProxy.shouldHint(replica); + } + + public Replica getReplica() + { + return replica; } public Mutation mutation() diff --git a/src/java/org/apache/cassandra/repair/AbstractSyncTask.java b/src/java/org/apache/cassandra/repair/AbstractSyncTask.java new file mode 100644 index 0000000000..124baa17a5 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/AbstractSyncTask.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.List; + +import com.google.common.util.concurrent.AbstractFuture; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +public abstract class AbstractSyncTask extends AbstractFuture implements Runnable +{ + protected abstract void startSync(List> rangesToStream); +} diff --git a/src/java/org/apache/cassandra/repair/AsymmetricLocalSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricLocalSyncTask.java index 2ca524ff5f..eaf890ac39 100644 --- a/src/java/org/apache/cassandra/repair/AsymmetricLocalSyncTask.java +++ b/src/java/org/apache/cassandra/repair/AsymmetricLocalSyncTask.java @@ -18,12 +18,14 @@ package org.apache.cassandra.repair; +import java.util.Collections; import java.util.List; import java.util.UUID; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.ProgressInfo; import org.apache.cassandra.streaming.StreamEvent; @@ -54,8 +56,9 @@ public class AsymmetricLocalSyncTask extends AsymmetricSyncTask implements Strea previewKind) .listeners(this) .flushBeforeTransfer(pendingRepair == null) - // request ranges from the remote node - .requestRanges(fetchFrom, desc.keyspace, rangesToFetch, desc.columnFamily); + // request ranges from the remote node, see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here + .requestRanges(fetchFrom, desc.keyspace, RangesAtEndpoint.toDummyList(rangesToFetch), + RangesAtEndpoint.toDummyList(Collections.emptyList()), desc.columnFamily); plan.execute(); } diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java index e24d8542d9..2b171c94d3 100644 --- a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java @@ -30,6 +30,7 @@ import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MerkleTrees; public class AsymmetricRemoteSyncTask extends AsymmetricSyncTask implements CompletableRemoteSyncTask { @@ -37,6 +38,10 @@ public class AsymmetricRemoteSyncTask extends AsymmetricSyncTask implements Comp { super(desc, fetchNode, fetchFrom, rangesToFetch, previewKind); } + public AsymmetricRemoteSyncTask(RepairJobDesc desc, TreeResponse to, TreeResponse from, PreviewKind previewKind) + { + this(desc, to.endpoint, from.endpoint, MerkleTrees.difference(to.trees, from.trees), previewKind); + } public void startSync(List> rangesToFetch) { @@ -46,6 +51,7 @@ public class AsymmetricRemoteSyncTask extends AsymmetricSyncTask implements Comp Tracing.traceRepair(message); MessagingService.instance().sendOneWay(request.createMessage(), request.fetchingNode); } + public void syncComplete(boolean success, List summaries) { if (success) diff --git a/src/java/org/apache/cassandra/repair/AsymmetricSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricSyncTask.java index 4d38e8a61f..35474afbad 100644 --- a/src/java/org/apache/cassandra/repair/AsymmetricSyncTask.java +++ b/src/java/org/apache/cassandra/repair/AsymmetricSyncTask.java @@ -21,8 +21,6 @@ package org.apache.cassandra.repair; import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.AbstractFuture; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +31,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tracing.Tracing; -public abstract class AsymmetricSyncTask extends AbstractFuture implements Runnable +public abstract class AsymmetricSyncTask extends AbstractSyncTask { private static Logger logger = LoggerFactory.getLogger(AsymmetricSyncTask.class); protected final RepairJobDesc desc; @@ -44,9 +42,9 @@ public abstract class AsymmetricSyncTask extends AbstractFuture implem private long startTime = Long.MIN_VALUE; protected volatile SyncStat stat; - public AsymmetricSyncTask(RepairJobDesc desc, InetAddressAndPort fetchingNode, InetAddressAndPort fetchFrom, List> rangesToFetch, PreviewKind previewKind) { + assert !fetchFrom.equals(fetchingNode) : "Fetching from self " + fetchFrom; this.desc = desc; this.fetchFrom = fetchFrom; this.fetchingNode = fetchingNode; @@ -55,6 +53,7 @@ public abstract class AsymmetricSyncTask extends AbstractFuture implem stat = new SyncStat(new NodePair(fetchingNode, fetchFrom), rangesToFetch.size()); this.previewKind = previewKind; } + public void run() { startTime = System.currentTimeMillis(); @@ -79,7 +78,4 @@ public abstract class AsymmetricSyncTask extends AbstractFuture implem if (startTime != Long.MIN_VALUE) Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.syncTime.update(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS); } - - - public abstract void startSync(List> rangesToFetch); } diff --git a/src/java/org/apache/cassandra/repair/CommonRange.java b/src/java/org/apache/cassandra/repair/CommonRange.java new file mode 100644 index 0000000000..928e570c42 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/CommonRange.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; + +/** + * Groups ranges with identical endpoints/transient endpoints + */ +public class CommonRange +{ + public final ImmutableSet endpoints; + public final ImmutableSet transEndpoints; + public final Collection> ranges; + + public CommonRange(Set endpoints, Set transEndpoints, Collection> ranges) + { + Preconditions.checkArgument(endpoints != null && !endpoints.isEmpty()); + Preconditions.checkArgument(transEndpoints != null); + Preconditions.checkArgument(endpoints.containsAll(transEndpoints), "transEndpoints must be a subset of endpoints"); + Preconditions.checkArgument(ranges != null && !ranges.isEmpty()); + + this.endpoints = ImmutableSet.copyOf(endpoints); + this.transEndpoints = ImmutableSet.copyOf(transEndpoints); + this.ranges = new ArrayList(ranges); + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CommonRange that = (CommonRange) o; + + if (!endpoints.equals(that.endpoints)) return false; + if (!transEndpoints.equals(that.transEndpoints)) return false; + return ranges.equals(that.ranges); + } + + public int hashCode() + { + int result = endpoints.hashCode(); + result = 31 * result + transEndpoints.hashCode(); + result = 31 * result + ranges.hashCode(); + return result; + } + + public String toString() + { + return "CommonRange{" + + "endpoints=" + endpoints + + ", transEndpoints=" + transEndpoints + + ", ranges=" + ranges + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java b/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java index 8aa438199f..bc614dc201 100644 --- a/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java +++ b/src/java/org/apache/cassandra/repair/KeyspaceRepairManager.java @@ -25,8 +25,7 @@ import java.util.concurrent.ExecutorService; import com.google.common.util.concurrent.ListenableFuture; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.RangesAtEndpoint; /** * Keyspace level hook for repair. @@ -38,5 +37,8 @@ public interface KeyspaceRepairManager * been notified that the repair session has been completed, the data associated with the given session id must * not be combined with repaired or unrepaired data, or data from other repair sessions. */ - ListenableFuture prepareIncrementalRepair(UUID sessionID, Collection tables, Collection> ranges, ExecutorService executor); + ListenableFuture prepareIncrementalRepair(UUID sessionID, + Collection tables, + RangesAtEndpoint tokenRanges, + ExecutorService executor); } diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 48973d21cf..d38435b657 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -64,7 +64,7 @@ public class RepairJob extends AbstractFuture implements Runnable public RepairJob(RepairSession session, String columnFamily, boolean isIncremental, PreviewKind previewKind, boolean optimiseStreams) { this.session = session; - this.desc = new RepairJobDesc(session.parentRepairSession, session.getId(), session.keyspace, columnFamily, session.getRanges()); + this.desc = new RepairJobDesc(session.parentRepairSession, session.getId(), session.keyspace, columnFamily, session.commonRange.ranges); this.taskExecutor = session.taskExecutor; this.parallelismDegree = session.parallelismDegree; this.isIncremental = isIncremental; @@ -83,7 +83,7 @@ public class RepairJob extends AbstractFuture implements Runnable Keyspace ks = Keyspace.open(desc.keyspace); ColumnFamilyStore cfs = ks.getColumnFamilyStore(desc.columnFamily); cfs.metric.repairsStarted.inc(); - List allEndpoints = new ArrayList<>(session.endpoints); + List allEndpoints = new ArrayList<>(session.commonRange.endpoints); allEndpoints.add(FBUtilities.getBroadcastAddressAndPort()); ListenableFuture> validations; @@ -160,13 +160,18 @@ public class RepairJob extends AbstractFuture implements Runnable }, taskExecutor); } + private boolean isTransient(InetAddressAndPort ep) + { + return session.commonRange.transEndpoints.contains(ep); + } + private AsyncFunction, List> standardSyncing() { return trees -> { InetAddressAndPort local = FBUtilities.getLocalAddressAndPort(); - List syncTasks = new ArrayList<>(); + List syncTasks = new ArrayList<>(); // We need to difference all trees one against another for (int i = 0; i < trees.size() - 1; ++i) { @@ -174,17 +179,29 @@ public class RepairJob extends AbstractFuture implements Runnable for (int j = i + 1; j < trees.size(); ++j) { TreeResponse r2 = trees.get(j); - SyncTask task; + + if (isTransient(r1.endpoint) && isTransient(r2.endpoint)) + continue; + + AbstractSyncTask task; if (r1.endpoint.equals(local) || r2.endpoint.equals(local)) { - task = new LocalSyncTask(desc, r1, r2, isIncremental ? desc.parentSessionId : null, session.pullRepair, session.previewKind); + InetAddressAndPort remote = r1.endpoint.equals(local) ? r2.endpoint : r1.endpoint; + task = new SymmetricLocalSyncTask(desc, r1, r2, isTransient(remote), isIncremental ? desc.parentSessionId : null, session.pullRepair, session.previewKind); + } + else if (isTransient(r1.endpoint) || isTransient(r2.endpoint)) + { + TreeResponse streamFrom = isTransient(r1.endpoint) ? r1 : r2; + TreeResponse streamTo = isTransient(r1.endpoint) ? r2: r1; + task = new AsymmetricRemoteSyncTask(desc, streamTo, streamFrom, previewKind); + session.waitForSync(Pair.create(desc, new NodePair(streamTo.endpoint, streamFrom.endpoint)), (AsymmetricRemoteSyncTask) task); } else { - task = new RemoteSyncTask(desc, r1, r2, session.previewKind); - // RemoteSyncTask expects SyncComplete message sent back. + task = new SymmetricRemoteSyncTask(desc, r1, r2, session.previewKind); + // SymmetricRemoteSyncTask expects SyncComplete message sent back. // Register task to RepairSession to receive response. - session.waitForSync(Pair.create(desc, new NodePair(r1.endpoint, r2.endpoint)), (RemoteSyncTask) task); + session.waitForSync(Pair.create(desc, new NodePair(r1.endpoint, r2.endpoint)), (SymmetricRemoteSyncTask) task); } syncTasks.add(task); taskExecutor.submit(task); @@ -200,7 +217,7 @@ public class RepairJob extends AbstractFuture implements Runnable { InetAddressAndPort local = FBUtilities.getLocalAddressAndPort(); - List syncTasks = new ArrayList<>(); + List syncTasks = new ArrayList<>(); // We need to difference all trees one against another DifferenceHolder diffHolder = new DifferenceHolder(trees); @@ -215,6 +232,11 @@ public class RepairJob extends AbstractFuture implements Runnable for (int i = 0; i < trees.size(); i++) { InetAddressAndPort address = trees.get(i).endpoint; + + // we don't stream to transient replicas + if (isTransient(address)) + continue; + HostDifferences streamsFor = reducedDifferences.get(address); if (streamsFor != null) { @@ -373,4 +395,4 @@ public class RepairJob extends AbstractFuture implements Runnable } return Futures.allAsList(tasks); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/RepairRunnable.java b/src/java/org/apache/cassandra/repair/RepairRunnable.java index 90c0146e7c..8d3cd54aa3 100644 --- a/src/java/org/apache/cassandra/repair/RepairRunnable.java +++ b/src/java/org/apache/cassandra/repair/RepairRunnable.java @@ -18,7 +18,6 @@ package org.apache.cassandra.repair; import java.io.IOException; -import java.net.InetAddress; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ExecutorService; @@ -29,8 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -38,8 +36,9 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.*; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.Replica; import org.apache.commons.lang3.time.DurationFormatUtils; -import org.junit.internal.runners.statements.Fail; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +73,6 @@ import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.UUIDGen; import org.apache.cassandra.utils.WrappedRunnable; import org.apache.cassandra.utils.progress.ProgressEvent; @@ -141,46 +139,6 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti recordFailure(message, completionMessage); } - @VisibleForTesting - static class CommonRange - { - public final Set endpoints; - public final Collection> ranges; - - public CommonRange(Set endpoints, Collection> ranges) - { - Preconditions.checkArgument(endpoints != null && !endpoints.isEmpty()); - Preconditions.checkArgument(ranges != null && !ranges.isEmpty()); - this.endpoints = endpoints; - this.ranges = ranges; - } - - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - CommonRange that = (CommonRange) o; - - if (!endpoints.equals(that.endpoints)) return false; - return ranges.equals(that.ranges); - } - - public int hashCode() - { - int result = endpoints.hashCode(); - result = 31 * result + ranges.hashCode(); - return result; - } - - public String toString() - { - return "CommonRange{" + - "endpoints=" + endpoints + - ", ranges=" + ranges + - '}'; - } - } protected void runMayThrow() throws Exception { @@ -238,20 +196,24 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti Set allNeighbors = new HashSet<>(); List commonRanges = new ArrayList<>(); - //pre-calculate output of getLocalRanges and pass it to getNeighbors to increase performance and prevent + //pre-calculate output of getLocalReplicas and pass it to getNeighbors to increase performance and prevent //calculation multiple times - Collection> keyspaceLocalRanges = storageService.getLocalRanges(keyspace); + // we arbitrarily limit ourselves to only full replicas, in lieu of ensuring it is safe to coordinate from a transient replica + Iterable> keyspaceLocalRanges = storageService + .getLocalReplicas(keyspace) + .filter(Replica::isFull) + .ranges(); try { for (Range range : options.getRanges()) { - Set neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range, - options.getDataCenters(), - options.getHosts()); + EndpointsForRange neighbors = ActiveRepairService.getNeighbors(keyspace, keyspaceLocalRanges, range, + options.getDataCenters(), + options.getHosts()); addRangeToNeighbors(commonRanges, range, neighbors); - allNeighbors.addAll(neighbors); + allNeighbors.addAll(neighbors.endpoints()); } progress.incrementAndGet(); @@ -387,11 +349,13 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti for (CommonRange commonRange: commonRanges) { Set endpoints = ImmutableSet.copyOf(Iterables.filter(commonRange.endpoints, liveEndpoints::contains)); + Set transEndpoints = ImmutableSet.copyOf(Iterables.filter(commonRange.transEndpoints, liveEndpoints::contains)); + Preconditions.checkState(endpoints.containsAll(transEndpoints), "transEndpoints must be a subset of endpoints"); // this node is implicitly a participant in this repair, so a single endpoint is ok here if (!endpoints.isEmpty()) { - filtered.add(new CommonRange(endpoints, commonRange.ranges)); + filtered.add(new CommonRange(endpoints, transEndpoints, commonRange.ranges)); } } Preconditions.checkState(!filtered.isEmpty(), "Not enough live endpoints for a repair"); @@ -514,14 +478,13 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti // we do endpoint filtering at the start of an incremental repair, // so repair sessions shouldn't also be checking liveness boolean force = options.isForcedRepair() && !isIncremental; - for (CommonRange cr : commonRanges) + for (CommonRange commonRange : commonRanges) { - logger.info("Starting RepairSession for {}", cr); + logger.info("Starting RepairSession for {}", commonRange); RepairSession session = ActiveRepairService.instance.submitRepairSession(parentSession, - cr.ranges, + commonRange, keyspace, options.getParallelism(), - cr.endpoints, isIncremental, options.isPullRepair(), force, @@ -559,7 +522,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti public void onSuccess(RepairSessionResult result) { String message = String.format("Repair session %s for range %s finished", session.getId(), - session.getRanges().toString()); + session.ranges().toString()); logger.info(message); fireProgressEvent(new ProgressEvent(ProgressEventType.PROGRESS, progress.incrementAndGet(), @@ -572,7 +535,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti StorageMetrics.repairExceptions.inc(); String message = String.format("Repair session %s for range %s failed with error %s", - session.getId(), session.getRanges().toString(), t.getMessage()); + session.getId(), session.ranges().toString(), t.getMessage()); logger.error(message, t); fireProgressEvent(new ProgressEvent(ProgressEventType.ERROR, progress.incrementAndGet(), @@ -684,13 +647,15 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti ImmutableList.of(failureMessage, completionMessage)); } - private void addRangeToNeighbors(List neighborRangeList, Range range, Set neighbors) + private void addRangeToNeighbors(List neighborRangeList, Range range, EndpointsForRange neighbors) { + Set endpoints = neighbors.endpoints(); + Set transEndpoints = neighbors.filter(Replica::isTransient).endpoints(); for (int i = 0; i < neighborRangeList.size(); i++) { CommonRange cr = neighborRangeList.get(i); - if (cr.endpoints.containsAll(neighbors)) + if (cr.endpoints.containsAll(endpoints) && cr.transEndpoints.containsAll(transEndpoints)) { cr.ranges.add(range); return; @@ -699,7 +664,7 @@ public class RepairRunnable extends WrappedRunnable implements ProgressEventNoti List> ranges = new ArrayList<>(); ranges.add(range); - neighborRangeList.add(new CommonRange(neighbors, ranges)); + neighborRangeList.add(new CommonRange(endpoints, transEndpoints, ranges)); } private Thread createQueryThread(final int cmd, final UUID sessionId) diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index ec06f375eb..2ff60ec0af 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -55,8 +55,8 @@ import org.apache.cassandra.utils.Pair; * validationComplete()). * *

  • Synchronization phase: once all trees are received, the job compares each tree with - * all the other using a so-called {@link SyncTask}. If there is difference between 2 trees, the - * concerned SyncTask will start a streaming of the difference between the 2 endpoint concerned. + * all the other using a so-called {@link SymmetricSyncTask}. If there is difference between 2 trees, the + * concerned SymmetricSyncTask will start a streaming of the difference between the 2 endpoint concerned. *
    • * * The job is done once all its SyncTasks are done (i.e. have either computed no differences @@ -74,7 +74,7 @@ import org.apache.cassandra.utils.Pair; * we still first send a message to each node to flush and snapshot data so each merkle tree * creation is still done on similar data, even if the actual creation is not * done simulatneously). If not sequential, all merkle tree are requested in parallel. - * Similarly, if a job is sequential, it will handle one SyncTask at a time, but will handle + * Similarly, if a job is sequential, it will handle one SymmetricSyncTask at a time, but will handle * all of them in parallel otherwise. */ public class RepairSession extends AbstractFuture implements IEndpointStateChangeSubscriber, @@ -94,8 +94,7 @@ public class RepairSession extends AbstractFuture implement public final boolean skippedReplicas; /** Range to repair */ - public final Collection> ranges; - public final Set endpoints; + public final CommonRange commonRange; public final boolean isIncremental; public final PreviewKind previewKind; @@ -114,23 +113,20 @@ public class RepairSession extends AbstractFuture implement /** * Create new repair session. - * * @param parentRepairSession the parent sessions id * @param id this sessions id - * @param ranges ranges to repair + * @param commonRange ranges to repair * @param keyspace name of keyspace * @param parallelismDegree specifies the degree of parallelism when calculating the merkle trees - * @param endpoints the data centers that should be part of the repair; null for all DCs * @param pullRepair true if the repair should be one way (from remote host to this host and only applicable between two hosts--see RepairOption) * @param force true if the repair should ignore dead endpoints (instead of failing) * @param cfnames names of columnfamilies */ public RepairSession(UUID parentRepairSession, UUID id, - Collection> ranges, + CommonRange commonRange, String keyspace, RepairParallelism parallelismDegree, - Set endpoints, boolean isIncremental, boolean pullRepair, boolean force, @@ -145,7 +141,6 @@ public class RepairSession extends AbstractFuture implement this.parallelismDegree = parallelismDegree; this.keyspace = keyspace; this.cfnames = cfnames; - this.ranges = ranges; //If force then filter out dead endpoints boolean forceSkippedReplicas = false; @@ -153,7 +148,7 @@ public class RepairSession extends AbstractFuture implement { logger.debug("force flag set, removing dead endpoints"); final Set removeCandidates = new HashSet<>(); - for (final InetAddressAndPort endpoint : endpoints) + for (final InetAddressAndPort endpoint : commonRange.endpoints) { if (!FailureDetector.instance.isAlive(endpoint)) { @@ -166,12 +161,13 @@ public class RepairSession extends AbstractFuture implement // we shouldn't be recording a successful repair if // any replicas are excluded from the repair forceSkippedReplicas = true; - endpoints = new HashSet<>(endpoints); - endpoints.removeAll(removeCandidates); + Set filteredEndpoints = new HashSet<>(commonRange.endpoints); + filteredEndpoints.removeAll(removeCandidates); + commonRange = new CommonRange(filteredEndpoints, commonRange.transEndpoints, commonRange.ranges); } } - this.endpoints = endpoints; + this.commonRange = commonRange; this.isIncremental = isIncremental; this.previewKind = previewKind; this.pullRepair = pullRepair; @@ -184,9 +180,14 @@ public class RepairSession extends AbstractFuture implement return id; } - public Collection> getRanges() + public Collection> ranges() { - return ranges; + return commonRange.ranges; + } + + public Collection endpoints() + { + return commonRange.endpoints; } public void waitForValidation(Pair key, ValidationTask task) @@ -247,7 +248,7 @@ public class RepairSession extends AbstractFuture implement { StringBuilder sb = new StringBuilder(); sb.append(FBUtilities.getBroadcastAddressAndPort()); - for (InetAddressAndPort ep : endpoints) + for (InetAddressAndPort ep : commonRange.endpoints) sb.append(", ").append(ep); return sb.toString(); } @@ -266,18 +267,18 @@ public class RepairSession extends AbstractFuture implement if (terminated) return; - logger.info("{} new session: will sync {} on range {} for {}.{}", previewKind.logPrefix(getId()), repairedNodes(), ranges, keyspace, Arrays.toString(cfnames)); - Tracing.traceRepair("Syncing range {}", ranges); + logger.info("{} new session: will sync {} on range {} for {}.{}", previewKind.logPrefix(getId()), repairedNodes(), commonRange, keyspace, Arrays.toString(cfnames)); + Tracing.traceRepair("Syncing range {}", commonRange); if (!previewKind.isPreview()) { - SystemDistributedKeyspace.startRepairs(getId(), parentRepairSession, keyspace, cfnames, ranges, endpoints); + SystemDistributedKeyspace.startRepairs(getId(), parentRepairSession, keyspace, cfnames, commonRange); } - if (endpoints.isEmpty()) + if (commonRange.endpoints.isEmpty()) { - logger.info("{} {}", previewKind.logPrefix(getId()), message = String.format("No neighbors to repair with on range %s: session completed", ranges)); + logger.info("{} {}", previewKind.logPrefix(getId()), message = String.format("No neighbors to repair with on range %s: session completed", commonRange)); Tracing.traceRepair(message); - set(new RepairSessionResult(id, keyspace, ranges, Lists.newArrayList(), skippedReplicas)); + set(new RepairSessionResult(id, keyspace, commonRange.ranges, Lists.newArrayList(), skippedReplicas)); if (!previewKind.isPreview()) { SystemDistributedKeyspace.failRepairs(getId(), keyspace, cfnames, new RuntimeException(message)); @@ -286,7 +287,7 @@ public class RepairSession extends AbstractFuture implement } // Checking all nodes are live - for (InetAddressAndPort endpoint : endpoints) + for (InetAddressAndPort endpoint : commonRange.endpoints) { if (!FailureDetector.instance.isAlive(endpoint) && !skippedReplicas) { @@ -318,8 +319,8 @@ public class RepairSession extends AbstractFuture implement { // this repair session is completed logger.info("{} {}", previewKind.logPrefix(getId()), "Session completed successfully"); - Tracing.traceRepair("Completed sync of range {}", ranges); - set(new RepairSessionResult(id, keyspace, ranges, results, skippedReplicas)); + Tracing.traceRepair("Completed sync of range {}", commonRange); + set(new RepairSessionResult(id, keyspace, commonRange.ranges, results, skippedReplicas)); taskExecutor.shutdown(); // mark this session as terminated @@ -372,7 +373,7 @@ public class RepairSession extends AbstractFuture implement public void convict(InetAddressAndPort endpoint, double phi) { - if (!endpoints.contains(endpoint)) + if (!commonRange.endpoints.contains(endpoint)) return; // We want a higher confidence in the failure detection than usual because failing a repair wrongly has a high cost. diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java index 5d2b2ecd0c..e9cba89323 100644 --- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java +++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.Collection; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,9 +79,12 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler StreamPlan sp = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind) .listeners(this) .flushBeforeTransfer(pendingRepair == null) // sstables are isolated at the beginning of an incremental repair session, so flushing isn't neccessary - .requestRanges(dest, desc.keyspace, ranges, desc.columnFamily); // request ranges from the remote node + // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here + .requestRanges(dest, desc.keyspace, RangesAtEndpoint.toDummyList(ranges), + RangesAtEndpoint.toDummyList(Collections.emptyList()), desc.columnFamily); // request ranges from the remote node if (!asymmetric) - sp.transferRanges(dest, desc.keyspace, ranges, desc.columnFamily); // send ranges to the remote node + // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here + sp.transferRanges(dest, desc.keyspace, RangesAtEndpoint.toDummyList(ranges), desc.columnFamily); // send ranges to the remote node return sp; } diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricLocalSyncTask.java similarity index 81% rename from src/java/org/apache/cassandra/repair/LocalSyncTask.java rename to src/java/org/apache/cassandra/repair/SymmetricLocalSyncTask.java index d7e03874a4..7eedab761e 100644 --- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java +++ b/src/java/org/apache/cassandra/repair/SymmetricLocalSyncTask.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.repair; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -24,36 +25,38 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.ProgressInfo; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; +import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamPlan; import org.apache.cassandra.streaming.StreamState; -import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; /** - * LocalSyncTask performs streaming between local(coordinator) node and remote replica. + * SymmetricLocalSyncTask performs streaming between local(coordinator) node and remote replica. */ -public class LocalSyncTask extends SyncTask implements StreamEventHandler +public class SymmetricLocalSyncTask extends SymmetricSyncTask implements StreamEventHandler { private final TraceState state = Tracing.instance.get(); - private static final Logger logger = LoggerFactory.getLogger(LocalSyncTask.class); + private static final Logger logger = LoggerFactory.getLogger(SymmetricLocalSyncTask.class); + private final boolean remoteIsTransient; private final UUID pendingRepair; private final boolean pullRepair; - public LocalSyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, UUID pendingRepair, boolean pullRepair, PreviewKind previewKind) + public SymmetricLocalSyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, boolean remoteIsTransient, UUID pendingRepair, boolean pullRepair, PreviewKind previewKind) { super(desc, r1, r2, previewKind); + this.remoteIsTransient = remoteIsTransient; this.pendingRepair = pendingRepair; this.pullRepair = pullRepair; } @@ -64,11 +67,15 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler StreamPlan plan = new StreamPlan(StreamOperation.REPAIR, 1, false, pendingRepair, previewKind) .listeners(this) .flushBeforeTransfer(pendingRepair == null) - .requestRanges(dst, desc.keyspace, differences, desc.columnFamily); // request ranges from the remote node - if (!pullRepair) + // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here + .requestRanges(dst, desc.keyspace, RangesAtEndpoint.toDummyList(differences), + RangesAtEndpoint.toDummyList(Collections.emptyList()), desc.columnFamily); // request ranges from the remote node + + if (!pullRepair && !remoteIsTransient) { // send ranges to the remote node if we are not performing a pull repair - plan.transferRanges(dst, desc.keyspace, differences, desc.columnFamily); + // see comment on RangesAtEndpoint.toDummyList for why we synthesize replicas here + plan.transferRanges(dst, desc.keyspace, RangesAtEndpoint.toDummyList(differences), desc.columnFamily); } return plan; diff --git a/src/java/org/apache/cassandra/repair/RemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java similarity index 77% rename from src/java/org/apache/cassandra/repair/RemoteSyncTask.java rename to src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java index 0a47f73c8b..1f2740f8e3 100644 --- a/src/java/org/apache/cassandra/repair/RemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java @@ -18,7 +18,9 @@ package org.apache.cassandra.repair; import java.util.List; +import java.util.function.Predicate; +import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +29,8 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.messages.AsymmetricSyncRequest; +import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; @@ -34,29 +38,35 @@ import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; /** - * RemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node + * SymmetricRemoteSyncTask sends {@link SyncRequest} to remote(non-coordinator) node * to repair(stream) data with other replica. * - * When RemoteSyncTask receives SyncComplete from remote node, task completes. + * When SymmetricRemoteSyncTask receives SyncComplete from remote node, task completes. */ -public class RemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask +public class SymmetricRemoteSyncTask extends SymmetricSyncTask implements CompletableRemoteSyncTask { - private static final Logger logger = LoggerFactory.getLogger(RemoteSyncTask.class); + private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class); - public RemoteSyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, PreviewKind previewKind) + public SymmetricRemoteSyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, PreviewKind previewKind) { super(desc, r1, r2, previewKind); } + void sendRequest(RepairMessage request, InetAddressAndPort to) + { + MessagingService.instance().sendOneWay(request.createMessage(), to); + } + @Override protected void startSync(List> differences) { InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + SyncRequest request = new SyncRequest(desc, local, r1.endpoint, r2.endpoint, differences, previewKind); String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst); logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message); Tracing.traceRepair(message); - MessagingService.instance().sendOneWay(request.createMessage(), request.src); + sendRequest(request, request.src); } public void syncComplete(boolean success, List summaries) diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricSyncTask.java similarity index 87% rename from src/java/org/apache/cassandra/repair/SyncTask.java rename to src/java/org/apache/cassandra/repair/SymmetricSyncTask.java index f7cf5f1e08..3da2293d00 100644 --- a/src/java/org/apache/cassandra/repair/SyncTask.java +++ b/src/java/org/apache/cassandra/repair/SymmetricSyncTask.java @@ -20,7 +20,6 @@ package org.apache.cassandra.repair; import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.AbstractFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,12 +31,12 @@ import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.MerkleTrees; /** - * SyncTask will calculate the difference of MerkleTree between two nodes + * SymmetricSyncTask will calculate the difference of MerkleTree between two nodes * and perform necessary operation to repair replica. */ -public abstract class SyncTask extends AbstractFuture implements Runnable +public abstract class SymmetricSyncTask extends AbstractSyncTask { - private static Logger logger = LoggerFactory.getLogger(SyncTask.class); + private static Logger logger = LoggerFactory.getLogger(SymmetricSyncTask.class); protected final RepairJobDesc desc; protected final TreeResponse r1; @@ -47,7 +46,7 @@ public abstract class SyncTask extends AbstractFuture implements Runna protected volatile SyncStat stat; protected long startTime = Long.MIN_VALUE; - public SyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, PreviewKind previewKind) + public SymmetricSyncTask(RepairJobDesc desc, TreeResponse r1, TreeResponse r2, PreviewKind previewKind) { this.desc = desc; this.r1 = r1; @@ -92,6 +91,4 @@ public abstract class SyncTask extends AbstractFuture implements Runna if (startTime != Long.MIN_VALUE) Keyspace.open(desc.keyspace).getColumnFamilyStore(desc.columnFamily).metric.syncTime.update(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS); } - - protected abstract void startSync(List> differences); } diff --git a/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java index a85a1e5a6d..fc09e71623 100644 --- a/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/repair/SystemDistributedKeyspace.java @@ -185,13 +185,13 @@ public final class SystemDistributedKeyspace processSilent(fmtQuery); } - public static void startRepairs(UUID id, UUID parent_id, String keyspaceName, String[] cfnames, Collection> ranges, Iterable endpoints) + public static void startRepairs(UUID id, UUID parent_id, String keyspaceName, String[] cfnames, CommonRange commonRange) { InetAddressAndPort coordinator = FBUtilities.getBroadcastAddressAndPort(); Set participants = Sets.newHashSet(); Set participants_v2 = Sets.newHashSet(); - for (InetAddressAndPort endpoint : endpoints) + for (InetAddressAndPort endpoint : commonRange.endpoints) { participants.add(endpoint.getHostAddress(false)); participants_v2.add(endpoint.toString()); @@ -203,7 +203,7 @@ public final class SystemDistributedKeyspace for (String cfname : cfnames) { - for (Range range : ranges) + for (Range range : commonRange.ranges) { String fmtQry = format(query, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, REPAIR_HISTORY, keyspaceName, diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java index ed25166513..4089e77425 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java @@ -47,11 +47,14 @@ import com.google.common.primitives.Ints; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; + +import org.apache.cassandra.locator.RangesAtEndpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.repair.KeyspaceRepairManager; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.marshal.UTF8Type; @@ -543,9 +546,35 @@ public class LocalSessions } @VisibleForTesting - ListenableFuture prepareSession(KeyspaceRepairManager repairManager, UUID sessionID, Collection tables, Collection> ranges, ExecutorService executor) + ListenableFuture prepareSession(KeyspaceRepairManager repairManager, + UUID sessionID, + Collection tables, + RangesAtEndpoint tokenRanges, + ExecutorService executor) { - return repairManager.prepareIncrementalRepair(sessionID, tables, ranges, executor); + return repairManager.prepareIncrementalRepair(sessionID, tables, tokenRanges, executor); + } + + RangesAtEndpoint filterLocalRanges(String keyspace, Set> ranges) + { + RangesAtEndpoint localRanges = StorageService.instance.getLocalReplicas(keyspace); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(localRanges.endpoint()); + for (Range range : ranges) + { + for (Replica replica : localRanges) + { + if (replica.range().equals(range)) + { + builder.add(replica); + } + else if (replica.contains(range)) + { + builder.add(replica.decorateSubrange(range)); + } + } + + } + return builder.build(); } /** @@ -582,7 +611,8 @@ public class LocalSessions ExecutorService executor = Executors.newFixedThreadPool(parentSession.getColumnFamilyStores().size()); KeyspaceRepairManager repairManager = parentSession.getKeyspace().getRepairManager(); - ListenableFuture repairPreparation = prepareSession(repairManager, sessionID, parentSession.getColumnFamilyStores(), parentSession.getRanges(), executor); + RangesAtEndpoint tokenRanges = filterLocalRanges(parentSession.getKeyspace().getName(), parentSession.getRanges()); + ListenableFuture repairPreparation = prepareSession(repairManager, sessionID, parentSession.getColumnFamilyStores(), tokenRanges, executor); Futures.addCallback(repairPreparation, new FutureCallback() { diff --git a/src/java/org/apache/cassandra/schema/KeyspaceParams.java b/src/java/org/apache/cassandra/schema/KeyspaceParams.java index 68ac5e49ba..cc464743ff 100644 --- a/src/java/org/apache/cassandra/schema/KeyspaceParams.java +++ b/src/java/org/apache/cassandra/schema/KeyspaceParams.java @@ -74,6 +74,11 @@ public final class KeyspaceParams return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor)); } + public static KeyspaceParams simple(String replicationFactor) + { + return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor)); + } + public static KeyspaceParams simpleTransient(int replicationFactor) { return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor)); diff --git a/src/java/org/apache/cassandra/schema/ReplicationParams.java b/src/java/org/apache/cassandra/schema/ReplicationParams.java index 21c029e7c4..21e19d6ad5 100644 --- a/src/java/org/apache/cassandra/schema/ReplicationParams.java +++ b/src/java/org/apache/cassandra/schema/ReplicationParams.java @@ -51,6 +51,11 @@ public final class ReplicationParams return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", Integer.toString(replicationFactor))); } + static ReplicationParams simple(String replicationFactor) + { + return new ReplicationParams(SimpleStrategy.class, ImmutableMap.of("replication_factor", replicationFactor)); + } + static ReplicationParams nts(Object... args) { assert args.length % 2 == 0; @@ -58,9 +63,7 @@ public final class ReplicationParams Map options = new HashMap<>(); for (int i = 0; i < args.length; i += 2) { - String dc = (String) args[i]; - Integer rf = (Integer) args[i + 1]; - options.put(dc, rf.toString()); + options.put((String) args[i], args[i + 1].toString()); } return new ReplicationParams(NetworkTopologyStrategy.class, options); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index a88aebb681..22a8c39a55 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -137,6 +137,7 @@ public final class SchemaKeyspace + "min_index_interval int," + "read_repair_chance double," // no longer used, left for drivers' sake + "speculative_retry text," + + "speculative_write_threshold text," + "cdc boolean," + "read_repair text," + "PRIMARY KEY ((keyspace_name), table_name))"); @@ -203,6 +204,7 @@ public final class SchemaKeyspace + "min_index_interval int," + "read_repair_chance double," // no longer used, left for drivers' sake + "speculative_retry text," + + "speculative_write_threshold text," + "cdc boolean," + "read_repair text," + "PRIMARY KEY ((keyspace_name), view_name))"); @@ -563,6 +565,7 @@ public final class SchemaKeyspace .add("min_index_interval", params.minIndexInterval) .add("read_repair_chance", 0.0) // no longer used, left for drivers' sake .add("speculative_retry", params.speculativeRetry.toString()) + .add("speculative_write_threshold", params.speculativeWriteThreshold.toString()) .add("crc_check_chance", params.crcCheckChance) .add("caching", params.caching.asMap()) .add("compaction", params.compaction.asMap()) @@ -991,6 +994,7 @@ public final class SchemaKeyspace .minIndexInterval(row.getInt("min_index_interval")) .crcCheckChance(row.getDouble("crc_check_chance")) .speculativeRetry(SpeculativeRetryPolicy.fromString(row.getString("speculative_retry"))) + .speculativeWriteThreshold(SpeculativeRetryPolicy.fromString(row.getString("speculative_write_threshold"))) .cdc(row.has("cdc") && row.getBoolean("cdc")) .readRepair(getReadRepairStrategy(row)) .build(); diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 6466e2e31d..10edf4dc9e 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -834,6 +834,12 @@ public final class TableMetadata return this; } + public Builder speculativeWriteThreshold(SpeculativeRetryPolicy val) + { + params.speculativeWriteThreshold(val); + return this; + } + public Builder extensions(Map val) { params.extensions(val); diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index afbf26c877..0bba5e148d 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -51,6 +51,7 @@ public final class TableParams MEMTABLE_FLUSH_PERIOD_IN_MS, MIN_INDEX_INTERVAL, SPECULATIVE_RETRY, + SPECULATIVE_WRITE_THRESHOLD, CRC_CHECK_CHANCE, CDC, READ_REPAIR; @@ -71,6 +72,7 @@ public final class TableParams public final int minIndexInterval; public final int maxIndexInterval; public final SpeculativeRetryPolicy speculativeRetry; + public final SpeculativeRetryPolicy speculativeWriteThreshold; public final CachingParams caching; public final CompactionParams compaction; public final CompressionParams compression; @@ -91,6 +93,7 @@ public final class TableParams minIndexInterval = builder.minIndexInterval; maxIndexInterval = builder.maxIndexInterval; speculativeRetry = builder.speculativeRetry; + speculativeWriteThreshold = builder.speculativeWriteThreshold; caching = builder.caching; compaction = builder.compaction; compression = builder.compression; @@ -118,6 +121,7 @@ public final class TableParams .memtableFlushPeriodInMs(params.memtableFlushPeriodInMs) .minIndexInterval(params.minIndexInterval) .speculativeRetry(params.speculativeRetry) + .speculativeWriteThreshold(params.speculativeWriteThreshold) .extensions(params.extensions) .cdc(params.cdc) .readRepair(params.readRepair); @@ -260,6 +264,7 @@ public final class TableParams private int minIndexInterval = 128; private int maxIndexInterval = 2048; private SpeculativeRetryPolicy speculativeRetry = PercentileSpeculativeRetryPolicy.NINETY_NINE_P; + private SpeculativeRetryPolicy speculativeWriteThreshold = PercentileSpeculativeRetryPolicy.NINETY_NINE_P; private CachingParams caching = CachingParams.DEFAULT; private CompactionParams compaction = CompactionParams.DEFAULT; private CompressionParams compression = CompressionParams.DEFAULT; @@ -330,6 +335,12 @@ public final class TableParams return this; } + public Builder speculativeWriteThreshold(SpeculativeRetryPolicy val) + { + speculativeWriteThreshold = val; + return this; + } + public Builder caching(CachingParams val) { caching = val; diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java index 9d800a08bb..e817cc88c6 100644 --- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java @@ -17,28 +17,35 @@ */ package org.apache.cassandra.service; -import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.stream.Collectors; -import com.google.common.collect.Iterables; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.locator.ReplicaLayout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.IMutation; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.exceptions.WriteFailureException; +import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IAsyncCallbackWithFailure; import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.concurrent.SimpleCondition; + public abstract class AbstractWriteResponseHandler implements IAsyncCallbackWithFailure { protected static final Logger logger = LoggerFactory.getLogger(AbstractWriteResponseHandler.class); @@ -46,11 +53,9 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW //Count down until all responses and expirations have occured before deciding whether the ideal CL was reached. private AtomicInteger responsesAndExpirations; private final SimpleCondition condition = new SimpleCondition(); - protected final Keyspace keyspace; - protected final Collection naturalEndpoints; - public final ConsistencyLevel consistencyLevel; + protected final ReplicaLayout.ForToken replicaLayout; + protected final Runnable callback; - protected final Collection pendingEndpoints; protected final WriteType writeType; private static final AtomicIntegerFieldUpdater failuresUpdater = AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures"); @@ -60,7 +65,7 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW private volatile boolean supportsBackPressure = true; /** - * Delegate to another WriteReponseHandler or possibly this one to track if the ideal consistency level was reached. + * Delegate to another WriteResponseHandler or possibly this one to track if the ideal consistency level was reached. * Will be set to null if ideal CL was not configured * Will be set to an AWRH delegate if ideal CL was configured * Will be same as "this" if this AWRH is the ideal consistency level @@ -71,18 +76,12 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW * @param callback A callback to be called when the write is successful. * @param queryStartNanoTime */ - protected AbstractWriteResponseHandler(Keyspace keyspace, - Collection naturalEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistencyLevel, + protected AbstractWriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime) { - this.keyspace = keyspace; - this.pendingEndpoints = pendingEndpoints; - this.consistencyLevel = consistencyLevel; - this.naturalEndpoints = naturalEndpoints; + this.replicaLayout = replicaLayout; this.callback = callback; this.writeType = writeType; this.failureReasonByEndpoint = new ConcurrentHashMap<>(); @@ -112,12 +111,12 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW // avoid sending confusing info to the user (see CASSANDRA-6491). if (acks >= blockedFor) acks = blockedFor - 1; - throw new WriteTimeoutException(writeType, consistencyLevel, acks, blockedFor); + throw new WriteTimeoutException(writeType, replicaLayout.consistencyLevel(), acks, blockedFor); } if (totalBlockFor() + failures > totalEndpoints()) { - throw new WriteFailureException(consistencyLevel, ackCount(), totalBlockFor(), writeType, failureReasonByEndpoint); + throw new WriteFailureException(replicaLayout.consistencyLevel(), ackCount(), totalBlockFor(), writeType, failureReasonByEndpoint); } } @@ -136,7 +135,7 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW public void setIdealCLResponseHandler(AbstractWriteResponseHandler handler) { this.idealCLDelegate = handler; - idealCLDelegate.responsesAndExpirations = new AtomicInteger(naturalEndpoints.size() + pendingEndpoints.size()); + idealCLDelegate.responsesAndExpirations = new AtomicInteger(replicaLayout.selected().size()); } /** @@ -194,15 +193,20 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW { // During bootstrap, we have to include the pending endpoints or we may fail the consistency level // guarantees (see #833) - return consistencyLevel.blockFor(keyspace) + pendingEndpoints.size(); + return replicaLayout.consistencyLevel().blockForWrite(replicaLayout.keyspace(), replicaLayout.pending()); } /** - * @return the total number of endpoints the request has been sent to. + * @return the total number of endpoints the request can been sent to. */ protected int totalEndpoints() { - return naturalEndpoints.size() + pendingEndpoints.size(); + return replicaLayout.all().size(); + } + + public ConsistencyLevel consistencyLevel() + { + return replicaLayout.consistencyLevel(); } /** @@ -225,7 +229,7 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW public void assureSufficientLiveNodes() throws UnavailableException { - consistencyLevel.assureSufficientLiveNodes(keyspace, Iterables.filter(Iterables.concat(naturalEndpoints, pendingEndpoints), isAlive)); + replicaLayout.consistencyLevel().assureSufficientLiveNodesForWrite(replicaLayout.keyspace(), replicaLayout.all().filter(isReplicaAlive), replicaLayout.pending()); } protected void signal() @@ -274,12 +278,49 @@ public abstract class AbstractWriteResponseHandler implements IAsyncCallbackW //The condition being signaled is a valid proxy for the CL being achieved if (!condition.isSignaled()) { - keyspace.metric.writeFailedIdealCL.inc(); + replicaLayout.keyspace().metric.writeFailedIdealCL.inc(); } else { - keyspace.metric.idealCLWriteLatency.addNano(System.nanoTime() - queryStartNanoTime); + replicaLayout.keyspace().metric.idealCLWriteLatency.addNano(System.nanoTime() - queryStartNanoTime); } } } + + /** + * Cheap Quorum backup. If we failed to reach quorum with our initial (full) nodes, reach out to other nodes. + */ + public void maybeTryAdditionalReplicas(IMutation mutation, StorageProxy.WritePerformer writePerformer, String localDC) + { + if (replicaLayout.all().size() == replicaLayout.selected().size()) + return; + + long timeout = Long.MAX_VALUE; + List cfs = mutation.getTableIds().stream() + .map(Schema.instance::getColumnFamilyStoreInstance) + .collect(Collectors.toList()); + for (ColumnFamilyStore cf : cfs) + timeout = Math.min(timeout, cf.transientWriteLatencyNanos); + + // no latency information, or we're overloaded + if (timeout > TimeUnit.MILLISECONDS.toNanos(mutation.getTimeout())) + return; + + try + { + if (!condition.await(timeout, TimeUnit.NANOSECONDS)) + { + for (ColumnFamilyStore cf : cfs) + cf.metric.speculativeWrites.inc(); + + writePerformer.apply(mutation, replicaLayout.forNaturalUncontacted(), + (AbstractWriteResponseHandler) this, + localDC); + } + } + catch (InterruptedException ex) + { + throw new AssertionError(ex); + } + } } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index b60088cad6..9f37095199 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -38,6 +38,8 @@ import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import org.apache.cassandra.locator.EndpointsByRange; +import org.apache.cassandra.locator.EndpointsForRange; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,11 +62,14 @@ import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; import org.apache.cassandra.gms.IFailureDetectionEventListener; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replicas; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.net.IAsyncCallbackWithFailure; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.CommonRange; +import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.RepairParallelism; @@ -79,7 +84,8 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.UUIDGen; -import static org.apache.cassandra.config.Config.RepairCommandPoolFullStrategy.queue; +import static com.google.common.collect.Iterables.concat; +import static com.google.common.collect.Iterables.transform; /** * ActiveRepairService is the starting point for manual "active" repairs. @@ -204,10 +210,9 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai * @return Future for asynchronous call or null if there is no need to repair */ public RepairSession submitRepairSession(UUID parentRepairSession, - Collection> range, + CommonRange range, String keyspace, RepairParallelism parallelismDegree, - Set endpoints, boolean isIncremental, boolean pullRepair, boolean force, @@ -216,14 +221,15 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai ListeningExecutorService executor, String... cfnames) { - if (endpoints.isEmpty()) + if (range.endpoints.isEmpty()) return null; if (cfnames.length == 0) return null; - - final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(), range, keyspace, parallelismDegree, endpoints, isIncremental, pullRepair, force, previewKind, optimiseStreams, cfnames); + final RepairSession session = new RepairSession(parentRepairSession, UUIDGen.getTimeUUID(), range, keyspace, + parallelismDegree, isIncremental, pullRepair, force, + previewKind, optimiseStreams, cfnames); sessions.put(session.getId(), session); // register listeners @@ -296,12 +302,12 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai * * @return neighbors with whom we share the provided range */ - public static Set getNeighbors(String keyspaceName, Collection> keyspaceLocalRanges, - Range toRepair, Collection dataCenters, - Collection hosts) + public static EndpointsForRange getNeighbors(String keyspaceName, Iterable> keyspaceLocalRanges, + Range toRepair, Collection dataCenters, + Collection hosts) { StorageService ss = StorageService.instance; - Map, List> replicaSets = ss.getRangeToAddressMap(keyspaceName); + EndpointsByRange replicaSets = ss.getRangeToAddressMap(keyspaceName); Range rangeSuperSet = null; for (Range range : keyspaceLocalRanges) { @@ -319,23 +325,16 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } } if (rangeSuperSet == null || !replicaSets.containsKey(rangeSuperSet)) - return Collections.emptySet(); + return EndpointsForRange.empty(toRepair); - Set neighbors = new HashSet<>(replicaSets.get(rangeSuperSet)); - neighbors.remove(FBUtilities.getBroadcastAddressAndPort()); + EndpointsForRange neighbors = replicaSets.get(rangeSuperSet).withoutSelf(); if (dataCenters != null && !dataCenters.isEmpty()) { TokenMetadata.Topology topology = ss.getTokenMetadata().cloneOnlyTokenMap().getTopology(); - Set dcEndpoints = Sets.newHashSet(); - Multimap dcEndpointsMap = topology.getDatacenterEndpoints(); - for (String dc : dataCenters) - { - Collection c = dcEndpointsMap.get(dc); - if (c != null) - dcEndpoints.addAll(c); - } - return Sets.intersection(neighbors, dcEndpoints); + Multimap dcEndpointsMap = topology.getDatacenterEndpoints(); + Iterable dcEndpoints = concat(transform(dataCenters, dcEndpointsMap::get)); + return neighbors.keep(dcEndpoints); } else if (hosts != null && !hosts.isEmpty()) { @@ -345,7 +344,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai try { final InetAddressAndPort endpoint = InetAddressAndPort.getByName(host.trim()); - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.contains(endpoint)) + if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()) || neighbors.endpoints().contains(endpoint)) specifiedHost.add(endpoint); } catch (UnknownHostException e) @@ -366,8 +365,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai } specifiedHost.remove(FBUtilities.getBroadcastAddressAndPort()); - return specifiedHost; - + return neighbors.keep(specifiedHost); } return neighbors; @@ -594,10 +592,10 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai public Set getTableIds() { - return ImmutableSet.copyOf(Iterables.transform(getColumnFamilyStores(), cfs -> cfs.metadata.id)); + return ImmutableSet.copyOf(transform(getColumnFamilyStores(), cfs -> cfs.metadata.id)); } - public Collection> getRanges() + public Set> getRanges() { return ImmutableSet.copyOf(ranges); } diff --git a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java index e373fb6dcb..ee74df59ea 100644 --- a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java +++ b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java @@ -36,7 +36,7 @@ public class BatchlogResponseHandler extends AbstractWriteResponseHandler public BatchlogResponseHandler(AbstractWriteResponseHandler wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, long queryStartNanoTime) { - super(wrapped.keyspace, wrapped.naturalEndpoints, wrapped.pendingEndpoints, wrapped.consistencyLevel, wrapped.callback, wrapped.writeType, queryStartNanoTime); + super(wrapped.replicaLayout, wrapped.callback, wrapped.writeType, queryStartNanoTime); this.wrapped = wrapped; this.requiredBeforeFinish = requiredBeforeFinish; this.cleanup = cleanup; diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java index dbd3667759..d4cdcc6133 100644 --- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java @@ -17,16 +17,15 @@ */ package org.apache.cassandra.service; -import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.locator.IEndpointSnitch; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; @@ -41,29 +40,26 @@ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponse private final Map responses = new HashMap(); private final AtomicInteger acks = new AtomicInteger(0); - public DatacenterSyncWriteResponseHandler(Collection naturalEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistencyLevel, - Keyspace keyspace, + public DatacenterSyncWriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime) { // Response is been managed by the map so make it 1 for the superclass. - super(keyspace, naturalEndpoints, pendingEndpoints, consistencyLevel, callback, writeType, queryStartNanoTime); - assert consistencyLevel == ConsistencyLevel.EACH_QUORUM; + super(replicaLayout, callback, writeType, queryStartNanoTime); + assert replicaLayout.consistencyLevel() == ConsistencyLevel.EACH_QUORUM; - NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy(); + NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) replicaLayout.keyspace().getReplicationStrategy(); for (String dc : strategy.getDatacenters()) { - int rf = strategy.getReplicationFactor(dc); + int rf = strategy.getReplicationFactor(dc).allReplicas; responses.put(dc, new AtomicInteger((rf / 2) + 1)); } // During bootstrap, we have to include the pending endpoints or we may fail the consistency level // guarantees (see #833) - for (InetAddressAndPort pending : pendingEndpoints) + for (Replica pending : replicaLayout.pending()) { responses.get(snitch.getDatacenter(pending)).incrementAndGet(); } @@ -105,4 +101,5 @@ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponse { return false; } + } diff --git a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java index a8d7b28a00..b458a71395 100644 --- a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java @@ -17,29 +17,23 @@ */ package org.apache.cassandra.service; -import java.util.Collection; - -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.net.MessageIn; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.net.MessageIn; /** * This class blocks for a quorum of responses _in the local datacenter only_ (CL.LOCAL_QUORUM). */ public class DatacenterWriteResponseHandler extends WriteResponseHandler { - public DatacenterWriteResponseHandler(Collection naturalEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistencyLevel, - Keyspace keyspace, + public DatacenterWriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime) { - super(naturalEndpoints, pendingEndpoints, consistencyLevel, keyspace, callback, writeType, queryStartNanoTime); - assert consistencyLevel.isDatacenterLocal(); + super(replicaLayout, callback, writeType, queryStartNanoTime); + assert replicaLayout.consistencyLevel().isDatacenterLocal(); } @Override @@ -57,17 +51,9 @@ public class DatacenterWriteResponseHandler extends WriteResponseHandler } } - @Override - protected int totalBlockFor() - { - // during bootstrap, include pending endpoints (only local here) in the count - // or we may fail the consistency level guarantees (see #833, #8058) - return consistencyLevel.blockFor(keyspace) + consistencyLevel.countLocalEndpoints(pendingEndpoints); - } - @Override protected boolean waitingFor(InetAddressAndPort from) { - return consistencyLevel.isLocal(from); + return replicaLayout.consistencyLevel().isLocal(from); } } diff --git a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java index e1c0f550c5..7b6bd58ae5 100644 --- a/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java +++ b/src/java/org/apache/cassandra/service/PendingRangeCalculatorService.java @@ -92,7 +92,7 @@ public class PendingRangeCalculatorService { int jobs = updateJobs.incrementAndGet(); PendingRangeCalculatorServiceDiagnostics.taskCountChanged(instance, jobs); - executor.submit(new PendingRangeTask(updateJobs)); + executor.execute(new PendingRangeTask(updateJobs)); } public void blockUntilFinished() diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index ed0cafc5b5..c23eb8810c 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -29,7 +29,6 @@ import javax.management.MBeanServer; import javax.management.ObjectName; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; import com.google.common.cache.CacheLoader; import com.google.common.collect.*; import com.google.common.primitives.Ints; @@ -133,18 +132,10 @@ public class StorageProxy implements StorageProxyMBean HintsService.instance.registerMBean(); HintedHandOffManager.instance.registerMBean(); - standardWritePerformer = new WritePerformer() + standardWritePerformer = (mutation, targets, responseHandler, localDataCenter) -> { - public void apply(IMutation mutation, - Iterable targets, - AbstractWriteResponseHandler responseHandler, - String localDataCenter, - ConsistencyLevel consistency_level) - throws OverloadedException - { - assert mutation instanceof Mutation; - sendToHintedEndpoints((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION); - } + assert mutation instanceof Mutation; + sendToHintedReplicas((Mutation) mutation, targets.selected(), responseHandler, localDataCenter, Stage.MUTATION); }; /* @@ -153,29 +144,19 @@ public class StorageProxy implements StorageProxyMBean * but on the latter case, the verb handler already run on the COUNTER_MUTATION stage, so we must not execute the * underlying on the stage otherwise we risk a deadlock. Hence two different performer. */ - counterWritePerformer = new WritePerformer() + counterWritePerformer = (mutation, targets, responseHandler, localDataCenter) -> { - public void apply(IMutation mutation, - Iterable targets, - AbstractWriteResponseHandler responseHandler, - String localDataCenter, - ConsistencyLevel consistencyLevel) - { - counterWriteTask(mutation, targets, responseHandler, localDataCenter).run(); - } + EndpointsForToken selected = targets.selected().withoutSelf(); + Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548 + counterWriteTask(mutation, selected, responseHandler, localDataCenter).run(); }; - counterWriteOnCoordinatorPerformer = new WritePerformer() + counterWriteOnCoordinatorPerformer = (mutation, targets, responseHandler, localDataCenter) -> { - public void apply(IMutation mutation, - Iterable targets, - AbstractWriteResponseHandler responseHandler, - String localDataCenter, - ConsistencyLevel consistencyLevel) - { - StageManager.getStage(Stage.COUNTER_MUTATION) - .execute(counterWriteTask(mutation, targets, responseHandler, localDataCenter)); - } + EndpointsForToken selected = targets.selected().withoutSelf(); + Replicas.temporaryAssertFull(selected); // TODO CASSANDRA-14548 + StageManager.getStage(Stage.COUNTER_MUTATION) + .execute(counterWriteTask(mutation, selected, responseHandler, localDataCenter)); }; for(ConsistencyLevel level : ConsistencyLevel.values()) @@ -251,11 +232,9 @@ public class StorageProxy implements StorageProxyMBean while (System.nanoTime() - queryStartNanoTime < timeout) { // for simplicity, we'll do a single liveness check at the start of each attempt - PaxosParticipants p = getPaxosParticipants(metadata, key, consistencyForPaxos); - List liveEndpoints = p.liveEndpoints; - int requiredParticipants = p.participants; + ReplicaLayout.ForPaxos replicaLayout = ReplicaLayout.forPaxos(Keyspace.open(keyspaceName), key, consistencyForPaxos); - final PaxosBallotAndContention pair = beginAndRepairPaxos(queryStartNanoTime, key, metadata, liveEndpoints, requiredParticipants, consistencyForPaxos, consistencyForCommit, true, state); + final PaxosBallotAndContention pair = beginAndRepairPaxos(queryStartNanoTime, key, metadata, replicaLayout, consistencyForPaxos, consistencyForCommit, true, state); final UUID ballot = pair.ballot; contentions += pair.contentions; @@ -297,7 +276,7 @@ public class StorageProxy implements StorageProxyMBean Commit proposal = Commit.newProposal(ballot, updates); Tracing.trace("CAS precondition is met; proposing client-requested updates for {}", ballot); - if (proposePaxos(proposal, liveEndpoints, requiredParticipants, true, consistencyForPaxos, queryStartNanoTime)) + if (proposePaxos(proposal, replicaLayout, true, queryStartNanoTime)) { commitPaxos(proposal, consistencyForCommit, true, queryStartNanoTime); Tracing.trace("CAS successful"); @@ -346,49 +325,6 @@ public class StorageProxy implements StorageProxyMBean casWriteMetrics.contention.update(contentions); } - private static Predicate sameDCPredicateFor(final String dc) - { - final IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - return new Predicate() - { - public boolean apply(InetAddressAndPort host) - { - return dc.equals(snitch.getDatacenter(host)); - } - }; - } - - private static PaxosParticipants getPaxosParticipants(TableMetadata metadata, DecoratedKey key, ConsistencyLevel consistencyForPaxos) throws UnavailableException - { - Token tk = key.getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(metadata.keyspace, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, metadata.keyspace); - if (consistencyForPaxos == ConsistencyLevel.LOCAL_SERIAL) - { - // Restrict naturalEndpoints and pendingEndpoints to node in the local DC only - String localDc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); - Predicate isLocalDc = sameDCPredicateFor(localDc); - naturalEndpoints = ImmutableList.copyOf(Iterables.filter(naturalEndpoints, isLocalDc)); - pendingEndpoints = ImmutableList.copyOf(Iterables.filter(pendingEndpoints, isLocalDc)); - } - int participants = pendingEndpoints.size() + naturalEndpoints.size(); - int requiredParticipants = participants / 2 + 1; // See CASSANDRA-8346, CASSANDRA-833 - List liveEndpoints = ImmutableList.copyOf(Iterables.filter(Iterables.concat(naturalEndpoints, pendingEndpoints), IAsyncCallback.isAlive)); - if (liveEndpoints.size() < requiredParticipants) - throw new UnavailableException(consistencyForPaxos, requiredParticipants, liveEndpoints.size()); - - // We cannot allow CAS operations with 2 or more pending endpoints, see #8346. - // Note that we fake an impossible number of required nodes in the unavailable exception - // to nail home the point that it's an impossible operation no matter how many nodes are live. - if (pendingEndpoints.size() > 1) - throw new UnavailableException(String.format("Cannot perform LWT operation as there is more than one (%d) pending range movement", pendingEndpoints.size()), - consistencyForPaxos, - participants + 1, - liveEndpoints.size()); - - return new PaxosParticipants(liveEndpoints, requiredParticipants); - } - /** * begin a Paxos session by sending a prepare request and completing any in-progress requests seen in the replies * @@ -396,14 +332,13 @@ public class StorageProxy implements StorageProxyMBean * nodes have seen the mostRecentCommit. Otherwise, return null. */ private static PaxosBallotAndContention beginAndRepairPaxos(long queryStartNanoTime, - DecoratedKey key, - TableMetadata metadata, - List liveEndpoints, - int requiredParticipants, - ConsistencyLevel consistencyForPaxos, - ConsistencyLevel consistencyForCommit, - final boolean isWrite, - ClientState state) + DecoratedKey key, + TableMetadata metadata, + ReplicaLayout.ForPaxos replicaLayout, + ConsistencyLevel consistencyForPaxos, + ConsistencyLevel consistencyForCommit, + final boolean isWrite, + ClientState state) throws WriteTimeoutException, WriteFailureException { long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getCasContentionTimeout()); @@ -425,7 +360,7 @@ public class StorageProxy implements StorageProxyMBean // prepare Tracing.trace("Preparing {}", ballot); Commit toPrepare = Commit.newPrepare(key, metadata, ballot); - summary = preparePaxos(toPrepare, liveEndpoints, requiredParticipants, consistencyForPaxos, queryStartNanoTime); + summary = preparePaxos(toPrepare, replicaLayout, queryStartNanoTime); if (!summary.promised) { Tracing.trace("Some replicas have already promised a higher ballot than ours; aborting"); @@ -448,7 +383,7 @@ public class StorageProxy implements StorageProxyMBean else casReadMetrics.unfinishedCommit.inc(); Commit refreshedInProgress = Commit.newProposal(ballot, inProgress.update); - if (proposePaxos(refreshedInProgress, liveEndpoints, requiredParticipants, false, consistencyForPaxos, queryStartNanoTime)) + if (proposePaxos(refreshedInProgress, replicaLayout, false, queryStartNanoTime)) { try { @@ -505,14 +440,14 @@ public class StorageProxy implements StorageProxyMBean MessagingService.instance().sendOneWay(message, target); } - private static PrepareCallback preparePaxos(Commit toPrepare, List endpoints, int requiredParticipants, ConsistencyLevel consistencyForPaxos, long queryStartNanoTime) + private static PrepareCallback preparePaxos(Commit toPrepare, ReplicaLayout.ForPaxos replicaLayout, long queryStartNanoTime) throws WriteTimeoutException { - PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), requiredParticipants, consistencyForPaxos, queryStartNanoTime); + PrepareCallback callback = new PrepareCallback(toPrepare.update.partitionKey(), toPrepare.update.metadata(), replicaLayout.getRequiredParticipants(), replicaLayout.consistencyLevel(), queryStartNanoTime); MessageOut message = new MessageOut(MessagingService.Verb.PAXOS_PREPARE, toPrepare, Commit.serializer); - for (InetAddressAndPort target : endpoints) + for (Replica replica: replicaLayout.selected()) { - if (canDoLocalRequest(target)) + if (replica.isLocal()) { StageManager.getStage(MessagingService.verbStages.get(MessagingService.Verb.PAXOS_PREPARE)).execute(new Runnable() { @@ -536,21 +471,21 @@ public class StorageProxy implements StorageProxyMBean } else { - MessagingService.instance().sendRR(message, target, callback); + MessagingService.instance().sendRR(message, replica.endpoint(), callback); } } callback.await(); return callback; } - private static boolean proposePaxos(Commit proposal, List endpoints, int requiredParticipants, boolean timeoutIfPartial, ConsistencyLevel consistencyLevel, long queryStartNanoTime) + private static boolean proposePaxos(Commit proposal, ReplicaLayout.ForPaxos replicaLayout, boolean timeoutIfPartial, long queryStartNanoTime) throws WriteTimeoutException { - ProposeCallback callback = new ProposeCallback(endpoints.size(), requiredParticipants, !timeoutIfPartial, consistencyLevel, queryStartNanoTime); + ProposeCallback callback = new ProposeCallback(replicaLayout.selected().size(), replicaLayout.getRequiredParticipants(), !timeoutIfPartial, replicaLayout.consistencyLevel(), queryStartNanoTime); MessageOut message = new MessageOut(MessagingService.Verb.PAXOS_PROPOSE, proposal, Commit.serializer); - for (InetAddressAndPort target : endpoints) + for (Replica replica : replicaLayout.selected()) { - if (canDoLocalRequest(target)) + if (replica.isLocal()) { StageManager.getStage(MessagingService.verbStages.get(MessagingService.Verb.PAXOS_PROPOSE)).execute(new Runnable() { @@ -574,7 +509,7 @@ public class StorageProxy implements StorageProxyMBean } else { - MessagingService.instance().sendRR(message, target, callback); + MessagingService.instance().sendRR(message, replica.endpoint(), callback); } } callback.await(); @@ -583,7 +518,7 @@ public class StorageProxy implements StorageProxyMBean return true; if (timeoutIfPartial && !callback.isFullyRefused()) - throw new WriteTimeoutException(WriteType.CAS, consistencyLevel, callback.getAcceptCount(), requiredParticipants); + throw new WriteTimeoutException(WriteType.CAS, replicaLayout.consistencyLevel(), callback.getAcceptCount(), replicaLayout.getRequiredParticipants()); return false; } @@ -594,30 +529,30 @@ public class StorageProxy implements StorageProxyMBean Keyspace keyspace = Keyspace.open(proposal.update.metadata().keyspace); Token tk = proposal.update.partitionKey().getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspace.getName(), tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspace.getName()); AbstractWriteResponseHandler responseHandler = null; + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWrite(keyspace, consistencyLevel, tk, FailureDetector.instance::isAlive); if (shouldBlock) { AbstractReplicationStrategy rs = keyspace.getReplicationStrategy(); - responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistencyLevel, null, WriteType.SIMPLE, queryStartNanoTime); + responseHandler = rs.getWriteResponseHandler(replicaLayout, null, WriteType.SIMPLE, queryStartNanoTime); responseHandler.setSupportsBackPressure(false); } MessageOut message = new MessageOut(MessagingService.Verb.PAXOS_COMMIT, proposal, Commit.serializer); - for (InetAddressAndPort destination : Iterables.concat(naturalEndpoints, pendingEndpoints)) + for (Replica replica : replicaLayout.all()) { - checkHintOverload(destination); + InetAddressAndPort destination = replica.endpoint(); + checkHintOverload(replica); if (FailureDetector.instance.isAlive(destination)) { if (shouldBlock) { - if (canDoLocalRequest(destination)) - commitPaxosLocal(message, responseHandler); + if (replica.isLocal()) + commitPaxosLocal(replica, message, responseHandler); else - MessagingService.instance().sendRR(message, destination, responseHandler, allowHints && shouldHint(destination)); + MessagingService.instance().sendWriteRR(message, replica, responseHandler, allowHints && shouldHint(replica)); } else { @@ -630,9 +565,9 @@ public class StorageProxy implements StorageProxyMBean { responseHandler.expired(); } - if (allowHints && shouldHint(destination)) + if (allowHints && shouldHint(replica)) { - submitHint(proposal.makeMutation(), destination, null); + submitHint(proposal.makeMutation(), replica, null); } } } @@ -646,9 +581,9 @@ public class StorageProxy implements StorageProxyMBean * submit a fake one that executes immediately on the mutation stage, but generates the necessary backpressure * signal for hints */ - private static void commitPaxosLocal(final MessageOut message, final AbstractWriteResponseHandler responseHandler) + private static void commitPaxosLocal(Replica localReplica, final MessageOut message, final AbstractWriteResponseHandler responseHandler) { - StageManager.getStage(MessagingService.verbStages.get(MessagingService.Verb.PAXOS_COMMIT)).maybeExecuteImmediately(new LocalMutationRunnable() + StageManager.getStage(MessagingService.verbStages.get(MessagingService.Verb.PAXOS_COMMIT)).maybeExecuteImmediately(new LocalMutationRunnable(localReplica) { public void runMayThrow() { @@ -684,35 +619,37 @@ public class StorageProxy implements StorageProxyMBean * @param consistency_level the consistency level for the operation * @param queryStartNanoTime the value of System.nanoTime() when the query started to be processed */ - public static void mutate(Collection mutations, ConsistencyLevel consistency_level, long queryStartNanoTime) + public static void mutate(List mutations, ConsistencyLevel consistency_level, long queryStartNanoTime) throws UnavailableException, OverloadedException, WriteTimeoutException, WriteFailureException { Tracing.trace("Determining replicas for mutation"); final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); long startTime = System.nanoTime(); + List> responseHandlers = new ArrayList<>(mutations.size()); + WriteType plainWriteType = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH; try { for (IMutation mutation : mutations) { if (mutation instanceof CounterMutation) - { responseHandlers.add(mutateCounter((CounterMutation)mutation, localDataCenter, queryStartNanoTime)); - } else - { - WriteType wt = mutations.size() <= 1 ? WriteType.SIMPLE : WriteType.UNLOGGED_BATCH; - responseHandlers.add(performWrite(mutation, consistency_level, localDataCenter, standardWritePerformer, null, wt, queryStartNanoTime)); - } + responseHandlers.add(performWrite(mutation, consistency_level, localDataCenter, standardWritePerformer, null, plainWriteType, queryStartNanoTime)); + } + + // upgrade to full quorum any failed cheap quorums + for (int i = 0 ; i < mutations.size() ; ++i) + { + if (!(mutations.get(i) instanceof CounterMutation)) // at the moment, only non-counter writes support cheap quorums + responseHandlers.get(i).maybeTryAdditionalReplicas(mutations.get(i), standardWritePerformer, localDataCenter); } // wait for writes. throws TimeoutException if necessary for (AbstractWriteResponseHandler responseHandler : responseHandlers) - { responseHandler.get(); - } } catch (WriteTimeoutException|WriteFailureException ex) { @@ -786,16 +723,12 @@ public class StorageProxy implements StorageProxyMBean String keyspaceName = mutation.getKeyspaceName(); Token token = mutation.key().getToken(); - Iterable endpoints = StorageService.instance.getNaturalAndPendingEndpoints(keyspaceName, token); - ArrayList endpointsToHint = new ArrayList<>(Iterables.size(endpoints)); - // local writes can timeout, but cannot be dropped (see LocalMutationRunnable and CASSANDRA-6510), // so there is no need to hint or retry. - for (InetAddressAndPort target : endpoints) - if (!target.equals(FBUtilities.getBroadcastAddressAndPort()) && shouldHint(target)) - endpointsToHint.add(target); + EndpointsForToken replicasToHint = StorageService.instance.getNaturalAndPendingReplicasForToken(keyspaceName, token) + .filter(StorageProxy::shouldHint); - submitHint(mutation, endpointsToHint, null); + submitHint(mutation, replicasToHint, null); } public boolean appliesLocally(Mutation mutation) @@ -804,8 +737,8 @@ public class StorageProxy implements StorageProxyMBean Token token = mutation.key().getToken(); InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); - return StorageService.instance.getNaturalEndpoints(keyspaceName, token).contains(local) - || StorageService.instance.getTokenMetadata().pendingEndpointsFor(token, keyspaceName).contains(local); + return StorageService.instance.getNaturalReplicasForToken(keyspaceName, token).endpoints().contains(local) + || StorageService.instance.getTokenMetadata().pendingEndpointsForToken(token, keyspaceName).endpoints().contains(local); } /** @@ -854,13 +787,13 @@ public class StorageProxy implements StorageProxyMBean { String keyspaceName = mutation.getKeyspaceName(); Token tk = mutation.key().getToken(); - Optional pairedEndpoint = ViewUtils.getViewNaturalEndpoint(keyspaceName, baseToken, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); + Optional pairedEndpoint = ViewUtils.getViewNaturalEndpoint(keyspaceName, baseToken, tk); + EndpointsForToken pendingReplicas = StorageService.instance.getTokenMetadata().pendingEndpointsForToken(tk, keyspaceName); // if there are no paired endpoints there are probably range movements going on, so we write to the local batchlog to replay later if (!pairedEndpoint.isPresent()) { - if (pendingEndpoints.isEmpty()) + if (pendingReplicas.isEmpty()) logger.warn("Received base materialized view mutation for key {} that does not belong " + "to this node. There is probably a range movement happening (move or decommission)," + "but this node hasn't updated its ring metadata yet. Adding mutation to " + @@ -872,8 +805,8 @@ public class StorageProxy implements StorageProxyMBean // When local node is the endpoint we can just apply the mutation locally, // unless there are pending endpoints, in which case we want to do an ordinary // write so the view mutation is sent to the pending endpoint - if (pairedEndpoint.get().equals(FBUtilities.getBroadcastAddressAndPort()) && StorageService.instance.isJoined() - && pendingEndpoints.isEmpty()) + if (pairedEndpoint.get().isLocal() && StorageService.instance.isJoined() + && pendingReplicas.isEmpty()) { try { @@ -892,7 +825,8 @@ public class StorageProxy implements StorageProxyMBean wrappers.add(wrapViewBatchResponseHandler(mutation, consistencyLevel, consistencyLevel, - Collections.singletonList(pairedEndpoint.get()), + EndpointsForToken.of(tk, pairedEndpoint.get()), + pendingReplicas, baseComplete, WriteType.BATCH, cleanup, @@ -916,7 +850,7 @@ public class StorageProxy implements StorageProxyMBean } @SuppressWarnings("unchecked") - public static void mutateWithTriggers(Collection mutations, + public static void mutateWithTriggers(List mutations, ConsistencyLevel consistencyLevel, boolean mutateAtomically, long queryStartNanoTime) @@ -966,6 +900,9 @@ public class StorageProxy implements StorageProxyMBean List wrappers = new ArrayList(mutations.size()); String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort()); + if (mutations.stream().anyMatch(mutation -> Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy().hasTransientReplicas())) + throw new AssertionError("Logged batches are unsupported with transient replication"); + try { @@ -982,7 +919,7 @@ public class StorageProxy implements StorageProxyMBean batchConsistencyLevel = consistency_level; } - final Collection batchlogEndpoints = getBatchlogEndpoints(localDataCenter, batchConsistencyLevel); + final Collection batchlogEndpoints = getBatchlogReplicas(localDataCenter, batchConsistencyLevel); final UUID batchUUID = UUIDGen.getTimeUUID(); BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(), () -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID)); @@ -1037,11 +974,6 @@ public class StorageProxy implements StorageProxyMBean } } - public static boolean canDoLocalRequest(InetAddressAndPort replica) - { - return replica.equals(FBUtilities.getBroadcastAddressAndPort()); - } - private static void updateCoordinatorWriteLatencyTableMetric(Collection mutations, long latency) { if (null == mutations) @@ -1069,24 +1001,22 @@ public class StorageProxy implements StorageProxyMBean private static void syncWriteToBatchlog(Collection mutations, Collection endpoints, UUID uuid, long queryStartNanoTime) throws WriteTimeoutException, WriteFailureException { - WriteResponseHandler handler = new WriteResponseHandler<>(endpoints, - Collections.emptyList(), - endpoints.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO, - Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME), - null, - WriteType.BATCH_LOG, - queryStartNanoTime); + Keyspace systemKeypsace = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME); + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forBatchlogWrite(systemKeypsace, endpoints); + WriteResponseHandler handler = new WriteResponseHandler(replicaLayout, + WriteType.BATCH_LOG, + queryStartNanoTime); Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations); MessageOut message = new MessageOut<>(MessagingService.Verb.BATCH_STORE, batch, Batch.serializer); - for (InetAddressAndPort target : endpoints) + for (Replica replica : replicaLayout.all()) { - logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, target, batch.size()); + logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, replica, batch.size()); - if (canDoLocalRequest(target)) - performLocally(Stage.MUTATION, Optional.empty(), () -> BatchlogManager.store(batch), handler); + if (replica.isLocal()) + performLocally(Stage.MUTATION, replica, Optional.empty(), () -> BatchlogManager.store(batch), handler); else - MessagingService.instance().sendRR(message, target, handler); + MessagingService.instance().sendRR(message, replica.endpoint(), handler); } handler.get(); } @@ -1099,8 +1029,8 @@ public class StorageProxy implements StorageProxyMBean if (logger.isTraceEnabled()) logger.trace("Sending batchlog remove request {} to {}", uuid, target); - if (canDoLocalRequest(target)) - performLocally(Stage.MUTATION, () -> BatchlogManager.remove(uuid)); + if (target.equals(FBUtilities.getBroadcastAddressAndPort())) + performLocally(Stage.MUTATION, SystemReplicas.getSystemReplica(target), () -> BatchlogManager.remove(uuid)); else MessagingService.instance().sendOneWay(message, target); } @@ -1110,11 +1040,12 @@ public class StorageProxy implements StorageProxyMBean { for (WriteResponseHandlerWrapper wrapper : wrappers) { - Iterable endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints); + Replicas.temporaryAssertFull(wrapper.handler.replicaLayout.all()); // TODO: CASSANDRA-14549 + ReplicaLayout.ForToken replicas = wrapper.handler.replicaLayout.withSelected(wrapper.handler.replicaLayout.all()); try { - sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage); + sendToHintedReplicas(wrapper.mutation, replicas.selected(), wrapper.handler, localDataCenter, stage); } catch (OverloadedException | WriteTimeoutException e) { @@ -1128,8 +1059,8 @@ public class StorageProxy implements StorageProxyMBean { for (WriteResponseHandlerWrapper wrapper : wrappers) { - Iterable endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints); - sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage); + Replicas.temporaryAssertFull(wrapper.handler.replicaLayout.all()); // TODO: CASSANDRA-14549 + sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaLayout.all(), wrapper.handler, localDataCenter, stage); } @@ -1144,7 +1075,7 @@ public class StorageProxy implements StorageProxyMBean * responses based on consistency level. * * @param mutation the mutation to be applied - * @param consistency_level the consistency level for the write operation + * @param consistencyLevel the consistency level for the write operation * @param performer the WritePerformer in charge of appliying the mutation * given the list of write endpoints (either standardWritePerformer for * standard writes or counterWritePerformer for counter writes). @@ -1152,33 +1083,32 @@ public class StorageProxy implements StorageProxyMBean * @param queryStartNanoTime the value of System.nanoTime() when the query started to be processed */ public static AbstractWriteResponseHandler performWrite(IMutation mutation, - ConsistencyLevel consistency_level, + ConsistencyLevel consistencyLevel, String localDataCenter, WritePerformer performer, Runnable callback, WriteType writeType, long queryStartNanoTime) - throws UnavailableException, OverloadedException { String keyspaceName = mutation.getKeyspaceName(); - AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy(); + Keyspace keyspace = Keyspace.open(keyspaceName); + AbstractReplicationStrategy rs = keyspace.getReplicationStrategy(); Token tk = mutation.key().getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); - AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, callback, writeType, queryStartNanoTime); + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWriteWithDownNodes(keyspace, consistencyLevel, tk); + AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(replicaLayout, callback, writeType, queryStartNanoTime); // exit early if we can't fulfill the CL at this time responseHandler.assureSufficientLiveNodes(); - performer.apply(mutation, Iterables.concat(naturalEndpoints, pendingEndpoints), responseHandler, localDataCenter, consistency_level); + performer.apply(mutation, replicaLayout, responseHandler, localDataCenter); return responseHandler; } // same as performWrites except does not initiate writes (but does perform availability checks). private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation, - ConsistencyLevel consistency_level, + ConsistencyLevel consistencyLevel, ConsistencyLevel batchConsistencyLevel, WriteType writeType, BatchlogResponseHandler.BatchlogCleanup cleanup, @@ -1186,11 +1116,10 @@ public class StorageProxy implements StorageProxyMBean { Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName()); AbstractReplicationStrategy rs = keyspace.getReplicationStrategy(); - String keyspaceName = mutation.getKeyspaceName(); Token tk = mutation.key().getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); - AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType, queryStartNanoTime); + + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWrite(keyspace, consistencyLevel, tk, FailureDetector.instance::isAlive); + AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(replicaLayout,null, writeType, queryStartNanoTime); BatchlogResponseHandler batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(keyspace), cleanup, queryStartNanoTime); return new WriteResponseHandlerWrapper(batchHandler, mutation); } @@ -1200,9 +1129,10 @@ public class StorageProxy implements StorageProxyMBean * Keeps track of ViewWriteMetrics */ private static WriteResponseHandlerWrapper wrapViewBatchResponseHandler(Mutation mutation, - ConsistencyLevel consistency_level, + ConsistencyLevel consistencyLevel, ConsistencyLevel batchConsistencyLevel, - List naturalEndpoints, + EndpointsForToken naturalEndpoints, + EndpointsForToken pendingEndpoints, AtomicLong baseComplete, WriteType writeType, BatchlogResponseHandler.BatchlogCleanup cleanup, @@ -1210,10 +1140,10 @@ public class StorageProxy implements StorageProxyMBean { Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName()); AbstractReplicationStrategy rs = keyspace.getReplicationStrategy(); - String keyspaceName = mutation.getKeyspaceName(); Token tk = mutation.key().getToken(); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); - AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, () -> { + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWriteWithDownNodes(keyspace, consistencyLevel, tk, naturalEndpoints, pendingEndpoints); + + AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(replicaLayout, () -> { long delay = Math.max(0, System.currentTimeMillis() - baseComplete.get()); viewWriteMetrics.viewWriteLatency.update(delay, TimeUnit.MILLISECONDS); }, writeType, queryStartNanoTime); @@ -1241,7 +1171,7 @@ public class StorageProxy implements StorageProxyMBean * - choose min(2, number of qualifying candiates above) * - allow the local node to be the only replica only if it's a single-node DC */ - private static Collection getBatchlogEndpoints(String localDataCenter, ConsistencyLevel consistencyLevel) + private static Collection getBatchlogReplicas(String localDataCenter, ConsistencyLevel consistencyLevel) throws UnavailableException { TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology(); @@ -1254,7 +1184,7 @@ public class StorageProxy implements StorageProxyMBean if (consistencyLevel == ConsistencyLevel.ANY) return Collections.singleton(FBUtilities.getBroadcastAddressAndPort()); - throw new UnavailableException(ConsistencyLevel.ONE, 1, 0); + throw UnavailableException.create(ConsistencyLevel.ONE, 1, 0); } return chosenEndpoints; @@ -1277,36 +1207,36 @@ public class StorageProxy implements StorageProxyMBean * * @throws OverloadedException if the hints cannot be written/enqueued */ - public static void sendToHintedEndpoints(final Mutation mutation, - Iterable targets, - AbstractWriteResponseHandler responseHandler, - String localDataCenter, - Stage stage) + public static void sendToHintedReplicas(final Mutation mutation, + EndpointsForToken targets, + AbstractWriteResponseHandler responseHandler, + String localDataCenter, + Stage stage) throws OverloadedException { - int targetsSize = Iterables.size(targets); - // this dc replicas: - Collection localDc = null; + Collection localDc = null; // extra-datacenter replicas, grouped by dc - Map> dcGroups = null; + Map> dcGroups = null; // only need to create a Message for non-local writes MessageOut message = null; boolean insertLocal = false; - ArrayList endpointsToHint = null; + Replica localReplica = null; + Collection endpointsToHint = null; List backPressureHosts = null; - for (InetAddressAndPort destination : targets) + for (Replica destination : targets) { checkHintOverload(destination); - if (FailureDetector.instance.isAlive(destination)) + if (FailureDetector.instance.isAlive(destination.endpoint())) { - if (canDoLocalRequest(destination)) + if (destination.isLocal()) { insertLocal = true; + localReplica = destination; } else { @@ -1321,28 +1251,26 @@ public class StorageProxy implements StorageProxyMBean if (localDataCenter.equals(dc)) { if (localDc == null) - localDc = new ArrayList<>(targetsSize); + localDc = new ArrayList<>(targets.size()); localDc.add(destination); } else { - Collection messages = (dcGroups != null) ? dcGroups.get(dc) : null; + if (dcGroups == null) + dcGroups = new HashMap<>(); + + Collection messages = dcGroups.get(dc); if (messages == null) - { - messages = new ArrayList<>(3); // most DCs will have <= 3 replicas - if (dcGroups == null) - dcGroups = new HashMap<>(); - dcGroups.put(dc, messages); - } + messages = dcGroups.computeIfAbsent(dc, (v) -> new ArrayList<>(3)); // most DCs will have <= 3 replicas messages.add(destination); } if (backPressureHosts == null) - backPressureHosts = new ArrayList<>(targetsSize); + backPressureHosts = new ArrayList<>(targets.size()); - backPressureHosts.add(destination); + backPressureHosts.add(destination.endpoint()); } } else @@ -1352,7 +1280,7 @@ public class StorageProxy implements StorageProxyMBean if (shouldHint(destination)) { if (endpointsToHint == null) - endpointsToHint = new ArrayList<>(targetsSize); + endpointsToHint = new ArrayList<>(); endpointsToHint.add(destination); } @@ -1363,25 +1291,28 @@ public class StorageProxy implements StorageProxyMBean MessagingService.instance().applyBackPressure(backPressureHosts, responseHandler.currentTimeout()); if (endpointsToHint != null) - submitHint(mutation, endpointsToHint, responseHandler); + submitHint(mutation, EndpointsForToken.copyOf(mutation.key().getToken(), endpointsToHint), responseHandler); if (insertLocal) - performLocally(stage, Optional.of(mutation), mutation::apply, responseHandler); + { + Preconditions.checkNotNull(localReplica); + performLocally(stage, localReplica, Optional.of(mutation), mutation::apply, responseHandler); + } if (localDc != null) { - for (InetAddressAndPort destination : localDc) - MessagingService.instance().sendRR(message, destination, responseHandler, true); + for (Replica destination : localDc) + MessagingService.instance().sendWriteRR(message, destination, responseHandler, true); } if (dcGroups != null) { // for each datacenter, send the message to one node to relay the write to other replicas - for (Collection dcTargets : dcGroups.values()) - sendMessagesToNonlocalDC(message, dcTargets, responseHandler); + for (Collection dcTargets : dcGroups.values()) + sendMessagesToNonlocalDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcTargets), responseHandler); } } - private static void checkHintOverload(InetAddressAndPort destination) + private static void checkHintOverload(Replica destination) { // avoid OOMing due to excess hints. we need to do this check even for "live" nodes, since we can // still generate hints for those if it's overloaded or simply dead but not yet known-to-be-dead. @@ -1389,45 +1320,46 @@ public class StorageProxy implements StorageProxyMBean // a small number of nodes causing problems, so we should avoid shutting down writes completely to // healthy nodes. Any node with no hintsInProgress is considered healthy. if (StorageMetrics.totalHintsInProgress.getCount() > maxHintsInProgress - && (getHintsInProgressFor(destination).get() > 0 && shouldHint(destination))) + && (getHintsInProgressFor(destination.endpoint()).get() > 0 && shouldHint(destination))) { throw new OverloadedException("Too many in flight hints: " + StorageMetrics.totalHintsInProgress.getCount() + " destination: " + destination + - " destination hints: " + getHintsInProgressFor(destination).get()); + " destination hints: " + getHintsInProgressFor(destination.endpoint()).get()); } } private static void sendMessagesToNonlocalDC(MessageOut message, - Collection targets, + EndpointsForToken targets, AbstractWriteResponseHandler handler) { - Iterator iter = targets.iterator(); + Iterator iter = targets.iterator(); int[] messageIds = new int[targets.size()]; - InetAddressAndPort target = iter.next(); + Replica target = iter.next(); int idIdx = 0; // Add the other destinations of the same message as a FORWARD_HEADER entry while (iter.hasNext()) { - InetAddressAndPort destination = iter.next(); - int id = MessagingService.instance().addCallback(handler, - message, - destination, - message.getTimeout(), - handler.consistencyLevel, - true); + Replica destination = iter.next(); + int id = MessagingService.instance().addWriteCallback(handler, + message, + destination, + message.getTimeout(), + handler.replicaLayout.consistencyLevel(), + true); messageIds[idIdx++] = id; logger.trace("Adding FWD message to {}@{}", id, destination); } - message = message.withParameter(ParameterType.FORWARD_TO.FORWARD_TO, new ForwardToContainer(targets, messageIds)); + + message = message.withParameter(ParameterType.FORWARD_TO, new ForwardToContainer(targets.endpoints(), messageIds)); // send the combined message + forward headers - int id = MessagingService.instance().sendRR(message, target, handler, true); + int id = MessagingService.instance().sendWriteRR(message, target, handler, true); logger.trace("Sending message to {}@{}", id, target); } - private static void performLocally(Stage stage, final Runnable runnable) + private static void performLocally(Stage stage, Replica localReplica, final Runnable runnable) { - StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable() + StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable(localReplica) { public void runMayThrow() { @@ -1449,9 +1381,9 @@ public class StorageProxy implements StorageProxyMBean }); } - private static void performLocally(Stage stage, Optional mutation, final Runnable runnable, final IAsyncCallbackWithFailure handler) + private static void performLocally(Stage stage, Replica localReplica, Optional mutation, final Runnable runnable, final IAsyncCallbackWithFailure handler) { - StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable(mutation) + StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable(localReplica, mutation) { public void runMayThrow() { @@ -1492,9 +1424,9 @@ public class StorageProxy implements StorageProxyMBean */ public static AbstractWriteResponseHandler mutateCounter(CounterMutation cm, String localDataCenter, long queryStartNanoTime) throws UnavailableException, OverloadedException { - InetAddressAndPort endpoint = findSuitableEndpoint(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency()); + Replica replica = findSuitableReplica(cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency()); - if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort())) + if (replica.isLocal()) { return applyCounterMutationOnCoordinator(cm, localDataCenter, queryStartNanoTime); } @@ -1502,18 +1434,19 @@ public class StorageProxy implements StorageProxyMBean { // Exit now if we can't fulfill the CL here instead of forwarding to the leader replica String keyspaceName = cm.getKeyspaceName(); + Keyspace keyspace = Keyspace.open(keyspaceName); AbstractReplicationStrategy rs = Keyspace.open(keyspaceName).getReplicationStrategy(); Token tk = cm.key().getToken(); - List naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk); - Collection pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName); - rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, cm.consistency(), null, WriteType.COUNTER, queryStartNanoTime).assureSufficientLiveNodes(); + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWriteWithDownNodes(keyspace, cm.consistency(), tk); + rs.getWriteResponseHandler(replicaLayout, null, WriteType.COUNTER, queryStartNanoTime).assureSufficientLiveNodes(); // Forward the actual update to the chosen leader replica - AbstractWriteResponseHandler responseHandler = new WriteResponseHandler<>(endpoint, WriteType.COUNTER, queryStartNanoTime); + AbstractWriteResponseHandler responseHandler = new WriteResponseHandler<>(ReplicaLayout.forCounterWrite(keyspace, tk, replica), + WriteType.COUNTER, queryStartNanoTime); - Tracing.trace("Enqueuing counter update to {}", endpoint); - MessagingService.instance().sendRR(cm.makeMutationMessage(), endpoint, responseHandler, false); + Tracing.trace("Enqueuing counter update to {}", replica); + MessagingService.instance().sendWriteRR(cm.makeMutationMessage(), replica, responseHandler, false); return responseHandler; } } @@ -1528,38 +1461,37 @@ public class StorageProxy implements StorageProxyMBean * is unclear we want to mix those latencies with read latencies, so this * may be a bit involved. */ - private static InetAddressAndPort findSuitableEndpoint(String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException + private static Replica findSuitableReplica(String keyspaceName, DecoratedKey key, String localDataCenter, ConsistencyLevel cl) throws UnavailableException { Keyspace keyspace = Keyspace.open(keyspaceName); IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - List endpoints = new ArrayList<>(); - StorageService.instance.getLiveNaturalEndpoints(keyspace, key, endpoints); + EndpointsForToken replicas = StorageService.instance.getLiveNaturalReplicasForToken(keyspace, key); // CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping - endpoints.removeIf(endpoint -> !StorageService.instance.isRpcReady(endpoint)); + replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint())); // TODO have a way to compute the consistency level - if (endpoints.isEmpty()) - throw new UnavailableException(cl, cl.blockFor(keyspace), 0); + if (replicas.isEmpty()) + throw UnavailableException.create(cl, cl.blockFor(keyspace), 0); - List localEndpoints = new ArrayList<>(endpoints.size()); + List localReplicas = new ArrayList<>(replicas.size()); - for (InetAddressAndPort endpoint : endpoints) - if (snitch.getDatacenter(endpoint).equals(localDataCenter)) - localEndpoints.add(endpoint); + for (Replica replica : replicas) + if (snitch.getDatacenter(replica).equals(localDataCenter)) + localReplicas.add(replica); - if (localEndpoints.isEmpty()) + if (localReplicas.isEmpty()) { // If the consistency required is local then we should not involve other DCs if (cl.isDatacenterLocal()) - throw new UnavailableException(cl, cl.blockFor(keyspace), 0); + throw UnavailableException.create(cl, cl.blockFor(keyspace), 0); // No endpoint in local DC, pick the closest endpoint according to the snitch - snitch.sortByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints); - return endpoints.get(0); + replicas = snitch.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), replicas); + return replicas.get(0); } - return localEndpoints.get(ThreadLocalRandom.current().nextInt(localEndpoints.size())); + return localReplicas.get(ThreadLocalRandom.current().nextInt(localReplicas.size())); } // Must be called on a replica of the mutation. This replica becomes the @@ -1579,7 +1511,7 @@ public class StorageProxy implements StorageProxyMBean } private static Runnable counterWriteTask(final IMutation mutation, - final Iterable targets, + final EndpointsForToken targets, final AbstractWriteResponseHandler responseHandler, final String localDataCenter) { @@ -1592,11 +1524,7 @@ public class StorageProxy implements StorageProxyMBean Mutation result = ((CounterMutation) mutation).applyCounterMutation(); responseHandler.response(null); - - Set remotes = Sets.difference(ImmutableSet.copyOf(targets), - ImmutableSet.of(FBUtilities.getBroadcastAddressAndPort())); - if (!remotes.isEmpty()) - sendToHintedEndpoints(result, remotes, responseHandler, localDataCenter, Stage.COUNTER_MUTATION); + sendToHintedReplicas(result, targets, responseHandler, localDataCenter, Stage.COUNTER_MUTATION); } }; } @@ -1664,9 +1592,7 @@ public class StorageProxy implements StorageProxyMBean try { // make sure any in-progress paxos writes are done (i.e., committed to a majority of replicas), before performing a quorum read - PaxosParticipants p = getPaxosParticipants(metadata, key, consistencyLevel); - List liveEndpoints = p.liveEndpoints; - int requiredParticipants = p.participants; + ReplicaLayout.ForPaxos replicaLayout = ReplicaLayout.forPaxos(Keyspace.open(metadata.keyspace), key, consistencyLevel); // does the work of applying in-progress writes; throws UAE or timeout if it can't final ConsistencyLevel consistencyForCommitOrFetch = consistencyLevel == ConsistencyLevel.LOCAL_SERIAL @@ -1675,7 +1601,7 @@ public class StorageProxy implements StorageProxyMBean try { - final PaxosBallotAndContention pair = beginAndRepairPaxos(start, key, metadata, liveEndpoints, requiredParticipants, consistencyLevel, consistencyForCommitOrFetch, false, state); + final PaxosBallotAndContention pair = beginAndRepairPaxos(start, key, metadata, replicaLayout, consistencyLevel, consistencyForCommitOrFetch, false, state); if (pair.contentions > 0) casReadMetrics.contention.update(pair.contentions); } @@ -1924,28 +1850,19 @@ public class StorageProxy implements StorageProxyMBean } } - public static List getLiveSortedEndpoints(Keyspace keyspace, ByteBuffer key) + public static EndpointsForToken getLiveSortedReplicasForToken(Keyspace keyspace, RingPosition pos) { - return getLiveSortedEndpoints(keyspace, StorageService.instance.getTokenMetadata().decorateKey(key)); + return getLiveSortedReplicas(keyspace, pos).forToken(pos.getToken()); } - public static List getLiveSortedEndpoints(Keyspace keyspace, RingPosition pos) + public static EndpointsForRange getLiveSortedReplicas(Keyspace keyspace, RingPosition pos) { - List liveEndpoints = StorageService.instance.getLiveNaturalEndpoints(keyspace, pos); - DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getBroadcastAddressAndPort(), liveEndpoints); - return liveEndpoints; - } + EndpointsForRange liveReplicas = StorageService.instance.getLiveNaturalReplicas(keyspace, pos); + // Replica availability is considered by the query path + Preconditions.checkState(liveReplicas.isEmpty() || liveReplicas.stream().anyMatch(Replica::isFull), + "At least one full replica required for reads: " + liveReplicas); - private static List intersection(List l1, List l2) - { - // Note: we don't use Guava Sets.intersection() for 3 reasons: - // 1) retainAll would be inefficient if l1 and l2 are large but in practice both are the replicas for a range and - // so will be very small (< RF). In that case, retainAll is in fact more efficient. - // 2) we do ultimately need a list so converting everything to sets don't make sense - // 3) l1 and l2 are sorted by proximity. The use of retainAll maintain that sorting in the result, while using sets wouldn't. - List inter = new ArrayList<>(l1); - inter.retainAll(l2); - return inter; + return DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), liveReplicas); } /** @@ -1963,24 +1880,10 @@ public class StorageProxy implements StorageProxyMBean : index.getEstimatedResultRows(); // adjust maxExpectedResults by the number of tokens this node has and the replication factor for this ks - return (maxExpectedResults / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor(); + return (maxExpectedResults / DatabaseDescriptor.getNumTokens()) / keyspace.getReplicationStrategy().getReplicationFactor().allReplicas; } - private static class RangeForQuery - { - public final AbstractBounds range; - public final List liveEndpoints; - public final List filteredEndpoints; - - public RangeForQuery(AbstractBounds range, List liveEndpoints, List filteredEndpoints) - { - this.range = range; - this.liveEndpoints = liveEndpoints; - this.filteredEndpoints = filteredEndpoints; - } - } - - private static class RangeIterator extends AbstractIterator + private static class RangeIterator extends AbstractIterator { private final Keyspace keyspace; private final ConsistencyLevel consistency; @@ -2004,38 +1907,43 @@ public class StorageProxy implements StorageProxyMBean return rangeCount; } - protected RangeForQuery computeNext() + protected ReplicaLayout.ForRange computeNext() { if (!ranges.hasNext()) return endOfData(); AbstractBounds range = ranges.next(); - List liveEndpoints = getLiveSortedEndpoints(keyspace, range.right); - return new RangeForQuery(range, - liveEndpoints, - consistency.filterForQuery(keyspace, liveEndpoints)); + EndpointsForRange liveReplicas = getLiveSortedReplicas(keyspace, range.right); + + int blockFor = consistency.blockFor(keyspace); + EndpointsForRange targetReplicas = consistency.filterForQuery(keyspace, liveReplicas); + int minResponses = Math.min(targetReplicas.size(), blockFor); + + // Endpoitns for range here as well + return ReplicaLayout.forRangeRead(keyspace, consistency, range, + liveReplicas, targetReplicas.subList(0, minResponses)); } } - private static class RangeMerger extends AbstractIterator + private static class RangeMerger extends AbstractIterator { private final Keyspace keyspace; private final ConsistencyLevel consistency; - private final PeekingIterator ranges; + private final PeekingIterator ranges; - private RangeMerger(Iterator iterator, Keyspace keyspace, ConsistencyLevel consistency) + private RangeMerger(Iterator iterator, Keyspace keyspace, ConsistencyLevel consistency) { this.keyspace = keyspace; this.consistency = consistency; this.ranges = Iterators.peekingIterator(iterator); } - protected RangeForQuery computeNext() + protected ReplicaLayout.ForRange computeNext() { if (!ranges.hasNext()) return endOfData(); - RangeForQuery current = ranges.next(); + ReplicaLayout.ForRange current = ranges.next(); // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take // the replication factor into account. If the intersection of live endpoints for 2 consecutive ranges @@ -2050,22 +1958,22 @@ public class StorageProxy implements StorageProxyMBean if (current.range.right.isMinimum()) break; - RangeForQuery next = ranges.peek(); + ReplicaLayout.ForRange next = ranges.peek(); - List merged = intersection(current.liveEndpoints, next.liveEndpoints); + EndpointsForRange merged = current.all().keep(next.all().endpoints()); // Check if there is enough endpoint for the merge to be possible. - if (!consistency.isSufficientLiveNodes(keyspace, merged)) + if (!consistency.isSufficientLiveNodesForRead(keyspace, merged)) break; - List filteredMerged = consistency.filterForQuery(keyspace, merged); + EndpointsForRange filteredMerged = consistency.filterForQuery(keyspace, merged); // Estimate whether merging will be a win or not - if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, current.filteredEndpoints, next.filteredEndpoints)) + if (!DatabaseDescriptor.getEndpointSnitch().isWorthMergingForRangeQuery(filteredMerged, current.selected(), next.selected())) break; // If we get there, merge this range and the next one - current = new RangeForQuery(current.range.withNewRight(next.range.right), merged, filteredMerged); + current = ReplicaLayout.forRangeRead(keyspace, consistency, current.range.withNewRight(next.range.right), merged, filteredMerged); ranges.next(); // consume the range we just merged since we've only peeked so far } return current; @@ -2110,11 +2018,9 @@ public class StorageProxy implements StorageProxyMBean private static class RangeCommandIterator extends AbstractIterator implements PartitionIterator { - private final Iterator ranges; + private final Iterator ranges; private final int totalRangeCount; private final PartitionRangeReadCommand command; - private final Keyspace keyspace; - private final ConsistencyLevel consistency; private final boolean enforceStrictLiveness; private final long startTime; @@ -2135,8 +2041,6 @@ public class StorageProxy implements StorageProxyMBean this.startTime = System.nanoTime(); this.ranges = new RangeMerger(ranges, keyspace, consistency); this.totalRangeCount = ranges.rangeCount(); - this.consistency = consistency; - this.keyspace = keyspace; this.queryStartNanoTime = queryStartNanoTime; this.enforceStrictLiveness = command.metadata().enforceStrictLiveness(); } @@ -2204,36 +2108,36 @@ public class StorageProxy implements StorageProxyMBean /** * Queries the provided sub-range. * - * @param toQuery the subRange to query. + * @param replicaLayout the subRange to query. * @param isFirst in the case where multiple queries are sent in parallel, whether that's the first query on * that batch or not. The reason it matters is that whe paging queries, the command (more specifically the * {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in * that it's the query that "continues" whatever we're previously queried). */ - private SingleRangeResponse query(RangeForQuery toQuery, boolean isFirst) + private SingleRangeResponse query(ReplicaLayout.ForRange replicaLayout, boolean isFirst) { - PartitionRangeReadCommand rangeCommand = command.forSubRange(toQuery.range, isFirst); + PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaLayout.range, isFirst); + ReadRepair readRepair = ReadRepair.create(command, replicaLayout, queryStartNanoTime); + DataResolver resolver = new DataResolver<>(rangeCommand, replicaLayout, readRepair, queryStartNanoTime); + Keyspace keyspace = Keyspace.open(command.metadata().keyspace); - ReadRepair readRepair = ReadRepair.create(command, queryStartNanoTime, consistency); - DataResolver resolver = new DataResolver(keyspace, rangeCommand, consistency, toQuery.filteredEndpoints.size(), queryStartNanoTime, readRepair); - - int blockFor = consistency.blockFor(keyspace); - int minResponses = Math.min(toQuery.filteredEndpoints.size(), blockFor); - List minimalEndpoints = toQuery.filteredEndpoints.subList(0, minResponses); - ReadCallback handler = new ReadCallback(resolver, consistency, rangeCommand, minimalEndpoints, queryStartNanoTime); + ReadCallback handler = new ReadCallback<>(resolver, + replicaLayout.consistencyLevel().blockFor(keyspace), + rangeCommand, + replicaLayout, + queryStartNanoTime); handler.assureSufficientLiveNodes(); - - if (toQuery.filteredEndpoints.size() == 1 && canDoLocalRequest(toQuery.filteredEndpoints.get(0))) + if (replicaLayout.selected().size() == 1 && replicaLayout.selected().get(0).isLocal()) { StageManager.getStage(Stage.READ).execute(new LocalReadRunnable(rangeCommand, handler)); } else { - for (InetAddressAndPort endpoint : toQuery.filteredEndpoints) + for (Replica replica : replicaLayout.selected()) { - Tracing.trace("Enqueuing request to {}", endpoint); - MessagingService.instance().sendRRWithFailure(rangeCommand.createMessage(), endpoint, handler); + Tracing.trace("Enqueuing request to {}", replica); + MessagingService.instance().sendRRWithFailure(rangeCommand.createMessage(), replica.endpoint(), handler); } } @@ -2486,32 +2390,30 @@ public class StorageProxy implements StorageProxyMBean DatabaseDescriptor.setMaxHintWindow(ms); } - public static boolean shouldHint(InetAddressAndPort ep) + public static boolean shouldHint(Replica replica) { - if (DatabaseDescriptor.hintedHandoffEnabled()) - { - Set disabledDCs = DatabaseDescriptor.hintedHandoffDisabledDCs(); - if (!disabledDCs.isEmpty()) - { - final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(ep); - if (disabledDCs.contains(dc)) - { - Tracing.trace("Not hinting {} since its data center {} has been disabled {}", ep, dc, disabledDCs); - return false; - } - } - boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(ep) > DatabaseDescriptor.getMaxHintWindow(); - if (hintWindowExpired) - { - HintsService.instance.metrics.incrPastWindow(ep); - Tracing.trace("Not hinting {} which has been down {} ms", ep, Gossiper.instance.getEndpointDowntime(ep)); - } - return !hintWindowExpired; - } - else - { + if (!DatabaseDescriptor.hintedHandoffEnabled()) return false; + if (replica.isTransient() || replica.isLocal()) + return false; + + Set disabledDCs = DatabaseDescriptor.hintedHandoffDisabledDCs(); + if (!disabledDCs.isEmpty()) + { + final String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(replica); + if (disabledDCs.contains(dc)) + { + Tracing.trace("Not hinting {} since its data center {} has been disabled {}", replica, dc, disabledDCs); + return false; + } } + boolean hintWindowExpired = Gossiper.instance.getEndpointDowntime(replica.endpoint()) > DatabaseDescriptor.getMaxHintWindow(); + if (hintWindowExpired) + { + HintsService.instance.metrics.incrPastWindow(replica.endpoint()); + Tracing.trace("Not hinting {} which has been down {} ms", replica, Gossiper.instance.getEndpointDowntime(replica.endpoint())); + } + return !hintWindowExpired; } /** @@ -2532,7 +2434,7 @@ public class StorageProxy implements StorageProxyMBean // invoked by an admin, for simplicity we require that all nodes are up // to perform the operation. int liveMembers = Gossiper.instance.getLiveMembers().size(); - throw new UnavailableException(ConsistencyLevel.ALL, liveMembers + Gossiper.instance.getUnreachableMembers().size(), liveMembers); + throw UnavailableException.create(ConsistencyLevel.ALL, liveMembers + Gossiper.instance.getUnreachableMembers().size(), liveMembers); } Set allEndpoints = StorageService.instance.getLiveRingMembers(true); @@ -2571,10 +2473,9 @@ public class StorageProxy implements StorageProxyMBean public interface WritePerformer { public void apply(IMutation mutation, - Iterable targets, + ReplicaLayout.ForToken targets, AbstractWriteResponseHandler responseHandler, - String localDataCenter, - ConsistencyLevel consistencyLevel) throws OverloadedException; + String localDataCenter) throws OverloadedException; } /** @@ -2638,15 +2539,18 @@ public class StorageProxy implements StorageProxyMBean { private final long constructionTime = System.currentTimeMillis(); + private final Replica localReplica; private final Optional mutationOpt; - public LocalMutationRunnable(Optional mutationOpt) + public LocalMutationRunnable(Replica localReplica, Optional mutationOpt) { + this.localReplica = localReplica; this.mutationOpt = mutationOpt; } - public LocalMutationRunnable() + public LocalMutationRunnable(Replica localReplica) { + this.localReplica = localReplica; this.mutationOpt = Optional.empty(); } @@ -2659,7 +2563,8 @@ public class StorageProxy implements StorageProxyMBean { if (MessagingService.DROPPABLE_VERBS.contains(verb)) MessagingService.instance().incrementDroppedMutations(mutationOpt, timeTaken); - HintRunnable runnable = new HintRunnable(Collections.singleton(FBUtilities.getBroadcastAddressAndPort())) + + HintRunnable runnable = new HintRunnable(EndpointsForToken.of(localReplica.range().right, localReplica)) { protected void runMayThrow() throws Exception { @@ -2690,9 +2595,9 @@ public class StorageProxy implements StorageProxyMBean */ private abstract static class HintRunnable implements Runnable { - public final Collection targets; + public final EndpointsForToken targets; - protected HintRunnable(Collection targets) + protected HintRunnable(EndpointsForToken targets) { this.targets = targets; } @@ -2710,7 +2615,7 @@ public class StorageProxy implements StorageProxyMBean finally { StorageMetrics.totalHintsInProgress.dec(targets.size()); - for (InetAddressAndPort target : targets) + for (InetAddressAndPort target : targets.endpoints()) getHintsInProgressFor(target).decrementAndGet(); } } @@ -2756,22 +2661,23 @@ public class StorageProxy implements StorageProxyMBean } } - public static Future submitHint(Mutation mutation, InetAddressAndPort target, AbstractWriteResponseHandler responseHandler) + public static Future submitHint(Mutation mutation, Replica target, AbstractWriteResponseHandler responseHandler) { - return submitHint(mutation, Collections.singleton(target), responseHandler); + return submitHint(mutation, EndpointsForToken.of(target.range().right, target), responseHandler); } public static Future submitHint(Mutation mutation, - Collection targets, + EndpointsForToken targets, AbstractWriteResponseHandler responseHandler) { + Replicas.assertFull(targets); // hints should not be written for transient replicas HintRunnable runnable = new HintRunnable(targets) { public void runMayThrow() { Set validTargets = new HashSet<>(targets.size()); Set hostIds = new HashSet<>(targets.size()); - for (InetAddressAndPort target : targets) + for (InetAddressAndPort target : targets.endpoints()) { UUID hostId = StorageService.instance.getHostIdForEndpoint(target); if (hostId != null) @@ -2786,7 +2692,7 @@ public class StorageProxy implements StorageProxyMBean HintsService.instance.write(hostIds, Hint.create(mutation, System.currentTimeMillis())); validTargets.forEach(HintsService.instance.metrics::incrCreatedHints); // Notify the handler only for CL == ANY - if (responseHandler != null && responseHandler.consistencyLevel == ConsistencyLevel.ANY) + if (responseHandler != null && responseHandler.replicaLayout.consistencyLevel() == ConsistencyLevel.ANY) responseHandler.response(null); } }; @@ -2797,8 +2703,8 @@ public class StorageProxy implements StorageProxyMBean private static Future submitHint(HintRunnable runnable) { StorageMetrics.totalHintsInProgress.inc(runnable.targets.size()); - for (InetAddressAndPort target : runnable.targets) - getHintsInProgressFor(target).incrementAndGet(); + for (Replica target : runnable.targets) + getHintsInProgressFor(target.endpoint()).incrementAndGet(); return (Future) StageManager.getStage(Stage.MUTATION).submit(runnable); } @@ -2892,36 +2798,6 @@ public class StorageProxy implements StorageProxyMBean DatabaseDescriptor.setOtcBacklogExpirationInterval(intervalInMillis); } - - static class PaxosParticipants - { - final List liveEndpoints; - final int participants; - - PaxosParticipants(List liveEndpoints, int participants) - { - this.liveEndpoints = liveEndpoints; - this.participants = participants; - } - - @Override - public final int hashCode() - { - int hashCode = 31 + (liveEndpoints == null ? 0 : liveEndpoints.hashCode()); - return 31 * hashCode * this.participants; - } - - @Override - public final boolean equals(Object o) - { - if(!(o instanceof PaxosParticipants)) - return false; - PaxosParticipants that = (PaxosParticipants)o; - // handles nulls properly - return Objects.equals(liveEndpoints, that.liveEndpoints) && participants == that.participants; - } - } - static class PaxosBallotAndContention { final UUID ballot; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 9467c9aaba..7f4ae14918 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.MatchResult; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; import javax.annotation.Nullable; @@ -41,9 +42,12 @@ import javax.management.openmbean.TabularDataSupport; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.collect.*; import com.google.common.util.concurrent.*; +import org.apache.cassandra.dht.RangeStreamer.FetchReplica; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -110,6 +114,8 @@ import org.apache.cassandra.utils.progress.ProgressEventType; import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor; import org.apache.cassandra.utils.progress.jmx.JMXProgressSupport; +import static com.google.common.collect.Iterables.transform; +import static com.google.common.collect.Iterables.tryFind; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName; @@ -164,9 +170,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return isShutdown; } - public Collection> getLocalRanges(String keyspaceName) + public RangesAtEndpoint getLocalReplicas(String keyspaceName) { - return getRangesForEndpoint(keyspaceName, FBUtilities.getBroadcastAddressAndPort()); + return getReplicasForEndpoint(keyspaceName, FBUtilities.getBroadcastAddressAndPort()); } public List> getLocalAndPendingRanges(String ks) @@ -174,9 +180,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); Keyspace keyspace = Keyspace.open(ks); List> ranges = new ArrayList<>(); - ranges.addAll(keyspace.getReplicationStrategy().getAddressRanges().get(broadcastAddress)); - ranges.addAll(getTokenMetadata().getPendingRanges(ks, broadcastAddress)); - return Range.normalize(ranges); + for (Replica r : keyspace.getReplicationStrategy().getAddressReplicas(broadcastAddress)) + ranges.add(r.range()); + for (Replica r : getTokenMetadata().getPendingRanges(ks, broadcastAddress)) + ranges.add(r.range()); + return ranges; } public Collection> getPrimaryRanges(String keyspace) @@ -1225,11 +1233,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (keyspace == null) { for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) - streamer.addRanges(keyspaceName, getLocalRanges(keyspaceName)); + streamer.addRanges(keyspaceName, getLocalReplicas(keyspaceName)); } else if (tokens == null) { - streamer.addRanges(keyspace, getLocalRanges(keyspace)); + streamer.addRanges(keyspace, getLocalReplicas(keyspace)); } else { @@ -1251,14 +1259,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } // Ensure all specified ranges are actually ranges owned by this host - Collection> localRanges = getLocalRanges(keyspace); + RangesAtEndpoint localReplicas = getLocalReplicas(keyspace); + RangesAtEndpoint.Builder streamRanges = new RangesAtEndpoint.Builder(FBUtilities.getBroadcastAddressAndPort(), ranges.size()); for (Range specifiedRange : ranges) { boolean foundParentRange = false; - for (Range localRange : localRanges) + for (Replica localReplica : localReplicas) { - if (localRange.contains(specifiedRange)) + if (localReplica.contains(specifiedRange)) { + streamRanges.add(localReplica.decorateSubrange(specifiedRange)); foundParentRange = true; break; } @@ -1292,7 +1302,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE streamer.addSourceFilter(new RangeStreamer.WhitelistedSourcesFilter(sources)); } - streamer.addRanges(keyspace, ranges); + streamer.addRanges(keyspace, streamRanges.build()); } StreamResultFuture resultFuture = streamer.fetchAsync(); @@ -1700,9 +1710,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { /* All the ranges for the tokens */ Map, List> map = new HashMap<>(); - for (Map.Entry,List> entry : getRangeToAddressMap(keyspace).entrySet()) + for (Map.Entry, EndpointsForRange> entry : getRangeToAddressMap(keyspace).entrySet()) { - map.put(entry.getKey().asList(), stringify(entry.getValue(), withPort)); + map.put(entry.getKey().asList(), Replicas.stringify(entry.getValue(), withPort)); } return map; } @@ -1753,12 +1763,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { /* All the ranges for the tokens */ Map, List> map = new HashMap<>(); - for (Map.Entry, List> entry : getRangeToAddressMap(keyspace).entrySet()) + for (Map.Entry, EndpointsForRange> entry : getRangeToAddressMap(keyspace).entrySet()) { List rpcaddrs = new ArrayList<>(entry.getValue().size()); - for (InetAddressAndPort endpoint: entry.getValue()) + for (Replica replicas: entry.getValue()) { - rpcaddrs.add(getNativeaddress(endpoint, withPort)); + rpcaddrs.add(getNativeaddress(replicas.endpoint(), withPort)); } map.put(entry.getKey().asList(), rpcaddrs); } @@ -1783,38 +1793,31 @@ public class StorageService extends NotificationBroadcasterSupport implements IE keyspace = Schema.instance.getNonLocalStrategyKeyspaces().get(0); Map, List> map = new HashMap<>(); - for (Map.Entry, Collection> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet()) + for (Map.Entry, EndpointsForRange> entry : tokenMetadata.getPendingRangesMM(keyspace).asMap().entrySet()) { - List l = new ArrayList<>(entry.getValue()); - map.put(entry.getKey().asList(), stringify(l, withPort)); + map.put(entry.getKey().asList(), Replicas.stringify(entry.getValue(), withPort)); } return map; } - public Map, List> getRangeToAddressMap(String keyspace) + public EndpointsByRange getRangeToAddressMap(String keyspace) { return getRangeToAddressMap(keyspace, tokenMetadata.sortedTokens()); } - public Map, List> getRangeToAddressMapInLocalDC(String keyspace) + public EndpointsByRange getRangeToAddressMapInLocalDC(String keyspace) { - Predicate isLocalDC = new Predicate() - { - public boolean apply(InetAddressAndPort address) - { - return isLocalDC(address); - } - }; + Predicate isLocalDC = replica -> isLocalDC(replica.endpoint()); - Map, List> origMap = getRangeToAddressMap(keyspace, getTokensInLocalDC()); - Map, List> filteredMap = Maps.newHashMap(); - for (Map.Entry, List> entry : origMap.entrySet()) + EndpointsByRange origMap = getRangeToAddressMap(keyspace, getTokensInLocalDC()); + Map, EndpointsForRange> filteredMap = Maps.newHashMap(); + for (Map.Entry, EndpointsForRange> entry : origMap.entrySet()) { - List endpointsInLocalDC = Lists.newArrayList(Collections2.filter(entry.getValue(), isLocalDC)); + EndpointsForRange endpointsInLocalDC = entry.getValue().filter(isLocalDC); filteredMap.put(entry.getKey(), endpointsInLocalDC); } - return filteredMap; + return new EndpointsByRange(filteredMap); } private List getTokensInLocalDC() @@ -1836,7 +1839,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return remoteDC.equals(localDC); } - private Map, List> getRangeToAddressMap(String keyspace, List sortedTokens) + private EndpointsByRange getRangeToAddressMap(String keyspace, List sortedTokens) { // some people just want to get a visual representation of things. Allow null and set it to the first // non-system keyspace. @@ -1917,13 +1920,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE List ranges = new ArrayList<>(); Token.TokenFactory tf = getTokenFactory(); - Map, List> rangeToAddressMap = + EndpointsByRange rangeToAddressMap = includeOnlyLocalDC ? getRangeToAddressMapInLocalDC(keyspace) : getRangeToAddressMap(keyspace); - for (Map.Entry, List> entry : rangeToAddressMap.entrySet()) - ranges.add(TokenRange.create(tf, entry.getKey(), entry.getValue(), withPort)); + for (Map.Entry, EndpointsForRange> entry : rangeToAddressMap.entrySet()) + ranges.add(TokenRange.create(tf, entry.getKey(), ImmutableList.copyOf(entry.getValue().endpoints()), withPort)); return ranges; } @@ -2010,14 +2013,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * @param ranges * @return mapping of ranges to the replicas responsible for them. */ - private Map, List> constructRangeToEndpointMap(String keyspace, List> ranges) + private EndpointsByRange constructRangeToEndpointMap(String keyspace, List> ranges) { - Map, List> rangeToEndpointMap = new HashMap<>(ranges.size()); + Map, EndpointsForRange> rangeToEndpointMap = new HashMap<>(ranges.size()); for (Range range : ranges) { - rangeToEndpointMap.put(range, Keyspace.open(keyspace).getReplicationStrategy().getNaturalEndpoints(range.right)); + rangeToEndpointMap.put(range, Keyspace.open(keyspace).getReplicationStrategy().getNaturalReplicas(range.right)); } - return rangeToEndpointMap; + return new EndpointsByRange(rangeToEndpointMap); } public void beforeChange(InetAddressAndPort endpoint, EndpointState currentState, ApplicationState newStateKey, VersionedValue newValue) @@ -2735,32 +2738,56 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * Finds living endpoints responsible for the given ranges * * @param keyspaceName the keyspace ranges belong to - * @param ranges the ranges to find sources for + * @param leavingReplicas the ranges to find sources for * @return multimap of addresses to ranges the address is responsible for */ - private Multimap> getNewSourceRanges(String keyspaceName, Set> ranges) + private Multimap getNewSourceReplicas(String keyspaceName, Set leavingReplicas) { InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort(); - Multimap, InetAddressAndPort> rangeAddresses = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(tokenMetadata.cloneOnlyTokenMap()); - Multimap> sourceRanges = HashMultimap.create(); + EndpointsByRange rangeReplicas = Keyspace.open(keyspaceName).getReplicationStrategy().getRangeAddresses(tokenMetadata.cloneOnlyTokenMap()); + Multimap sourceRanges = HashMultimap.create(); IFailureDetector failureDetector = FailureDetector.instance; + logger.debug("Getting new source replicas for {}", leavingReplicas); + // find alive sources for our new ranges - for (Range range : ranges) + for (LeavingReplica leaver : leavingReplicas) { - Collection possibleRanges = rangeAddresses.get(range); + //We need this to find the replicas from before leaving to supply the data + Replica leavingReplica = leaver.leavingReplica; + //We need this to know what to fetch and what the transient status is + Replica ourReplica = leaver.ourReplica; + //If we are going to be a full replica only consider full replicas + Predicate replicaFilter = ourReplica.isFull() ? Replica::isFull : Predicates.alwaysTrue(); + Predicate notSelf = replica -> !replica.endpoint().equals(myAddress); + EndpointsForRange possibleReplicas = rangeReplicas.get(leavingReplica.range()); + logger.info("Possible replicas for newReplica {} are {}", ourReplica, possibleReplicas); IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - List sources = snitch.getSortedListByProximity(myAddress, possibleRanges); + EndpointsForRange sortedPossibleReplicas = snitch.sortedByProximity(myAddress, possibleReplicas); + logger.info("Sorted possible replicas starts as {}", sortedPossibleReplicas); + Optional myCurrentReplica = tryFind(possibleReplicas, replica -> replica.endpoint().equals(myAddress)).toJavaUtil(); - assert (!sources.contains(myAddress)); + boolean transientToFull = myCurrentReplica.isPresent() && myCurrentReplica.get().isTransient() && ourReplica.isFull(); + assert !sortedPossibleReplicas.endpoints().contains(myAddress) || transientToFull : String.format("My address %s, sortedPossibleReplicas %s, myCurrentReplica %s, myNewReplica %s", myAddress, sortedPossibleReplicas, myCurrentReplica, ourReplica); - for (InetAddressAndPort source : sources) + //Originally this didn't log if it couldn't restore replication and that seems wrong + boolean foundLiveReplica = false; + for (Replica possibleReplica : sortedPossibleReplicas.filter(Predicates.and(replicaFilter, notSelf))) { - if (failureDetector.isAlive(source)) + if (failureDetector.isAlive(possibleReplica.endpoint())) { - sourceRanges.put(source, range); + foundLiveReplica = true; + sourceRanges.put(possibleReplica.endpoint(), new FetchReplica(ourReplica, possibleReplica)); break; } + else + { + logger.debug("Skipping down replica {}", possibleReplica); + } + } + if (!foundLiveReplica) + { + logger.warn("Didn't find live replica to restore replication for " + ourReplica); } } return sourceRanges; @@ -2793,6 +2820,49 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } } + private static class LeavingReplica + { + //The node that is leaving + private final Replica leavingReplica; + + //Our range and transient status + private final Replica ourReplica; + + public LeavingReplica(Replica leavingReplica, Replica ourReplica) + { + Preconditions.checkNotNull(leavingReplica); + Preconditions.checkNotNull(ourReplica); + this.leavingReplica = leavingReplica; + this.ourReplica = ourReplica; + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + LeavingReplica that = (LeavingReplica) o; + + if (!leavingReplica.equals(that.leavingReplica)) return false; + return ourReplica.equals(that.ourReplica); + } + + public int hashCode() + { + int result = leavingReplica.hashCode(); + result = 31 * result + ourReplica.hashCode(); + return result; + } + + public String toString() + { + return "LeavingReplica{" + + "leavingReplica=" + leavingReplica + + ", ourReplica=" + ourReplica + + '}'; + } + } + /** * Called when an endpoint is removed from the ring. This function checks * whether this node becomes responsible for new ranges as a @@ -2805,38 +2875,52 @@ public class StorageService extends NotificationBroadcasterSupport implements IE */ private void restoreReplicaCount(InetAddressAndPort endpoint, final InetAddressAndPort notifyEndpoint) { - Multimap>>> rangesToFetch = HashMultimap.create(); + Map> replicasToFetch = new HashMap<>(); InetAddressAndPort myAddress = FBUtilities.getBroadcastAddressAndPort(); for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) { - Multimap, InetAddressAndPort> changedRanges = getChangedRangesForLeaving(keyspaceName, endpoint); - Set> myNewRanges = new HashSet<>(); - for (Map.Entry, InetAddressAndPort> entry : changedRanges.entries()) + logger.debug("Restoring replica count for keyspace {}", keyspaceName); + EndpointsByReplica changedReplicas = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy()); + Set myNewReplicas = new HashSet<>(); + for (Map.Entry entry : changedReplicas.flattenEntries()) { - if (entry.getValue().equals(myAddress)) - myNewRanges.add(entry.getKey()); - } - Multimap> sourceRanges = getNewSourceRanges(keyspaceName, myNewRanges); - for (Map.Entry>> entry : sourceRanges.asMap().entrySet()) - { - rangesToFetch.put(keyspaceName, entry); + Replica replica = entry.getValue(); + if (replica.endpoint().equals(myAddress)) + { + //Maybe we don't technically need to fetch transient data from somewhere + //but it's probably not a lot and it probably makes things a hair more resilient to people + //not running repair when they should. + myNewReplicas.add(new LeavingReplica(entry.getKey(), entry.getValue())); + } } + logger.debug("Changed replicas for leaving {}, myNewReplicas {}", changedReplicas, myNewReplicas); + replicasToFetch.put(keyspaceName, getNewSourceReplicas(keyspaceName, myNewReplicas)); } StreamPlan stream = new StreamPlan(StreamOperation.RESTORE_REPLICA_COUNT); - for (String keyspaceName : rangesToFetch.keySet()) - { - for (Map.Entry>> entry : rangesToFetch.get(keyspaceName)) - { - InetAddressAndPort source = entry.getKey(); - Collection> ranges = entry.getValue(); + replicasToFetch.forEach((keyspaceName, sources) -> { + logger.debug("Requesting keyspace {} sources", keyspaceName); + sources.asMap().forEach((sourceAddress, fetchReplicas) -> { + logger.debug("Source and our replicas are {}", fetchReplicas); + //Remember whether this node is providing the full or transient replicas for this range. We are going + //to pass streaming the local instance of Replica for the range which doesn't tell us anything about the source + //By encoding it as two separate sets we retain this information about the source. + RangesAtEndpoint full = fetchReplicas.stream() + .filter(f -> f.remote.isFull()) + .map(f -> f.local) + .collect(RangesAtEndpoint.collector(myAddress)); + RangesAtEndpoint transientReplicas = fetchReplicas.stream() + .filter(f -> f.remote.isTransient()) + .map(f -> f.local) + .collect(RangesAtEndpoint.collector(myAddress)); if (logger.isDebugEnabled()) - logger.debug("Requesting from {} ranges {}", source, StringUtils.join(ranges, ", ")); - stream.requestRanges(source, keyspaceName, ranges); - } - } + logger.debug("Requesting from {} full replicas {} transient replicas {}", sourceAddress, StringUtils.join(full, ", "), StringUtils.join(transientReplicas, ", ")); + + stream.requestRanges(sourceAddress, keyspaceName, full, transientReplicas); + }); + }); StreamResultFuture future = stream.execute(); Futures.addCallback(future, new FutureCallback() { @@ -2854,21 +2938,36 @@ public class StorageService extends NotificationBroadcasterSupport implements IE }); } + /** + * This is used in three contexts, graceful decomission, and restoreReplicaCount/removeNode. + * Graceful decomission should never lose data and it's going to be important that transient data + * is streamed to at least one other node from this one for each range. + * + * For ranges this node replicates its removal should cause a new replica to be selected either as transient or full + * for every range. So I believe the current code doesn't have to do anything special because it will engage in streaming + * for every range it replicates to at least one other node and that should propagate the transient data that was here. + * When I graphed this out on paper the result of removal looked correct and there are no issues such as + * this node needing to create a full replica for a range it transiently replicates because what is created is just another + * transient replica to replace this node. + * @param keyspaceName + * @param endpoint + * @return + */ // needs to be modified to accept either a keyspace or ARS. - private Multimap, InetAddressAndPort> getChangedRangesForLeaving(String keyspaceName, InetAddressAndPort endpoint) + static EndpointsByReplica getChangedReplicasForLeaving(String keyspaceName, InetAddressAndPort endpoint, TokenMetadata tokenMetadata, AbstractReplicationStrategy strat) { // First get all ranges the leaving endpoint is responsible for - Collection> ranges = getRangesForEndpoint(keyspaceName, endpoint); + RangesAtEndpoint replicas = strat.getAddressReplicas(endpoint); if (logger.isDebugEnabled()) - logger.debug("Node {} ranges [{}]", endpoint, StringUtils.join(ranges, ", ")); + logger.debug("Node {} replicas [{}]", endpoint, StringUtils.join(replicas, ", ")); - Map, List> currentReplicaEndpoints = new HashMap<>(ranges.size()); + Map currentReplicaEndpoints = Maps.newHashMapWithExpectedSize(replicas.size()); // Find (for each range) all nodes that store replicas for these ranges as well TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap(); // don't do this in the loop! #7758 - for (Range range : ranges) - currentReplicaEndpoints.put(range, Keyspace.open(keyspaceName).getReplicationStrategy().calculateNaturalEndpoints(range.right, metadata)); + for (Replica replica : replicas) + currentReplicaEndpoints.put(replica, strat.calculateNaturalReplicas(replica.range().right, metadata)); TokenMetadata temp = tokenMetadata.cloneAfterAllLeft(); @@ -2877,26 +2976,43 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (temp.isMember(endpoint)) temp.removeEndpoint(endpoint); - Multimap, InetAddressAndPort> changedRanges = HashMultimap.create(); + EndpointsByReplica.Mutable changedRanges = new EndpointsByReplica.Mutable(); // Go through the ranges and for each range check who will be // storing replicas for these ranges when the leaving endpoint // is gone. Whoever is present in newReplicaEndpoints list, but // not in the currentReplicaEndpoints list, will be needing the // range. - for (Range range : ranges) + for (Replica replica : replicas) { - Collection newReplicaEndpoints = Keyspace.open(keyspaceName).getReplicationStrategy().calculateNaturalEndpoints(range.right, temp); - newReplicaEndpoints.removeAll(currentReplicaEndpoints.get(range)); + EndpointsForRange newReplicaEndpoints = strat.calculateNaturalReplicas(replica.range().right, temp); + newReplicaEndpoints = newReplicaEndpoints.filter(newReplica -> { + Optional currentReplicaOptional = + tryFind(currentReplicaEndpoints.get(replica), + currentReplica -> newReplica.endpoint().equals(currentReplica.endpoint()) + ).toJavaUtil(); + //If it is newly replicating then yes we must do something to get the data there + if (!currentReplicaOptional.isPresent()) + return true; + + Replica currentReplica = currentReplicaOptional.get(); + //This transition requires streaming to occur + //Full -> transient is handled by nodetool cleanup + //transient -> transient and full -> full don't require any action + if (currentReplica.isTransient() && newReplica.isFull()) + return true; + return false; + }); + if (logger.isDebugEnabled()) if (newReplicaEndpoints.isEmpty()) - logger.debug("Range {} already in all replicas", range); + logger.debug("Replica {} already in all replicas", replica); else - logger.debug("Range {} will be responsibility of {}", range, StringUtils.join(newReplicaEndpoints, ", ")); - changedRanges.putAll(range, newReplicaEndpoints); + logger.debug("Replica {} will be responsibility of {}", replica, StringUtils.join(newReplicaEndpoints, ", ")); + changedRanges.putAll(replica, newReplicaEndpoints, Conflict.NONE); } - return changedRanges; + return changedRanges.asImmutableView(); } public void onJoin(InetAddressAndPort endpoint, EndpointState epState) @@ -3602,10 +3718,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } else { - option.getRanges().addAll(getLocalRanges(keyspace)); + Iterables.addAll(option.getRanges(), getLocalReplicas(keyspace).filter(Replica::isFull).ranges()); } } - if (option.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor() < 2) + if (option.getRanges().isEmpty() || Keyspace.open(keyspace).getReplicationStrategy().getReplicationFactor().allReplicas < 2) return 0; int cmd = nextRepairCommand.incrementAndGet(); @@ -3703,7 +3819,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * Get the "primary ranges" for the specified keyspace and endpoint. * "Primary ranges" are the ranges that the node is responsible for storing replica primarily. * The node that stores replica primarily is defined as the first node returned - * by {@link AbstractReplicationStrategy#calculateNaturalEndpoints}. + * by {@link AbstractReplicationStrategy#calculateNaturalReplicas}. * * @param keyspace Keyspace name to check primary ranges * @param ep endpoint we are interested in. @@ -3716,9 +3832,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE TokenMetadata metadata = tokenMetadata.cloneOnlyTokenMap(); for (Token token : metadata.sortedTokens()) { - List endpoints = strategy.calculateNaturalEndpoints(token, metadata); - if (endpoints.size() > 0 && endpoints.get(0).equals(ep)) + EndpointsForRange replicas = strategy.calculateNaturalReplicas(token, metadata); + if (replicas.size() > 0 && replicas.get(0).endpoint().equals(ep)) + { + Preconditions.checkState(replicas.get(0).isFull()); primaryRanges.add(new Range<>(metadata.getPredecessor(token), token)); + } } return primaryRanges; } @@ -3741,12 +3860,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE Collection> localDCPrimaryRanges = new HashSet<>(); for (Token token : metadata.sortedTokens()) { - List endpoints = strategy.calculateNaturalEndpoints(token, metadata); - for (InetAddressAndPort endpoint : endpoints) + EndpointsForRange replicas = strategy.calculateNaturalReplicas(token, metadata); + for (Replica replica : replicas) { - if (localDcNodes.contains(endpoint)) + if (localDcNodes.contains(replica.endpoint())) { - if (endpoint.equals(referenceEndpoint)) + if (replica.endpoint().equals(referenceEndpoint)) { localDCPrimaryRanges.add(new Range<>(metadata.getPredecessor(token), token)); } @@ -3763,9 +3882,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * @param ep endpoint we are interested in. * @return ranges for the specified endpoint. */ - Collection> getRangesForEndpoint(String keyspaceName, InetAddressAndPort ep) + RangesAtEndpoint getReplicasForEndpoint(String keyspaceName, InetAddressAndPort ep) { - return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressRanges().get(ep); + return Keyspace.open(keyspaceName).getReplicationStrategy().getAddressReplicas(ep); } /** @@ -3806,40 +3925,53 @@ public class StorageService extends NotificationBroadcasterSupport implements IE @Deprecated public List getNaturalEndpoints(String keyspaceName, String cf, String key) { - KeyspaceMetadata ksMetaData = Schema.instance.getKeyspaceMetadata(keyspaceName); - if (ksMetaData == null) - throw new IllegalArgumentException("Unknown keyspace '" + keyspaceName + "'"); - - TableMetadata metadata = ksMetaData.getTableOrViewNullable(cf); - if (metadata == null) - throw new IllegalArgumentException("Unknown table '" + cf + "' in keyspace '" + keyspaceName + "'"); - - return getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(metadata.partitionKeyType.fromString(key))).stream().map(i -> i.address).collect(toList()); + EndpointsForToken replicas = getNaturalReplicasForToken(keyspaceName, cf, key); + List inetList = new ArrayList<>(replicas.size()); + replicas.forEach(r -> inetList.add(r.endpoint().address)); + return inetList; } public List getNaturalEndpointsWithPort(String keyspaceName, String cf, String key) { - KeyspaceMetadata ksMetaData = Schema.instance.getKeyspaceMetadata(keyspaceName); - if (ksMetaData == null) - throw new IllegalArgumentException("Unknown keyspace '" + keyspaceName + "'"); - - TableMetadata metadata = ksMetaData.getTableOrViewNullable(cf); - if (metadata == null) - throw new IllegalArgumentException("Unknown table '" + cf + "' in keyspace '" + keyspaceName + "'"); - - return stringify(getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(metadata.partitionKeyType.fromString(key))), true); + return Replicas.stringify(getNaturalReplicasForToken(keyspaceName, cf, key), true); } @Deprecated public List getNaturalEndpoints(String keyspaceName, ByteBuffer key) { - return getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(key)).stream().map(i -> i.address).collect(toList()); + EndpointsForToken replicas = getNaturalReplicasForToken(keyspaceName, tokenMetadata.partitioner.getToken(key)); + List inetList = new ArrayList<>(replicas.size()); + replicas.forEach(r -> inetList.add(r.endpoint().address)); + return inetList; } public List getNaturalEndpointsWithPort(String keyspaceName, ByteBuffer key) { - return stringify(getNaturalEndpoints(keyspaceName, tokenMetadata.partitioner.getToken(key)), true); + return Replicas.stringify(getNaturalReplicasForToken(keyspaceName, tokenMetadata.partitioner.getToken(key)), true); + } + + public List getReplicas(String keyspaceName, String cf, String key) + { + List res = new ArrayList<>(); + for (Replica replica : getNaturalReplicasForToken(keyspaceName, cf, key)) + { + res.add(replica.toString()); + } + return res; + } + + public EndpointsForToken getNaturalReplicasForToken(String keyspaceName, String cf, String key) + { + KeyspaceMetadata ksMetaData = Schema.instance.getKeyspaceMetadata(keyspaceName); + if (ksMetaData == null) + throw new IllegalArgumentException("Unknown keyspace '" + keyspaceName + "'"); + + TableMetadata metadata = ksMetaData.getTableOrViewNullable(cf); + if (metadata == null) + throw new IllegalArgumentException("Unknown table '" + cf + "' in keyspace '" + keyspaceName + "'"); + + return getNaturalReplicasForToken(keyspaceName, tokenMetadata.partitioner.getToken(metadata.partitionKeyType.fromString(key))); } /** @@ -3850,37 +3982,25 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * @param pos position for which we need to find the endpoint * @return the endpoint responsible for this token */ - public List getNaturalEndpoints(String keyspaceName, RingPosition pos) + public static EndpointsForToken getNaturalReplicasForToken(String keyspaceName, RingPosition pos) { - return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalEndpoints(pos); + return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalReplicasForToken(pos); } /** * Returns the endpoints currently responsible for storing the token plus pending ones */ - public Iterable getNaturalAndPendingEndpoints(String keyspaceName, Token token) + public EndpointsForToken getNaturalAndPendingReplicasForToken(String keyspaceName, Token token) { - return Iterables.concat(getNaturalEndpoints(keyspaceName, token), tokenMetadata.pendingEndpointsFor(token, keyspaceName)); - } - - /** - * This method attempts to return N endpoints that are responsible for storing the - * specified key i.e for replication. - * - * @param keyspace keyspace name also known as keyspace - * @param key key for which we need to find the endpoint - * @return the endpoint responsible for this key - */ - public List getLiveNaturalEndpoints(Keyspace keyspace, ByteBuffer key) - { - return getLiveNaturalEndpoints(keyspace, tokenMetadata.decorateKey(key)); - } - - public List getLiveNaturalEndpoints(Keyspace keyspace, RingPosition pos) - { - List liveEps = new ArrayList<>(); - getLiveNaturalEndpoints(keyspace, pos, liveEps); - return liveEps; + // TODO: race condition to fetch these. impliciations?? + EndpointsForToken natural = getNaturalReplicasForToken(keyspaceName, token); + EndpointsForToken pending = tokenMetadata.pendingEndpointsForToken(token, keyspaceName); + if (Endpoints.haveConflicts(natural, pending)) + { + natural = Endpoints.resolveConflictsInNatural(natural, pending); + pending = Endpoints.resolveConflictsInPending(natural, pending); + } + return Endpoints.concat(natural, pending); } /** @@ -3889,17 +4009,23 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * * @param keyspace keyspace name also known as keyspace * @param pos position for which we need to find the endpoint - * @param liveEps the list of endpoints to mutate */ - public void getLiveNaturalEndpoints(Keyspace keyspace, RingPosition pos, List liveEps) + public EndpointsForToken getLiveNaturalReplicasForToken(Keyspace keyspace, RingPosition pos) { - List endpoints = keyspace.getReplicationStrategy().getNaturalEndpoints(pos); + return getLiveNaturalReplicas(keyspace, pos).forToken(pos.getToken()); + } - for (InetAddressAndPort endpoint : endpoints) - { - if (FailureDetector.instance.isAlive(endpoint)) - liveEps.add(endpoint); - } + /** + * This method attempts to return N endpoints that are responsible for storing the + * specified key i.e for replication. + * + * @param keyspace keyspace name also known as keyspace + * @param pos position for which we need to find the endpoint + */ + public EndpointsForRange getLiveNaturalReplicas(Keyspace keyspace, RingPosition pos) + { + EndpointsForRange replicas = keyspace.getReplicationStrategy().getNaturalReplicas(pos); + return replicas.filter(r -> FailureDetector.instance.isAlive(r.endpoint())); } public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception @@ -4019,13 +4145,13 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy) { NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) keyspace.getReplicationStrategy(); - rf = strategy.getReplicationFactor(dc); + rf = strategy.getReplicationFactor(dc).allReplicas; numNodes = metadata.getTopology().getDatacenterEndpoints().get(dc).size(); } else { numNodes = metadata.getAllEndpoints().size(); - rf = keyspace.getReplicationStrategy().getReplicationFactor(); + rf = keyspace.getReplicationStrategy().getReplicationFactor().allReplicas; } if (numNodes <= rf) @@ -4033,6 +4159,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE + keyspaceName + " (RF = " + rf + ", N = " + numNodes + ")." + " Perform a forceful decommission to ignore."); } + // TODO: do we care about fixing transient/full self-movements here? probably if (tokenMetadata.getPendingRanges(keyspaceName, FBUtilities.getBroadcastAddressAndPort()).size() > 0) throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring"); } @@ -4095,11 +4222,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private void unbootstrap(Runnable onFinish) throws ExecutionException, InterruptedException { - Map, InetAddressAndPort>> rangesToStream = new HashMap<>(); + Map rangesToStream = new HashMap<>(); for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) { - Multimap, InetAddressAndPort> rangesMM = getChangedRangesForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort()); + EndpointsByReplica rangesMM = getChangedReplicasForLeaving(keyspaceName, FBUtilities.getBroadcastAddressAndPort(), tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy()); if (logger.isDebugEnabled()) logger.debug("Ranges needing transfer are [{}]", StringUtils.join(rangesMM.keySet(), ",")); @@ -4135,20 +4262,22 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return HintsService.instance.transferHints(this::getPreferredHintsStreamTarget); } + private static EndpointsForRange getStreamCandidates(Collection endpoints) + { + endpoints = endpoints.stream() + .filter(endpoint -> FailureDetector.instance.isAlive(endpoint) && !FBUtilities.getBroadcastAddressAndPort().equals(endpoint)) + .collect(Collectors.toList()); + + return EndpointsForRange.copyOf(SystemReplicas.getSystemReplicas(endpoints)); + } /** * Find the best target to stream hints to. Currently the closest peer according to the snitch */ private UUID getPreferredHintsStreamTarget() { - List candidates = new ArrayList<>(StorageService.instance.getTokenMetadata().cloneAfterAllLeft().getAllEndpoints()); - candidates.remove(FBUtilities.getBroadcastAddressAndPort()); - for (Iterator iter = candidates.iterator(); iter.hasNext(); ) - { - InetAddressAndPort address = iter.next(); - if (!FailureDetector.instance.isAlive(address)) - iter.remove(); - } + Set endpoints = StorageService.instance.getTokenMetadata().cloneAfterAllLeft().getAllEndpoints(); + EndpointsForRange candidates = getStreamCandidates(endpoints); if (candidates.isEmpty()) { logger.warn("Unable to stream hints since no live endpoints seen"); @@ -4157,8 +4286,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE else { // stream to the closest peer as chosen by the snitch - DatabaseDescriptor.getEndpointSnitch().sortByProximity(FBUtilities.getBroadcastAddressAndPort(), candidates); - InetAddressAndPort hintsDestinationHost = candidates.get(0); + candidates = DatabaseDescriptor.getEndpointSnitch().sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), candidates); + InetAddressAndPort hintsDestinationHost = candidates.get(0).endpoint(); return tokenMetadata.getHostId(hintsDestinationHost); } } @@ -4207,6 +4336,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // checking if data is moving to this node for (String keyspaceName : keyspacesToProcess) { + // TODO: do we care about fixing transient/full self-movements here? if (tokenMetadata.getPendingRanges(keyspaceName, localAddress).size() > 0) throw new UnsupportedOperationException("data is currently moving to this node; unable to leave the ring"); } @@ -4218,7 +4348,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE setMode(Mode.MOVING, String.format("Sleeping %s ms before start streaming/fetching ranges", RING_DELAY), true); Uninterruptibles.sleepUninterruptibly(RING_DELAY, TimeUnit.MILLISECONDS); - RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess); + RangeRelocator relocator = new RangeRelocator(Collections.singleton(newToken), keyspacesToProcess, tokenMetadata); + relocator.calculateToFromStreams(); if (relocator.streamsNeeded()) { @@ -4243,131 +4374,191 @@ public class StorageService extends NotificationBroadcasterSupport implements IE logger.debug("Successfully moved to new token {}", getLocalTokens().iterator().next()); } - private class RangeRelocator + @VisibleForTesting + public static class RangeRelocator { private final StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION); + private final InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort(); + private final TokenMetadata tokenMetaCloneAllSettled; + // clone to avoid concurrent modification in calculateNaturalReplicas + private final TokenMetadata tokenMetaClone; + private final Collection tokens; + private final List keyspaceNames; - private RangeRelocator(Collection tokens, List keyspaceNames) + + private RangeRelocator(Collection tokens, List keyspaceNames, TokenMetadata tmd) { - calculateToFromStreams(tokens, keyspaceNames); + this.tokens = tokens; + this.keyspaceNames = keyspaceNames; + this.tokenMetaCloneAllSettled = tmd.cloneAfterAllSettled(); + // clone to avoid concurrent modification in calculateNaturalReplicas + this.tokenMetaClone = tmd.cloneOnlyTokenMap(); } - private void calculateToFromStreams(Collection newTokens, List keyspaceNames) + @VisibleForTesting + public RangeRelocator() { - InetAddressAndPort localAddress = FBUtilities.getBroadcastAddressAndPort(); - IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch(); - TokenMetadata tokenMetaCloneAllSettled = tokenMetadata.cloneAfterAllSettled(); - // clone to avoid concurrent modification in calculateNaturalEndpoints - TokenMetadata tokenMetaClone = tokenMetadata.cloneOnlyTokenMap(); + this.tokens = null; + this.keyspaceNames = null; + this.tokenMetaCloneAllSettled = null; + this.tokenMetaClone = null; + } + /** + * Wrapper that supplies accessors to the real implementations of the various dependencies for this method + */ + private Multimap calculateRangesToFetchWithPreferredEndpoints(AbstractReplicationStrategy strategy, RangesAtEndpoint fetchRanges, String keyspace) + { + EndpointsByReplica preferredEndpoints = + RangeStreamer.calculateRangesToFetchWithPreferredEndpoints((address, replicas) -> DatabaseDescriptor.getEndpointSnitch().sortedByProximity(address, replicas), + strategy, + fetchRanges, + useStrictConsistency, + tokenMetaClone, + tokenMetaCloneAllSettled, + RangeStreamer.ALIVE_PREDICATE, + keyspace, + Collections.emptyList()); + return RangeStreamer.convertPreferredEndpointsToWorkMap(preferredEndpoints); + } + + /** + * calculating endpoints to stream current ranges to if needed + * in some situations node will handle current ranges as part of the new ranges + **/ + public RangesByEndpoint calculateRangesToStreamWithEndpoints(RangesAtEndpoint streamRanges, + AbstractReplicationStrategy strat, + TokenMetadata tmdBefore, + TokenMetadata tmdAfter) + { + RangesByEndpoint.Mutable endpointRanges = new RangesByEndpoint.Mutable(); + for (Replica toStream : streamRanges) + { + //If the range we are sending is full only send it to the new full replica + //There will also be a new transient replica we need to send the data to, but not + //the repaired data + EndpointsForRange currentEndpoints = strat.calculateNaturalReplicas(toStream.range().right, tmdBefore); + EndpointsForRange newEndpoints = strat.calculateNaturalReplicas(toStream.range().right, tmdAfter); + logger.debug("Need to stream {}, current endpoints {}, new endpoints {}", toStream, currentEndpoints, newEndpoints); + + for (Replica current : currentEndpoints) + { + for (Replica updated : newEndpoints) + { + if (current.endpoint().equals(updated.endpoint())) + { + //Nothing to do + if (current.equals(updated)) + break; + + //In these two (really three) cases the existing data is sufficient and we should subtract whatever is already replicated + if (current.isFull() == updated.isFull() || current.isFull()) + { + //First subtract what we already have + Set> subsToStream = toStream.range().subtract(current.range()); + //Now we only stream what is still replicated + subsToStream = subsToStream.stream().flatMap(range -> range.intersectionWith(updated.range()).stream()).collect(Collectors.toSet()); + for (Range subrange : subsToStream) + { + //Only stream what intersects with what is in the new world + Set> intersections = subrange.intersectionWith(updated.range()); + for (Range intersection : intersections) + { + endpointRanges.put(updated.endpoint(), updated.decorateSubrange(intersection)); + } + } + } + else + { + for (Range intersection : toStream.range().intersectionWith(updated.range())) + { + endpointRanges.put(updated.endpoint(), updated.decorateSubrange(intersection)); + } + } + } + } + } + + for (Replica updated : newEndpoints) + { + if (!currentEndpoints.byEndpoint().containsKey(updated.endpoint())) + { + // Completely new range for this endpoint + if (toStream.isTransient() && updated.isFull()) + { + throw new AssertionError(String.format("Need to stream %s, but only have %s which is transient and not full", updated, toStream)); + } + for (Range intersection : updated.range().intersectionWith(toStream.range())) + { + endpointRanges.put(updated.endpoint(), updated.decorateSubrange(intersection)); + } + } + } + } + return endpointRanges.asImmutableView(); + } + + private void calculateToFromStreams() + { + logger.debug("Current tmd " + tokenMetaClone); + logger.debug("Updated tmd " + tokenMetaCloneAllSettled); for (String keyspace : keyspaceNames) { // replication strategy of the current keyspace AbstractReplicationStrategy strategy = Keyspace.open(keyspace).getReplicationStrategy(); - Multimap> endpointToRanges = strategy.getAddressRanges(); + // getting collection of the currently used ranges by this keyspace + RangesAtEndpoint currentReplicas = strategy.getAddressReplicas(localAddress); - logger.debug("Calculating ranges to stream and request for keyspace {}", keyspace); - for (Token newToken : newTokens) + logger.info("Calculating ranges to stream and request for keyspace {}", keyspace); + //From what I have seen we only ever call this with a single token from StorageService.move(Token) + for (Token newToken : tokens) { - // getting collection of the currently used ranges by this keyspace - Collection> currentRanges = endpointToRanges.get(localAddress); - // collection of ranges which this node will serve after move to the new token - Collection> updatedRanges = strategy.getPendingAddressRanges(tokenMetaClone, newToken, localAddress); + Collection currentTokens = tokenMetaClone.getTokens(localAddress); + if (currentTokens.size() > 1 || currentTokens.isEmpty()) + { + throw new AssertionError("Unexpected current tokens: " + currentTokens); + } - // ring ranges and endpoints associated with them - // this used to determine what nodes should we ping about range data - Multimap, InetAddressAndPort> rangeAddresses = strategy.getRangeAddresses(tokenMetaClone); + // collection of ranges which this node will serve after move to the new token + RangesAtEndpoint updatedReplicas = strategy.getPendingAddressRanges(tokenMetaClone, newToken, localAddress); // calculated parts of the ranges to request/stream from/to nodes in the ring - Pair>, Set>> rangesPerKeyspace = calculateStreamAndFetchRanges(currentRanges, updatedRanges); - - /** - * In this loop we are going through all ranges "to fetch" and determining - * nodes in the ring responsible for data we are interested in - */ - Multimap, InetAddressAndPort> rangesToFetchWithPreferredEndpoints = ArrayListMultimap.create(); - for (Range toFetch : rangesPerKeyspace.right) + Pair streamAndFetchOwnRanges = Pair.create(RangesAtEndpoint.empty(localAddress), RangesAtEndpoint.empty(localAddress)); + //In the single node token move there is nothing to do and Range subtraction is broken + //so it's easier to just identify this case up front. + if (tokenMetaClone.getTopology().getDatacenterEndpoints().get(DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddressAndPort() +)).size() > 1) { - for (Range range : rangeAddresses.keySet()) - { - if (range.contains(toFetch)) - { - List endpoints = null; - - if (useStrictConsistency) - { - Set oldEndpoints = Sets.newHashSet(rangeAddresses.get(range)); - Set newEndpoints = Sets.newHashSet(strategy.calculateNaturalEndpoints(toFetch.right, tokenMetaCloneAllSettled)); - - //Due to CASSANDRA-5953 we can have a higher RF then we have endpoints. - //So we need to be careful to only be strict when endpoints == RF - if (oldEndpoints.size() == strategy.getReplicationFactor()) - { - oldEndpoints.removeAll(newEndpoints); - - //No relocation required - if (oldEndpoints.isEmpty()) - continue; - - assert oldEndpoints.size() == 1 : "Expected 1 endpoint but found " + oldEndpoints.size(); - } - - endpoints = Lists.newArrayList(oldEndpoints.iterator().next()); - } - else - { - endpoints = snitch.getSortedListByProximity(localAddress, rangeAddresses.get(range)); - } - - // storing range and preferred endpoint set - rangesToFetchWithPreferredEndpoints.putAll(toFetch, endpoints); - } - } - - Collection addressList = rangesToFetchWithPreferredEndpoints.get(toFetch); - if (addressList == null || addressList.isEmpty()) - continue; - - if (useStrictConsistency) - { - if (addressList.size() > 1) - throw new IllegalStateException("Multiple strict sources found for " + toFetch); - - InetAddressAndPort sourceIp = addressList.iterator().next(); - if (Gossiper.instance.isEnabled() && !Gossiper.instance.getEndpointStateForEndpoint(sourceIp).isAlive()) - throw new RuntimeException("A node required to move the data consistently is down ("+sourceIp+"). If you wish to move the data from a potentially inconsistent replica, restart the node with -Dcassandra.consistent.rangemovement=false"); - } + streamAndFetchOwnRanges = calculateStreamAndFetchRanges(currentReplicas, updatedReplicas); } - // calculating endpoints to stream current ranges to if needed - // in some situations node will handle current ranges as part of the new ranges - Multimap> endpointRanges = HashMultimap.create(); - for (Range toStream : rangesPerKeyspace.left) - { - Set currentEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(toStream.right, tokenMetaClone)); - Set newEndpoints = ImmutableSet.copyOf(strategy.calculateNaturalEndpoints(toStream.right, tokenMetaCloneAllSettled)); - logger.debug("Range: {} Current endpoints: {} New endpoints: {}", toStream, currentEndpoints, newEndpoints); - for (InetAddressAndPort address : Sets.difference(newEndpoints, currentEndpoints)) - { - logger.debug("Range {} has new owner {}", toStream, address); - endpointRanges.put(address, toStream); - } - } + Multimap workMap = calculateRangesToFetchWithPreferredEndpoints(strategy, streamAndFetchOwnRanges.right, keyspace); + + RangesByEndpoint endpointRanges = calculateRangesToStreamWithEndpoints(streamAndFetchOwnRanges.left, strategy, tokenMetaClone, tokenMetaCloneAllSettled); + + logger.info("Endpoint ranges to stream to " + endpointRanges); // stream ranges for (InetAddressAndPort address : endpointRanges.keySet()) { logger.debug("Will stream range {} of keyspace {} to endpoint {}", endpointRanges.get(address), keyspace, address); - streamPlan.transferRanges(address, keyspace, endpointRanges.get(address)); + RangesAtEndpoint ranges = endpointRanges.get(address); + streamPlan.transferRanges(address, keyspace, ranges); } // stream requests - Multimap> workMap = RangeStreamer.getWorkMap(rangesToFetchWithPreferredEndpoints, keyspace, FailureDetector.instance, useStrictConsistency); - for (InetAddressAndPort address : workMap.keySet()) - { + workMap.asMap().forEach((address, sourceAndOurReplicas) -> { + RangesAtEndpoint full = sourceAndOurReplicas.stream() + .filter(pair -> pair.remote.isFull()) + .map(pair -> pair.local) + .collect(RangesAtEndpoint.collector(localAddress)); + RangesAtEndpoint transientReplicas = sourceAndOurReplicas.stream() + .filter(pair -> pair.remote.isTransient()) + .map(pair -> pair.local) + .collect(RangesAtEndpoint.collector(localAddress)); logger.debug("Will request range {} of keyspace {} from endpoint {}", workMap.get(address), keyspace, address); - streamPlan.requestRanges(address, keyspace, workMap.get(address)); - } + streamPlan.requestRanges(address, keyspace, full, transientReplicas); + }); logger.debug("Keyspace {}: work map {}.", keyspace, workMap); } @@ -4486,14 +4677,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) { // if the replication factor is 1 the data is lost so we shouldn't wait for confirmation - if (Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor() == 1) + if (Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor().allReplicas == 1) continue; // get all ranges that change ownership (that is, a node needs // to take responsibility for new range) - Multimap, InetAddressAndPort> changedRanges = getChangedRangesForLeaving(keyspaceName, endpoint); + EndpointsByReplica changedRanges = getChangedReplicasForLeaving(keyspaceName, endpoint, tokenMetadata, Keyspace.open(keyspaceName).getReplicationStrategy()); IFailureDetector failureDetector = FailureDetector.instance; - for (InetAddressAndPort ep : changedRanges.values()) + for (InetAddressAndPort ep : transform(changedRanges.flattenValues(), Replica::endpoint)) { if (failureDetector.isAlive(ep)) replicatingNodes.add(ep); @@ -4903,15 +5094,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE Collection> endpointsGroupedByDc = new ArrayList<>(); // mapping of dc's to nodes, use sorted map so that we get dcs sorted - SortedMap> sortedDcsToEndpoints = new TreeMap<>(); - sortedDcsToEndpoints.putAll(metadata.getTopology().getDatacenterEndpoints().asMap()); + SortedMap> sortedDcsToEndpoints = new TreeMap<>(metadata.getTopology().getDatacenterEndpoints().asMap()); for (Collection endpoints : sortedDcsToEndpoints.values()) endpointsGroupedByDc.add(endpoints); Map tokenOwnership = tokenMetadata.partitioner.describeOwnership(tokenMetadata.sortedTokens()); LinkedHashMap finalOwnership = Maps.newLinkedHashMap(); - Multimap> endpointToRanges = strategy.getAddressRanges(); + RangesByEndpoint endpointToRanges = strategy.getAddressReplicas(); // calculate ownership per dc for (Collection endpoints : endpointsGroupedByDc) { @@ -4919,10 +5109,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (InetAddressAndPort endpoint : endpoints) { float ownership = 0.0f; - for (Range range : endpointToRanges.get(endpoint)) + for (Replica replica : endpointToRanges.get(endpoint)) { - if (tokenOwnership.containsKey(range.right)) - ownership += tokenOwnership.get(range.right); + if (tokenOwnership.containsKey(replica.range().right)) + ownership += tokenOwnership.get(replica.range().right); } finalOwnership.put(endpoint, ownership); } @@ -4974,9 +5164,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE UUID hostId = entry.getValue(); InetAddressAndPort endpoint = entry.getKey(); result.put(endpoint.toString(withPort), - coreViewStatus.containsKey(hostId) - ? coreViewStatus.get(hostId) - : "UNKNOWN"); + coreViewStatus.getOrDefault(hostId, "UNKNOWN")); } return Collections.unmodifiableMap(result); @@ -5079,69 +5267,63 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } /** - * Seed data to the endpoints that will be responsible for it at the future + * Send data to the endpoints that will be responsible for it in the future * * @param rangesToStreamByKeyspace keyspaces and data ranges with endpoints included for each * @return async Future for whether stream was success */ - private Future streamRanges(Map, InetAddressAndPort>> rangesToStreamByKeyspace) + private Future streamRanges(Map rangesToStreamByKeyspace) { // First, we build a list of ranges to stream to each host, per table - Map>>> sessionsToStreamByKeyspace = new HashMap<>(); + Map sessionsToStreamByKeyspace = new HashMap<>(); - for (Map.Entry, InetAddressAndPort>> entry : rangesToStreamByKeyspace.entrySet()) + for (Map.Entry entry : rangesToStreamByKeyspace.entrySet()) { String keyspace = entry.getKey(); - Multimap, InetAddressAndPort> rangesWithEndpoints = entry.getValue(); + EndpointsByReplica rangesWithEndpoints = entry.getValue(); if (rangesWithEndpoints.isEmpty()) continue; + //Description is always Unbootstrap? Is that right? Map>> transferredRangePerKeyspace = SystemKeyspace.getTransferredRanges("Unbootstrap", keyspace, StorageService.instance.getTokenMetadata().partitioner); - Map>> rangesPerEndpoint = new HashMap<>(); - for (Map.Entry, InetAddressAndPort> endPointEntry : rangesWithEndpoints.entries()) + RangesByEndpoint.Mutable replicasPerEndpoint = new RangesByEndpoint.Mutable(); + for (Map.Entry endPointEntry : rangesWithEndpoints.flattenEntries()) { - Range range = endPointEntry.getKey(); - InetAddressAndPort endpoint = endPointEntry.getValue(); - - Set> transferredRanges = transferredRangePerKeyspace.get(endpoint); - if (transferredRanges != null && transferredRanges.contains(range)) + Replica local = endPointEntry.getKey(); + Replica remote = endPointEntry.getValue(); + Set> transferredRanges = transferredRangePerKeyspace.get(remote.endpoint()); + if (transferredRanges != null && transferredRanges.contains(local.range())) { - logger.debug("Skipping transferred range {} of keyspace {}, endpoint {}", range, keyspace, endpoint); + logger.debug("Skipping transferred range {} of keyspace {}, endpoint {}", local, keyspace, remote); continue; } - List> curRanges = rangesPerEndpoint.get(endpoint); - if (curRanges == null) - { - curRanges = new LinkedList<>(); - rangesPerEndpoint.put(endpoint, curRanges); - } - curRanges.add(range); + replicasPerEndpoint.put(remote.endpoint(), remote.decorateSubrange(local.range())); } - sessionsToStreamByKeyspace.put(keyspace, rangesPerEndpoint); + sessionsToStreamByKeyspace.put(keyspace, replicasPerEndpoint.asImmutableView()); } StreamPlan streamPlan = new StreamPlan(StreamOperation.DECOMMISSION); - // Vinculate StreamStateStore to current StreamPlan to update transferred ranges per StreamSession + // Vinculate StreamStateStore to current StreamPlan to update transferred rangeas per StreamSession streamPlan.listeners(streamStateStore); - for (Map.Entry>>> entry : sessionsToStreamByKeyspace.entrySet()) + for (Map.Entry entry : sessionsToStreamByKeyspace.entrySet()) { String keyspaceName = entry.getKey(); - Map>> rangesPerEndpoint = entry.getValue(); + RangesByEndpoint replicasPerEndpoint = entry.getValue(); - for (Map.Entry>> rangesEntry : rangesPerEndpoint.entrySet()) + for (Map.Entry rangesEntry : replicasPerEndpoint.asMap().entrySet()) { - List> ranges = rangesEntry.getValue(); + RangesAtEndpoint replicas = rangesEntry.getValue(); InetAddressAndPort newEndpoint = rangesEntry.getKey(); // TODO each call to transferRanges re-flushes, this is potentially a lot of waste - streamPlan.transferRanges(newEndpoint, keyspaceName, ranges); + streamPlan.transferRanges(newEndpoint, keyspaceName, replicas); } } return streamPlan.execute(); @@ -5151,53 +5333,109 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * Calculate pair of ranges to stream/fetch for given two range collections * (current ranges for keyspace and ranges after move to new token) * + * With transient replication the added wrinkle is that if a range transitions from full to transient then + * we need to stream the range despite the fact that we are retaining it as transient. Some replica + * somewhere needs to transition from transient to full and we wll be the source. + * + * If the range is transient and is transitioning to full then always fetch even if the range was already transient + * since a transiently replicated obviously needs to fetch data to become full. + * + * This why there is a continue after checking for instersection because intersection is not sufficient reason + * to do the subtraction since we might need to stream/fetch data anyways. + * * @param current collection of the ranges by current token * @param updated collection of the ranges after token is changed * @return pair of ranges to stream/fetch for given current and updated range collections */ - public Pair>, Set>> calculateStreamAndFetchRanges(Collection> current, Collection> updated) + public static Pair calculateStreamAndFetchRanges(RangesAtEndpoint current, RangesAtEndpoint updated) { - Set> toStream = new HashSet<>(); - Set> toFetch = new HashSet<>(); + // FIXME: transient replication + // this should always be the local node, except for tests TODO: assert this + RangesAtEndpoint.Builder toStream = RangesAtEndpoint.builder(current.endpoint()); + RangesAtEndpoint.Builder toFetch = RangesAtEndpoint.builder(current.endpoint()); - - for (Range r1 : current) + logger.debug("Calculating toStream"); + for (Replica r1 : current) { boolean intersect = false; - for (Range r2 : updated) + RangesAtEndpoint.Mutable remainder = null; + for (Replica r2 : updated) { - if (r1.intersects(r2)) + logger.debug("Comparing {} and {}", r1, r2); + //If we will end up transiently replicating send the entire thing and don't subtract + if (r1.intersectsOnRange(r2) && !(r1.isFull() && r2.isTransient())) { - // adding difference ranges to fetch from a ring - toStream.addAll(r1.subtract(r2)); + RangesAtEndpoint.Mutable oldRemainder = remainder; + remainder = new RangesAtEndpoint.Mutable(current.endpoint()); + if (oldRemainder != null) + { + for (Replica replica : oldRemainder) + { + remainder.addAll(replica.subtractIgnoreTransientStatus(r2.range())); + } + } + else + { + remainder.addAll(r1.subtractIgnoreTransientStatus(r2.range())); + } + logger.debug(" Intersects adding {}", remainder); intersect = true; } } if (!intersect) { - toStream.add(r1); // should seed whole old range + logger.debug(" Doesn't intersect adding {}", r1); + toStream.add(r1); // should stream whole old range + } + else + { + toStream.addAll(remainder); } } - for (Range r2 : updated) + logger.debug("Calculating toFetch"); + for (Replica r2 : updated) { boolean intersect = false; - for (Range r1 : current) + RangesAtEndpoint.Mutable remainder = null; + for (Replica r1 : current) { - if (r2.intersects(r1)) + logger.info("Comparing {} and {}", r2, r1); + //Transitioning from transient to full means fetch everything so intersection doesn't matter. + if (r2.intersectsOnRange(r1) && !(r1.isTransient() && r2.isFull())) { - // adding difference ranges to fetch from a ring - toFetch.addAll(r2.subtract(r1)); + RangesAtEndpoint.Mutable oldRemainder = remainder; + remainder = new RangesAtEndpoint.Mutable(current.endpoint()); + if (oldRemainder != null) + { + for (Replica replica : oldRemainder) + { + remainder.addAll(replica.subtractIgnoreTransientStatus(r1.range())); + } + } + else + { + remainder.addAll(r2.subtractIgnoreTransientStatus(r1.range())); + } + logger.debug(" Intersects adding {}", remainder); intersect = true; } } if (!intersect) { + logger.debug(" Doesn't intersect adding {}", r2); toFetch.add(r2); // should fetch whole old range } + else + { + toFetch.addAll(remainder); + } } - return Pair.create(toStream, toFetch); + logger.debug("To stream {}", toStream); + logger.debug("To fetch {}", toFetch); + + return Pair.create(toStream.build(), toFetch.build()); } public void bulkLoad(String directory) @@ -5233,10 +5471,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE this.keyspace = keyspace; try { - for (Map.Entry, List> entry : StorageService.instance.getRangeToAddressMap(keyspace).entrySet()) + for (Map.Entry, EndpointsForRange> entry : StorageService.instance.getRangeToAddressMap(keyspace).entrySet()) { Range range = entry.getKey(); - for (InetAddressAndPort endpoint : entry.getValue()) + EndpointsForRange replicas = entry.getValue(); + Replicas.temporaryAssertFull(replicas); + for (InetAddressAndPort endpoint : replicas.endpoints()) addRangeForEndpoint(range, endpoint); } } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 0f4c7ddf24..4e6295a423 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -217,6 +217,8 @@ public interface StorageServiceMBean extends NotificationEmitter @Deprecated public List getNaturalEndpoints(String keyspaceName, ByteBuffer key); public List getNaturalEndpointsWithPort(String keysapceName, ByteBuffer key); + public List getReplicas(String keyspaceName, String cf, String key); + /** * @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. */ diff --git a/src/java/org/apache/cassandra/service/WriteResponseHandler.java b/src/java/org/apache/cassandra/service/WriteResponseHandler.java index 65efeff4cc..a07aae6f3d 100644 --- a/src/java/org/apache/cassandra/service/WriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/WriteResponseHandler.java @@ -17,18 +17,15 @@ */ package org.apache.cassandra.service; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.ReplicaLayout; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessageIn; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; /** @@ -42,26 +39,18 @@ public class WriteResponseHandler extends AbstractWriteResponseHandler private static final AtomicIntegerFieldUpdater responsesUpdater = AtomicIntegerFieldUpdater.newUpdater(WriteResponseHandler.class, "responses"); - public WriteResponseHandler(Collection writeEndpoints, - Collection pendingEndpoints, - ConsistencyLevel consistencyLevel, - Keyspace keyspace, + public WriteResponseHandler(ReplicaLayout.ForToken replicaLayout, Runnable callback, WriteType writeType, long queryStartNanoTime) { - super(keyspace, writeEndpoints, pendingEndpoints, consistencyLevel, callback, writeType, queryStartNanoTime); + super(replicaLayout, callback, writeType, queryStartNanoTime); responses = totalBlockFor(); } - public WriteResponseHandler(InetAddressAndPort endpoint, WriteType writeType, Runnable callback, long queryStartNanoTime) + public WriteResponseHandler(ReplicaLayout.ForToken replicaLayout, WriteType writeType, long queryStartNanoTime) { - this(Arrays.asList(endpoint), Collections.emptyList(), ConsistencyLevel.ONE, null, callback, writeType, queryStartNanoTime); - } - - public WriteResponseHandler(InetAddressAndPort endpoint, WriteType writeType, long queryStartNanoTime) - { - this(endpoint, writeType, null, queryStartNanoTime); + this(replicaLayout, null, writeType, queryStartNanoTime); } public void response(MessageIn m) diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java index 61b99488cd..031326ef28 100644 --- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java @@ -17,11 +17,12 @@ */ package org.apache.cassandra.service.reads; -import java.util.List; import java.util.concurrent.TimeUnit; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; + +import org.apache.cassandra.locator.ReplicaLayout; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,15 +38,20 @@ import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.service.StorageProxy.LocalReadRunnable; import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; +import static com.google.common.collect.Iterables.all; +import static com.google.common.collect.Iterables.tryFind; + /** * Sends a read request to the replicas needed to satisfy a given ConsistencyLevel. * @@ -59,32 +65,27 @@ public abstract class AbstractReadExecutor private static final Logger logger = LoggerFactory.getLogger(AbstractReadExecutor.class); protected final ReadCommand command; - protected final ConsistencyLevel consistency; - protected final List targetReplicas; - protected final ReadRepair readRepair; - protected final DigestResolver digestResolver; - protected final ReadCallback handler; + private final ReplicaLayout.ForToken replicaLayout; + protected final ReadRepair readRepair; + protected final DigestResolver digestResolver; + protected final ReadCallback handler; protected final TraceState traceState; protected final ColumnFamilyStore cfs; protected final long queryStartNanoTime; + private final int initialDataRequestCount; protected volatile PartitionIterator result = null; - protected final Keyspace keyspace; - protected final int blockFor; - - AbstractReadExecutor(Keyspace keyspace, ColumnFamilyStore cfs, ReadCommand command, ConsistencyLevel consistency, List targetReplicas, long queryStartNanoTime) + AbstractReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaLayout.ForToken replicaLayout, int initialDataRequestCount, long queryStartNanoTime) { this.command = command; - this.consistency = consistency; - this.targetReplicas = targetReplicas; - this.readRepair = ReadRepair.create(command, queryStartNanoTime, consistency); - this.digestResolver = new DigestResolver(keyspace, command, consistency, readRepair, targetReplicas.size()); - this.handler = new ReadCallback(digestResolver, consistency, command, targetReplicas, queryStartNanoTime); + this.replicaLayout = replicaLayout; + this.initialDataRequestCount = initialDataRequestCount; + this.readRepair = ReadRepair.create(command, replicaLayout, queryStartNanoTime); + this.digestResolver = new DigestResolver<>(command, replicaLayout, readRepair, queryStartNanoTime); + this.handler = new ReadCallback<>(digestResolver, replicaLayout.consistencyLevel().blockFor(replicaLayout.keyspace()), command, replicaLayout, queryStartNanoTime); this.cfs = cfs; this.traceState = Tracing.instance.get(); this.queryStartNanoTime = queryStartNanoTime; - this.keyspace = keyspace; - this.blockFor = consistency.blockFor(keyspace); // Set the digest version (if we request some digests). This is the smallest version amongst all our target replicas since new nodes @@ -92,8 +93,8 @@ public abstract class AbstractReadExecutor // TODO: we need this when talking with pre-3.0 nodes. So if we preserve the digest format moving forward, we can get rid of this once // we stop being compatible with pre-3.0 nodes. int digestVersion = MessagingService.current_version; - for (InetAddressAndPort replica : targetReplicas) - digestVersion = Math.min(digestVersion, MessagingService.instance().getVersion(replica)); + for (Replica replica : replicaLayout.selected()) + digestVersion = Math.min(digestVersion, MessagingService.instance().getVersion(replica.endpoint())); command.setDigestVersion(digestVersion); } @@ -109,24 +110,34 @@ public abstract class AbstractReadExecutor return readRepair; } - protected void makeDataRequests(Iterable endpoints) + protected void makeFullDataRequests(ReplicaCollection replicas) { - makeRequests(command, endpoints); - + assert all(replicas, Replica::isFull); + makeRequests(command, replicas.filter(Replica::isFull)); } - protected void makeDigestRequests(Iterable endpoints) + protected void makeTransientDataRequests(ReplicaCollection replicas) { - makeRequests(command.copyAsDigestQuery(), endpoints); + makeRequests(command.copyAsTransientQuery(), replicas); } - private void makeRequests(ReadCommand readCommand, Iterable endpoints) + protected void makeDigestRequests(ReplicaCollection replicas) + { + assert all(replicas, Replica::isFull); + // only send digest requests to full replicas, send data requests instead to the transient replicas + makeRequests(command.copyAsDigestQuery(), replicas); + } + + private void makeRequests(ReadCommand readCommand, ReplicaCollection replicas) { boolean hasLocalEndpoint = false; - for (InetAddressAndPort endpoint : endpoints) + Preconditions.checkArgument(replicas.stream().allMatch(replica -> replica.isFull() || !readCommand.isDigestQuery()), + "Can not send digest requests to transient replicas"); + for (Replica replica: replicas) { - if (StorageProxy.canDoLocalRequest(endpoint)) + InetAddressAndPort endpoint = replica.endpoint(); + if (replica.isLocal()) { hasLocalEndpoint = true; continue; @@ -134,7 +145,6 @@ public abstract class AbstractReadExecutor if (traceState != null) traceState.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint); - logger.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint); MessageOut message = readCommand.createMessage(); MessagingService.instance().sendRRWithFailure(message, endpoint, handler); } @@ -153,17 +163,17 @@ public abstract class AbstractReadExecutor */ public abstract void maybeTryAdditionalReplicas(); - /** - * Get the replicas involved in the [finished] request. - * - * @return target replicas + the extra replica, *IF* we speculated. - */ - public abstract List getContactedReplicas(); - /** * send the initial set of requests */ - public abstract void executeAsync(); + public void executeAsync() + { + EndpointsForToken selected = replicaLayout().selected(); + EndpointsForToken fullDataRequests = selected.filter(Replica::isFull, initialDataRequestCount); + makeFullDataRequests(fullDataRequests); + makeTransientDataRequests(selected.filter(Replica::isTransient)); + makeDigestRequests(selected.filter(r -> r.isFull() && !fullDataRequests.contains(r))); + } /** * @return an executor appropriate for the configured speculative read policy @@ -171,34 +181,33 @@ public abstract class AbstractReadExecutor public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException { Keyspace keyspace = Keyspace.open(command.metadata().keyspace); - - List allLiveReplicas = StorageProxy.getLiveSortedEndpoints(keyspace, command.partitionKey()); - List selectedReplicas = consistencyLevel.filterForQuery(keyspace, allLiveReplicas); - - // Throw UAE early if we don't have enough replicas. - consistencyLevel.assureSufficientLiveNodes(keyspace, selectedReplicas); - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; + // Endpoints for Token + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forRead(keyspace, command.partitionKey().getToken(), consistencyLevel, retry); + // Speculative retry is disabled *OR* // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM) - return new NeverSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, selectedReplicas, queryStartNanoTime, false); + // TODO Looks like we might want to move speculation into the replica layout, but that might be a story for post-4.0 + return new NeverSpeculatingReadExecutor(cfs, command, replicaLayout, queryStartNanoTime, false); // There are simply no extra replicas to speculate. // Handle this separately so it can record failed attempts to speculate due to lack of replicas - if (selectedReplicas.size() == allLiveReplicas.size()) + if (replicaLayout.selected().size() == replicaLayout.all().size()) { boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL; - return new NeverSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, selectedReplicas, queryStartNanoTime, recordFailedSpeculation); + return new NeverSpeculatingReadExecutor(cfs, command, replicaLayout, queryStartNanoTime, recordFailedSpeculation); } - selectedReplicas.add(allLiveReplicas.get(selectedReplicas.size())); + // If CL.ALL, upgrade to AlwaysSpeculating; + // If We are going to contact every node anyway, ask for 2 full data requests instead of 1, for redundancy + // (same amount of requests in total, but we turn 1 digest request into a full blown data request) if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE)) - return new AlwaysSpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, selectedReplicas, queryStartNanoTime); + return new AlwaysSpeculatingReadExecutor(cfs, command, replicaLayout, queryStartNanoTime); else // PERCENTILE or CUSTOM. - return new SpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, selectedReplicas, queryStartNanoTime); + return new SpeculatingReadExecutor(cfs, command, replicaLayout, queryStartNanoTime); } /** @@ -208,10 +217,15 @@ public abstract class AbstractReadExecutor boolean shouldSpeculateAndMaybeWait() { // no latency information, or we're overloaded - if (cfs.sampleLatencyNanos > TimeUnit.MILLISECONDS.toNanos(command.getTimeout())) + if (cfs.sampleReadLatencyNanos > TimeUnit.MILLISECONDS.toNanos(command.getTimeout())) return false; - return !handler.await(cfs.sampleLatencyNanos, TimeUnit.NANOSECONDS); + return !handler.await(cfs.sampleReadLatencyNanos, TimeUnit.NANOSECONDS); + } + + ReplicaLayout.ForToken replicaLayout() + { + return replicaLayout; } void onReadTimeout() {} @@ -223,78 +237,36 @@ public abstract class AbstractReadExecutor * log it is as a failure if it should have happened * but couldn't due to lack of replicas */ - private final boolean recordFailedSpeculation; + private final boolean logFailedSpeculation; - NeverSpeculatingReadExecutor(Keyspace keyspace, - ColumnFamilyStore cfs, - ReadCommand command, - ConsistencyLevel consistencyLevel, - List targetReplicas, - long queryStartNanoTime, - boolean recordFailedSpeculation) + public NeverSpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, ReplicaLayout.ForToken replicaLayout, long queryStartNanoTime, boolean logFailedSpeculation) { - super(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime); - this.recordFailedSpeculation = recordFailedSpeculation; - } - - public void executeAsync() - { - makeDataRequests(targetReplicas.subList(0, 1)); - if (targetReplicas.size() > 1) - makeDigestRequests(targetReplicas.subList(1, targetReplicas.size())); + super(cfs, command, replicaLayout, 1, queryStartNanoTime); + this.logFailedSpeculation = logFailedSpeculation; } public void maybeTryAdditionalReplicas() { - if (shouldSpeculateAndMaybeWait() && recordFailedSpeculation) + if (shouldSpeculateAndMaybeWait() && logFailedSpeculation) { cfs.metric.speculativeInsufficientReplicas.inc(); } } - - public List getContactedReplicas() - { - return targetReplicas; - } } static class SpeculatingReadExecutor extends AbstractReadExecutor { private volatile boolean speculated = false; - public SpeculatingReadExecutor(Keyspace keyspace, - ColumnFamilyStore cfs, + public SpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, - ConsistencyLevel consistencyLevel, - List targetReplicas, + ReplicaLayout.ForToken replicaLayout, long queryStartNanoTime) { - super(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime); - } - - public void executeAsync() - { - // if CL + RR result in covering all replicas, getReadExecutor forces AlwaysSpeculating. So we know - // that the last replica in our list is "extra." - List initialReplicas = targetReplicas.subList(0, targetReplicas.size() - 1); - - if (handler.blockfor < initialReplicas.size()) - { - // We're hitting additional targets for read repair. Since our "extra" replica is the least- - // preferred by the snitch, we do an extra data read to start with against a replica more - // likely to reply; better to let RR fail than the entire query. - makeDataRequests(initialReplicas.subList(0, 2)); - if (initialReplicas.size() > 2) - makeDigestRequests(initialReplicas.subList(2, initialReplicas.size())); - } - else - { - // not doing read repair; all replies are important, so it doesn't matter which nodes we - // perform data reads against vs digest. - makeDataRequests(initialReplicas.subList(0, 1)); - if (initialReplicas.size() > 1) - makeDigestRequests(initialReplicas.subList(1, initialReplicas.size())); - } + // We're hitting additional targets for read repair (??). Since our "extra" replica is the least- + // preferred by the snitch, we do an extra data read to start with against a replica more + // likely to reply; better to let RR fail than the entire query. + super(cfs, command, replicaLayout, replicaLayout.blockFor() < replicaLayout.selected().size() ? 2 : 1, queryStartNanoTime); } public void maybeTryAdditionalReplicas() @@ -302,28 +274,43 @@ public abstract class AbstractReadExecutor if (shouldSpeculateAndMaybeWait()) { //Handle speculation stats first in case the callback fires immediately - speculated = true; cfs.metric.speculativeRetries.inc(); - // Could be waiting on the data, or on enough digests. - ReadCommand retryCommand = command; - if (handler.resolver.isDataPresent()) - retryCommand = command.copyAsDigestQuery(); + speculated = true; + + ReadCommand retryCommand = command; + Replica extraReplica; + if (handler.resolver.isDataPresent()) + { + extraReplica = tryFind(replicaLayout().all(), + r -> !replicaLayout().selected().contains(r)).orNull(); + + // we should only use a SpeculatingReadExecutor if we have an extra replica to speculate against + assert extraReplica != null; + + retryCommand = extraReplica.isTransient() + ? command.copyAsTransientQuery() + : command.copyAsDigestQuery(); + } + else + { + extraReplica = tryFind(replicaLayout().all(), + r -> r.isFull() && !replicaLayout().selected().contains(r)).orNull(); + if (extraReplica == null) + { + cfs.metric.speculativeInsufficientReplicas.inc(); + // cannot safely speculate a new data request, without more work - requests assumed to be + // unique per endpoint, and we have no full nodes left to speculate against + return; + } + } - InetAddressAndPort extraReplica = Iterables.getLast(targetReplicas); if (traceState != null) traceState.trace("speculating read retry on {}", extraReplica); logger.trace("speculating read retry on {}", extraReplica); - MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(), extraReplica, handler); + MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(), extraReplica.endpoint(), handler); } } - public List getContactedReplicas() - { - return speculated - ? targetReplicas - : targetReplicas.subList(0, targetReplicas.size() - 1); - } - @Override void onReadTimeout() { @@ -336,14 +323,12 @@ public abstract class AbstractReadExecutor private static class AlwaysSpeculatingReadExecutor extends AbstractReadExecutor { - public AlwaysSpeculatingReadExecutor(Keyspace keyspace, - ColumnFamilyStore cfs, + public AlwaysSpeculatingReadExecutor(ColumnFamilyStore cfs, ReadCommand command, - ConsistencyLevel consistencyLevel, - List targetReplicas, + ReplicaLayout.ForToken replicaLayout, long queryStartNanoTime) { - super(keyspace, cfs, command, consistencyLevel, targetReplicas, queryStartNanoTime); + super(cfs, command, replicaLayout, replicaLayout.selected().size() > 1 ? 2 : 1, queryStartNanoTime); } public void maybeTryAdditionalReplicas() @@ -351,17 +336,10 @@ public abstract class AbstractReadExecutor // no-op } - public List getContactedReplicas() - { - return targetReplicas; - } - @Override public void executeAsync() { - makeDataRequests(targetReplicas.subList(0, targetReplicas.size() > 1 ? 2 : 1)); - if (targetReplicas.size() > 2) - makeDigestRequests(targetReplicas.subList(2, targetReplicas.size())); + super.executeAsync(); cfs.metric.speculativeRetries.inc(); } @@ -407,7 +385,7 @@ public abstract class AbstractReadExecutor else { Tracing.trace("Digest mismatch: Mismatch for key {}", getKey()); - readRepair.startRepair(digestResolver, handler.endpoints, getContactedReplicas(), this::setResult); + readRepair.startRepair(digestResolver, this::setResult); } } @@ -425,8 +403,7 @@ public abstract class AbstractReadExecutor logger.trace("Timed out waiting on digest mismatch repair requests"); // the caught exception here will have CL.ALL from the repair command, // not whatever CL the initial command was at (CASSANDRA-7947) - int blockFor = consistency.blockFor(Keyspace.open(command.metadata().keyspace)); - throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true); + throw new ReadTimeoutException(replicaLayout().consistencyLevel(), handler.blockfor - 1, handler.blockfor, true); } } diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java index c0bff7a54d..9043e87054 100644 --- a/src/java/org/apache/cassandra/service/reads/DataResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java @@ -17,39 +17,55 @@ */ package org.apache.cassandra.service.reads; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import com.google.common.base.Joiner; +import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.db.rows.RangeTombstoneMarker; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterators; -import org.apache.cassandra.db.transform.*; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.net.*; +import org.apache.cassandra.db.transform.EmptyPartitionsDiscarder; +import org.apache.cassandra.db.transform.Filter; +import org.apache.cassandra.db.transform.FilteredPartitions; +import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.Replicas; +import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.reads.repair.ReadRepair; -public class DataResolver extends ResponseResolver +import static com.google.common.collect.Iterables.*; + +public class DataResolver, L extends ReplicaLayout> extends ResponseResolver { - private final long queryStartNanoTime; private final boolean enforceStrictLiveness; - public DataResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, int maxResponseCount, long queryStartNanoTime, ReadRepair readRepair) + public DataResolver(ReadCommand command, L replicaLayout, ReadRepair readRepair, long queryStartNanoTime) { - super(keyspace, command, consistency, readRepair, maxResponseCount); - this.queryStartNanoTime = queryStartNanoTime; + super(command, replicaLayout, readRepair, queryStartNanoTime); this.enforceStrictLiveness = command.metadata().enforceStrictLiveness(); } public PartitionIterator getData() { - ReadResponse response = responses.iterator().next().payload; + ReadResponse response = responses.get(0).payload; return UnfilteredPartitionIterators.filter(response.makeIterator(command), command.nowInSec()); } @@ -63,15 +79,13 @@ public class DataResolver extends ResponseResolver { // We could get more responses while this method runs, which is ok (we're happy to ignore any response not here // at the beginning of this method), so grab the response count once and use that through the method. - int count = responses.size(); - List iters = new ArrayList<>(count); - InetAddressAndPort[] sources = new InetAddressAndPort[count]; - for (int i = 0; i < count; i++) - { - MessageIn msg = responses.get(i); - iters.add(msg.payload.makeIterator(command)); - sources[i] = msg.from; - } + Collection> messages = responses.snapshot(); + assert !any(messages, msg -> msg.payload.isDigestResponse()); + + E replicas = replicaLayout.all().keep(transform(messages, msg -> msg.from)); + List iters = new ArrayList<>( + Collections2.transform(messages, msg -> msg.payload.makeIterator(command))); + assert replicas.size() == iters.size(); /* * Even though every response, individually, will honor the limit, it is possible that we will, after the merge, @@ -86,18 +100,19 @@ public class DataResolver extends ResponseResolver * * See CASSANDRA-13747 for more details. */ - DataLimits.Counter mergedResultCounter = command.limits().newCounter(command.nowInSec(), true, command.selectsFullPartition(), enforceStrictLiveness); - UnfilteredPartitionIterator merged = mergeWithShortReadProtection(iters, sources, mergedResultCounter); + UnfilteredPartitionIterator merged = mergeWithShortReadProtection(iters, + replicaLayout.withSelected(replicas), + mergedResultCounter); FilteredPartitions filtered = FilteredPartitions.filter(merged, new Filter(command.nowInSec(), command.metadata().enforceStrictLiveness())); PartitionIterator counted = Transformation.apply(filtered, mergedResultCounter); return Transformation.apply(counted, new EmptyPartitionsDiscarder()); } private UnfilteredPartitionIterator mergeWithShortReadProtection(List results, - InetAddressAndPort[] sources, + L sources, DataLimits.Counter mergedResultCounter) { // If we have only one results, there is no read repair to do and we can't get short reads @@ -110,17 +125,17 @@ public class DataResolver extends ResponseResolver */ if (!command.limits().isUnlimited()) for (int i = 0; i < results.size(); i++) - results.set(i, ShortReadProtection.extend(sources[i], results.get(i), command, mergedResultCounter, queryStartNanoTime, enforceStrictLiveness)); + results.set(i, ShortReadProtection.extend(sources.selected().get(i), results.get(i), command, mergedResultCounter, queryStartNanoTime, enforceStrictLiveness)); return UnfilteredPartitionIterators.merge(results, wrapMergeListener(readRepair.getMergeListener(sources), sources)); } private String makeResponsesDebugString(DecoratedKey partitionKey) { - return Joiner.on(",\n").join(Iterables.transform(getMessages(), m -> m.from + " => " + m.payload.toDebugString(command, partitionKey))); + return Joiner.on(",\n").join(transform(getMessages().snapshot(), m -> m.from + " => " + m.payload.toDebugString(command, partitionKey))); } - private UnfilteredPartitionIterators.MergeListener wrapMergeListener(UnfilteredPartitionIterators.MergeListener partitionListener, InetAddressAndPort[] sources) + private UnfilteredPartitionIterators.MergeListener wrapMergeListener(UnfilteredPartitionIterators.MergeListener partitionListener, L sources) { return new UnfilteredPartitionIterators.MergeListener() { @@ -144,8 +159,8 @@ public class DataResolver extends ResponseResolver String details = String.format("Error merging partition level deletion on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s", table, mergedDeletion == null ? "null" : mergedDeletion.toString(), - '[' + Joiner.on(", ").join(Iterables.transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString())) + ']', - Arrays.toString(sources), + '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString())) + ']', + sources.selected(), makeResponsesDebugString(partitionKey)); throw new AssertionError(details, e); } @@ -165,8 +180,8 @@ public class DataResolver extends ResponseResolver String details = String.format("Error merging rows on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s", table, merged == null ? "null" : merged.toString(table), - '[' + Joiner.on(", ").join(Iterables.transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']', - Arrays.toString(sources), + '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']', + sources.selected(), makeResponsesDebugString(partitionKey)); throw new AssertionError(details, e); } @@ -191,8 +206,8 @@ public class DataResolver extends ResponseResolver String details = String.format("Error merging RTs on %s: merged=%s, versions=%s, sources={%s}, debug info:%n %s", table, merged == null ? "null" : merged.toString(table), - '[' + Joiner.on(", ").join(Iterables.transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']', - Arrays.toString(sources), + '[' + Joiner.on(", ").join(transform(Arrays.asList(versions), rt -> rt == null ? "null" : rt.toString(table))) + ']', + sources.selected(), makeResponsesDebugString(partitionKey)); throw new AssertionError(details, e); } diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java index 897892f288..c3eee437c2 100644 --- a/src/java/org/apache/cassandra/service/reads/DigestResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java @@ -18,25 +18,35 @@ package org.apache.cassandra.service.reads; import java.nio.ByteBuffer; +import java.util.Collection; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.utils.ByteBufferUtil; -public class DigestResolver extends ResponseResolver -{ - private volatile ReadResponse dataResponse; +import static com.google.common.collect.Iterables.any; - public DigestResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair, int maxResponseCount) +public class DigestResolver, L extends ReplicaLayout> extends ResponseResolver +{ + private volatile MessageIn dataResponse; + + public DigestResolver(ReadCommand command, L replicas, ReadRepair readRepair, long queryStartNanoTime) { - super(keyspace, command, consistency, readRepair, maxResponseCount); + super(command, replicas, readRepair, queryStartNanoTime); Preconditions.checkArgument(command instanceof SinglePartitionReadCommand, "DigestResolver can only be used with SinglePartitionReadCommand commands"); } @@ -45,14 +55,60 @@ public class DigestResolver extends ResponseResolver public void preprocess(MessageIn message) { super.preprocess(message); - if (dataResponse == null && !message.payload.isDigestResponse()) - dataResponse = message.payload; + Replica replica = replicaLayout.getReplicaFor(message.from); + if (dataResponse == null && !message.payload.isDigestResponse() && replica.isFull()) + { + dataResponse = message; + } + else if (replica.isTransient() && message.payload.isDigestResponse()) + { + throw new IllegalStateException("digest response received from transient replica"); + } + } + + @VisibleForTesting + public boolean hasTransientResponse() + { + return hasTransientResponse(responses.snapshot()); + } + + private boolean hasTransientResponse(Collection> responses) + { + return any(responses, + msg -> !msg.payload.isDigestResponse() + && replicaLayout.getReplicaFor(msg.from).isTransient()); } public PartitionIterator getData() { assert isDataPresent(); - return UnfilteredPartitionIterators.filter(dataResponse.makeIterator(command), command.nowInSec()); + + Collection> responses = this.responses.snapshot(); + + if (!hasTransientResponse(responses)) + { + return UnfilteredPartitionIterators.filter(dataResponse.payload.makeIterator(command), command.nowInSec()); + } + else + { + // This path can be triggered only if we've got responses from full replicas and they match, but + // transient replica response still contains data, which needs to be reconciled. + DataResolver dataResolver = new DataResolver<>(command, + replicaLayout, + (ReadRepair) NoopReadRepair.instance, + queryStartNanoTime); + + dataResolver.preprocess(dataResponse); + // Forward differences to all full nodes + for (MessageIn response : responses) + { + Replica replica = replicaLayout.getReplicaFor(response.from); + if (replica.isTransient()) + dataResolver.preprocess(response); + } + + return dataResolver.resolve(); + } } public boolean responsesMatch() @@ -61,11 +117,12 @@ public class DigestResolver extends ResponseResolver // validate digests against each other; return false immediately on mismatch. ByteBuffer digest = null; - for (MessageIn message : responses) + for (MessageIn message : responses.snapshot()) { - ReadResponse response = message.payload; + if (replicaLayout.getReplicaFor(message.from).isTransient()) + continue; - ByteBuffer newDigest = response.digest(command); + ByteBuffer newDigest = message.payload.digest(command); if (digest == null) digest = newDigest; else if (!digest.equals(newDigest)) diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java index 537e684b06..3d39377a84 100644 --- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java @@ -18,42 +18,43 @@ package org.apache.cassandra.service.reads; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.db.PartitionRangeReadCommand; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IAsyncCallbackWithFailure; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.SimpleCondition; -public class ReadCallback implements IAsyncCallbackWithFailure +public class ReadCallback, L extends ReplicaLayout> implements IAsyncCallbackWithFailure { protected static final Logger logger = LoggerFactory.getLogger( ReadCallback.class ); public final ResponseResolver resolver; final SimpleCondition condition = new SimpleCondition(); private final long queryStartNanoTime; + // TODO: move to replica layout as well? final int blockfor; - final List endpoints; + final L replicaLayout; private final ReadCommand command; - private final ConsistencyLevel consistencyLevel; private static final AtomicIntegerFieldUpdater recievedUpdater = AtomicIntegerFieldUpdater.newUpdater(ReadCallback.class, "received"); private volatile int received = 0; @@ -62,37 +63,19 @@ public class ReadCallback implements IAsyncCallbackWithFailure private volatile int failures = 0; private final Map failureReasonByEndpoint; - private final Keyspace keyspace; // TODO push this into ConsistencyLevel? - - /** - * Constructor when response count has to be calculated and blocked for. - */ - public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, ReadCommand command, List filteredEndpoints, long queryStartNanoTime) - { - this(resolver, - consistencyLevel, - consistencyLevel.blockFor(Keyspace.open(command.metadata().keyspace)), - command, - Keyspace.open(command.metadata().keyspace), - filteredEndpoints, - queryStartNanoTime); - } - - public ReadCallback(ResponseResolver resolver, ConsistencyLevel consistencyLevel, int blockfor, ReadCommand command, Keyspace keyspace, List endpoints, long queryStartNanoTime) + public ReadCallback(ResponseResolver resolver, int blockfor, ReadCommand command, L replicaLayout, long queryStartNanoTime) { this.command = command; - this.keyspace = keyspace; this.blockfor = blockfor; - this.consistencyLevel = consistencyLevel; this.resolver = resolver; this.queryStartNanoTime = queryStartNanoTime; - this.endpoints = endpoints; + this.replicaLayout = replicaLayout; this.failureReasonByEndpoint = new ConcurrentHashMap<>(); // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897) - assert !(command instanceof PartitionRangeReadCommand) || blockfor >= endpoints.size(); + assert !(command instanceof PartitionRangeReadCommand) || blockfor >= replicaLayout.selected().size(); if (logger.isTraceEnabled()) - logger.trace("Blockfor is {}; setting up requests to {}", blockfor, StringUtils.join(this.endpoints, ",")); + logger.trace("Blockfor is {}; setting up requests to {}", blockfor, this.replicaLayout); } public boolean await(long timePastStart, TimeUnit unit) @@ -111,7 +94,7 @@ public class ReadCallback implements IAsyncCallbackWithFailure public void awaitResults() throws ReadFailureException, ReadTimeoutException { boolean signaled = await(command.getTimeout(), TimeUnit.MILLISECONDS); - boolean failed = blockfor + failures > endpoints.size(); + boolean failed = failures > 0 && blockfor + failures > replicaLayout.selected().size(); if (signaled && !failed) return; @@ -128,8 +111,8 @@ public class ReadCallback implements IAsyncCallbackWithFailure // Same as for writes, see AbstractWriteResponseHandler throw failed - ? new ReadFailureException(consistencyLevel, received, blockfor, resolver.isDataPresent(), failureReasonByEndpoint) - : new ReadTimeoutException(consistencyLevel, received, blockfor, resolver.isDataPresent()); + ? new ReadFailureException(replicaLayout.consistencyLevel(), received, blockfor, resolver.isDataPresent(), failureReasonByEndpoint) + : new ReadTimeoutException(replicaLayout.consistencyLevel(), received, blockfor, resolver.isDataPresent()); } public int blockFor() @@ -153,9 +136,7 @@ public class ReadCallback implements IAsyncCallbackWithFailure */ private boolean waitingFor(InetAddressAndPort from) { - return consistencyLevel.isDatacenterLocal() - ? DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(from)) - : true; + return !replicaLayout.consistencyLevel().isDatacenterLocal() || DatabaseDescriptor.getLocalDataCenter().equals(DatabaseDescriptor.getEndpointSnitch().getDatacenter(from)); } /** @@ -178,7 +159,7 @@ public class ReadCallback implements IAsyncCallbackWithFailure public void assureSufficientLiveNodes() throws UnavailableException { - consistencyLevel.assureSufficientLiveNodes(keyspace, endpoints); + replicaLayout.consistencyLevel().assureSufficientLiveNodesForRead(replicaLayout.keyspace(), replicaLayout.selected()); } public boolean isLatencyForSnitch() @@ -195,7 +176,7 @@ public class ReadCallback implements IAsyncCallbackWithFailure failureReasonByEndpoint.put(from, failureReason); - if (blockfor + n > endpoints.size()) + if (blockfor + n > replicaLayout.selected().size()) condition.signalAll(); } } diff --git a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java index f4f00a2043..e306b4db24 100644 --- a/src/java/org/apache/cassandra/service/reads/ResponseResolver.java +++ b/src/java/org/apache/cassandra/service/reads/ResponseResolver.java @@ -20,37 +20,49 @@ package org.apache.cassandra.service.reads; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.utils.concurrent.Accumulator; -public abstract class ResponseResolver +public abstract class ResponseResolver, L extends ReplicaLayout> { protected static final Logger logger = LoggerFactory.getLogger(ResponseResolver.class); - protected final Keyspace keyspace; protected final ReadCommand command; - protected final ConsistencyLevel consistency; - protected final ReadRepair readRepair; + protected final L replicaLayout; + protected final ReadRepair readRepair; // Accumulator gives us non-blocking thread-safety with optimal algorithmic constraints protected final Accumulator> responses; + protected final long queryStartNanoTime; - public ResponseResolver(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair, int maxResponseCount) + public ResponseResolver(ReadCommand command, L replicaLayout, ReadRepair readRepair, long queryStartNanoTime) { - this.keyspace = keyspace; this.command = command; - this.consistency = consistency; + this.replicaLayout = replicaLayout; this.readRepair = readRepair; - this.responses = new Accumulator<>(maxResponseCount); + // TODO: calculate max possible replicas for the query (e.g. local dc queries won't contact remotes) + this.responses = new Accumulator<>(replicaLayout.all().size()); + this.queryStartNanoTime = queryStartNanoTime; } public abstract boolean isDataPresent(); public void preprocess(MessageIn message) { - responses.add(message); + try + { + responses.add(message); + } + catch (IllegalStateException e) + { + logger.error("Encountered error while trying to preprocess the message {}: %s in command {}, replicas: {}", message, command, readRepair, replicaLayout.consistencyLevel(), replicaLayout.selected()); + throw e; + } } public Accumulator> getMessages() diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java index d4e8957952..580b790407 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java @@ -18,12 +18,13 @@ package org.apache.cassandra.service.reads; -import java.util.Collections; +import org.apache.cassandra.locator.Endpoints; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.concurrent.StageManager; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -39,7 +40,8 @@ import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.Range; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.service.StorageProxy; @@ -47,8 +49,9 @@ import org.apache.cassandra.tracing.Tracing; public class ShortReadPartitionsProtection extends Transformation implements MorePartitions { + private static final Logger logger = LoggerFactory.getLogger(ShortReadPartitionsProtection.class); private final ReadCommand command; - private final InetAddressAndPort source; + private final Replica source; private final DataLimits.Counter singleResultCounter; // unmerged per-source counter private final DataLimits.Counter mergedResultCounter; // merged end-result counter @@ -59,7 +62,7 @@ public class ShortReadPartitionsProtection extends Transformation executeReadCommand(cmd, replicaLayout), singleResultCounter, mergedResultCounter); return Transformation.apply(MoreRows.extend(partition, protection), protection); @@ -140,9 +144,9 @@ public class ShortReadPartitionsProtection extends Transformation(lastPartitionKey, bounds.right); DataRange newDataRange = cmd.dataRange().forSubRange(newBounds); - return cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange); + ReplicaLayout.ForRange replicaLayout = ReplicaLayout.forSingleReplica(Keyspace.open(command.metadata().keyspace), cmd.dataRange().keyRange(), source); + return executeReadCommand(cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange), replicaLayout); } - private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd) + private , L extends ReplicaLayout> UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, L replicaLayout) { - Keyspace keyspace = Keyspace.open(command.metadata().keyspace); - DataResolver resolver = new DataResolver(keyspace, cmd, ConsistencyLevel.ONE, 1, queryStartNanoTime, NoopReadRepair.instance); - ReadCallback handler = new ReadCallback(resolver, ConsistencyLevel.ONE, cmd, Collections.singletonList(source), queryStartNanoTime); + DataResolver resolver = new DataResolver<>(cmd, replicaLayout, (NoopReadRepair)NoopReadRepair.instance, queryStartNanoTime); + ReadCallback handler = new ReadCallback<>(resolver, replicaLayout.consistencyLevel().blockFor(replicaLayout.keyspace()), cmd, replicaLayout, queryStartNanoTime); - if (StorageProxy.canDoLocalRequest(source)) + if (source.isLocal()) StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler)); else - MessagingService.instance().sendRRWithFailure(cmd.createMessage(), source, handler); + MessagingService.instance().sendRRWithFailure(cmd.createMessage(), source.endpoint(), handler); // We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results. handler.awaitResults(); diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java index f603e9ba34..ef1d45bfb7 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadProtection.java @@ -26,6 +26,7 @@ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.transform.MorePartitions; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; /** * We have a potential short read if the result from a given node contains the requested number of rows @@ -40,7 +41,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; public class ShortReadProtection { @SuppressWarnings("resource") - public static UnfilteredPartitionIterator extend(InetAddressAndPort source, UnfilteredPartitionIterator partitions, + public static UnfilteredPartitionIterator extend(Replica source, UnfilteredPartitionIterator partitions, ReadCommand command, DataLimits.Counter mergedResultCounter, long queryStartNanoTime, boolean enforceStrictLiveness) { diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java index 6b1da0bfd0..8dc7fc712c 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java @@ -33,14 +33,14 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.transform.MoreRows; import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tracing.Tracing; class ShortReadRowsProtection extends Transformation implements MoreRows { private final ReadCommand command; - private final InetAddressAndPort source; + private final Replica source; private final DataLimits.Counter singleResultCounter; // unmerged per-source counter private final DataLimits.Counter mergedResultCounter; // merged end-result counter private final Function commandExecutor; @@ -53,7 +53,7 @@ class ShortReadRowsProtection extends Transformation implements MoreRows commandExecutor, DataLimits.Counter singleResultCounter, DataLimits.Counter mergedResultCounter) { diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java index 7e3f0aefb5..30dea74f8b 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java @@ -18,29 +18,23 @@ package org.apache.cassandra.service.reads.repair; -import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; -import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import com.codahale.metrics.Meter; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.reads.DataResolver; @@ -48,11 +42,11 @@ import org.apache.cassandra.service.reads.DigestResolver; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.tracing.Tracing; -public abstract class AbstractReadRepair implements ReadRepair +public abstract class AbstractReadRepair, L extends ReplicaLayout> implements ReadRepair { protected final ReadCommand command; protected final long queryStartNanoTime; - protected final ConsistencyLevel consistency; + protected final L replicaLayout; protected final ColumnFamilyStore cfs; private volatile DigestRepair digestRepair = null; @@ -62,41 +56,25 @@ public abstract class AbstractReadRepair implements ReadRepair private final DataResolver dataResolver; private final ReadCallback readCallback; private final Consumer resultConsumer; - private final List initialContacts; - public DigestRepair(DataResolver dataResolver, ReadCallback readCallback, Consumer resultConsumer, List initialContacts) + public DigestRepair(DataResolver dataResolver, ReadCallback readCallback, Consumer resultConsumer) { this.dataResolver = dataResolver; this.readCallback = readCallback; this.resultConsumer = resultConsumer; - this.initialContacts = initialContacts; } } public AbstractReadRepair(ReadCommand command, - long queryStartNanoTime, - ConsistencyLevel consistency) + L replicaLayout, + long queryStartNanoTime) { this.command = command; this.queryStartNanoTime = queryStartNanoTime; - this.consistency = consistency; + this.replicaLayout = replicaLayout; this.cfs = Keyspace.openAndGetStore(command.metadata()); } - private int getMaxResponses() - { - AbstractReplicationStrategy strategy = cfs.keyspace.getReplicationStrategy(); - if (consistency.isDatacenterLocal() && strategy instanceof NetworkTopologyStrategy) - { - NetworkTopologyStrategy nts = (NetworkTopologyStrategy) strategy; - return nts.getReplicationFactor(DatabaseDescriptor.getLocalDataCenter()); - } - else - { - return strategy.getReplicationFactor(); - } - } - void sendReadCommand(InetAddressAndPort to, ReadCallback readCallback) { MessagingService.instance().sendRRWithFailure(command.createMessage(), to, readCallback); @@ -105,24 +83,23 @@ public abstract class AbstractReadRepair implements ReadRepair abstract Meter getRepairMeter(); // digestResolver isn't used here because we resend read requests to all participants - public void startRepair(DigestResolver digestResolver, List allEndpoints, List contactedEndpoints, Consumer resultConsumer) + public void startRepair(DigestResolver digestResolver, Consumer resultConsumer) { getRepairMeter().mark(); // Do a full data read to resolve the correct response (and repair node that need be) - Keyspace keyspace = Keyspace.open(command.metadata().keyspace); - DataResolver resolver = new DataResolver(keyspace, command, ConsistencyLevel.ALL, getMaxResponses(), queryStartNanoTime, this); - ReadCallback readCallback = new ReadCallback(resolver, ConsistencyLevel.ALL, consistency.blockFor(cfs.keyspace), command, - keyspace, allEndpoints, queryStartNanoTime); + DataResolver resolver = new DataResolver<>(command, replicaLayout, this, queryStartNanoTime); + ReadCallback readCallback = new ReadCallback<>(resolver, replicaLayout.consistencyLevel().blockFor(cfs.keyspace), + command, replicaLayout, queryStartNanoTime); - digestRepair = new DigestRepair(resolver, readCallback, resultConsumer, contactedEndpoints); + digestRepair = new DigestRepair(resolver, readCallback, resultConsumer); - for (InetAddressAndPort endpoint : contactedEndpoints) + for (Replica replica : replicaLayout.selected()) { - Tracing.trace("Enqueuing full data read to {}", endpoint); - sendReadCommand(endpoint, readCallback); + Tracing.trace("Enqueuing full data read to {}", replica); + sendReadCommand(replica.endpoint(), readCallback); } - ReadRepairDiagnostics.startRepair(this, contactedEndpoints, digestResolver, allEndpoints); + ReadRepairDiagnostics.startRepair(this, replicaLayout.selected().endpoints(), digestResolver, replicaLayout.all().endpoints()); } public void awaitReads() throws ReadTimeoutException @@ -137,15 +114,11 @@ public abstract class AbstractReadRepair implements ReadRepair private boolean shouldSpeculate() { + ConsistencyLevel consistency = replicaLayout.consistencyLevel(); ConsistencyLevel speculativeCL = consistency.isDatacenterLocal() ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; return consistency != ConsistencyLevel.EACH_QUORUM && consistency.satisfies(speculativeCL, cfs.keyspace) - && cfs.sampleLatencyNanos <= TimeUnit.MILLISECONDS.toNanos(command.getTimeout()); - } - - Iterable getCandidatesForToken(Token token) - { - return BlockingReadRepairs.getCandidateEndpoints(cfs.keyspace, token, consistency); + && cfs.sampleReadLatencyNanos <= TimeUnit.MILLISECONDS.toNanos(command.getTimeout()); } public void maybeSendAdditionalReads() @@ -156,20 +129,17 @@ public abstract class AbstractReadRepair implements ReadRepair if (repair == null) return; - if (shouldSpeculate() && !repair.readCallback.await(cfs.sampleLatencyNanos, TimeUnit.NANOSECONDS)) + if (shouldSpeculate() && !repair.readCallback.await(cfs.sampleReadLatencyNanos, TimeUnit.NANOSECONDS)) { - Set contacted = Sets.newHashSet(repair.initialContacts); - Token replicaToken = ((SinglePartitionReadCommand) command).partitionKey().getToken(); - Iterable candidates = getCandidatesForToken(replicaToken); + L uncontacted = replicaLayout.forNaturalUncontacted(); + if (uncontacted.selected().isEmpty()) + return; - Optional endpoint = Iterables.tryFind(candidates, e -> !contacted.contains(e)); - if (endpoint.isPresent()) - { - Tracing.trace("Enqueuing speculative full data read to {}", endpoint); - sendReadCommand(endpoint.get(), repair.readCallback); - ReadRepairMetrics.speculatedRead.mark(); - ReadRepairDiagnostics.speculatedRead(this, endpoint.get(), candidates); - } + Replica replica = uncontacted.selected().iterator().next(); + Tracing.trace("Enqueuing speculative full data read to {}", replica); + sendReadCommand(replica.endpoint(), repair.readCallback); + ReadRepairMetrics.speculatedRead.mark(); + ReadRepairDiagnostics.speculatedRead(this, replica.endpoint(), uncontacted.all().endpoints()); } } } diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java index 8d69bef508..54af2cf008 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingPartitionRepair.java @@ -20,16 +20,14 @@ package org.apache.cassandra.service.reads.repair; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.AbstractFuture; import org.apache.cassandra.db.ColumnFamilyStore; @@ -38,7 +36,11 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.Replicas; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.net.IAsyncCallback; import org.apache.cassandra.net.MessageIn; @@ -47,33 +49,29 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.tracing.Tracing; -public class BlockingPartitionRepair extends AbstractFuture implements IAsyncCallback +public class BlockingPartitionRepair, L extends ReplicaLayout> extends AbstractFuture implements IAsyncCallback { - private final Keyspace keyspace; private final DecoratedKey key; - private final ConsistencyLevel consistency; - private final InetAddressAndPort[] participants; - private final ConcurrentMap pendingRepairs; + private final L replicaLayout; + private final Map pendingRepairs; private final CountDownLatch latch; private volatile long mutationsSentTime; - public BlockingPartitionRepair(Keyspace keyspace, DecoratedKey key, ConsistencyLevel consistency, Map repairs, int maxBlockFor, InetAddressAndPort[] participants) + public BlockingPartitionRepair(DecoratedKey key, Map repairs, int maxBlockFor, L replicaLayout) { - this.keyspace = keyspace; this.key = key; - this.consistency = consistency; this.pendingRepairs = new ConcurrentHashMap<>(repairs); - this.participants = participants; + this.replicaLayout = replicaLayout; // here we remove empty repair mutations from the block for total, since // we're not sending them mutations int blockFor = maxBlockFor; - for (InetAddressAndPort participant: participants) + for (Replica participant: replicaLayout.selected()) { // remote dcs can sometimes get involved in dc-local reads. We want to repair // them if they do, but they shouldn't interfere with blocking the client read. - if (!repairs.containsKey(participant) && shouldBlockOn(participant)) + if (!repairs.containsKey(participant) && shouldBlockOn(participant.endpoint())) blockFor--; } @@ -99,7 +97,7 @@ public class BlockingPartitionRepair extends AbstractFuture implements I private boolean shouldBlockOn(InetAddressAndPort endpoint) { - return !consistency.isDatacenterLocal() || isLocal(endpoint); + return !replicaLayout.consistencyLevel().isDatacenterLocal() || isLocal(endpoint); } @VisibleForTesting @@ -107,7 +105,7 @@ public class BlockingPartitionRepair extends AbstractFuture implements I { if (shouldBlockOn(from)) { - pendingRepairs.remove(from); + pendingRepairs.remove(replicaLayout.getReplicaFor(from)); latch.countDown(); } } @@ -148,20 +146,23 @@ public class BlockingPartitionRepair extends AbstractFuture implements I public void sendInitialRepairs() { mutationsSentTime = System.nanoTime(); - for (Map.Entry entry: pendingRepairs.entrySet()) + Replicas.assertFull(pendingRepairs.keySet()); + + for (Map.Entry entry: pendingRepairs.entrySet()) { - InetAddressAndPort destination = entry.getKey(); + Replica destination = entry.getKey(); + Preconditions.checkArgument(destination.isFull(), "Can't send repairs to transient replicas: %s", destination); Mutation mutation = entry.getValue(); TableId tableId = extractUpdate(mutation).metadata().id; Tracing.trace("Sending read-repair-mutation to {}", destination); // use a separate verb here to avoid writing hints on timeouts - sendRR(mutation.createMessage(MessagingService.Verb.READ_REPAIR), destination); + sendRR(mutation.createMessage(MessagingService.Verb.READ_REPAIR), destination.endpoint()); ColumnFamilyStore.metricsFor(tableId).readRepairRequests.mark(); - if (!shouldBlockOn(destination)) + if (!shouldBlockOn(destination.endpoint())) pendingRepairs.remove(destination); - ReadRepairDiagnostics.sendInitialRepair(this, destination, mutation); + ReadRepairDiagnostics.sendInitialRepair(this, destination.endpoint(), mutation); } } @@ -197,9 +198,8 @@ public class BlockingPartitionRepair extends AbstractFuture implements I if (awaitRepairs(timeout, timeoutUnit)) return; - Set exclude = Sets.newHashSet(participants); - Iterable candidates = Iterables.filter(getCandidateEndpoints(), e -> !exclude.contains(e)); - if (Iterables.isEmpty(candidates)) + L newCandidates = replicaLayout.forNaturalUncontacted(); + if (newCandidates.selected().isEmpty()) return; PartitionUpdate update = mergeUnackedUpdates(); @@ -212,34 +212,34 @@ public class BlockingPartitionRepair extends AbstractFuture implements I Mutation[] versionedMutations = new Mutation[msgVersionIdx(MessagingService.current_version) + 1]; - for (InetAddressAndPort endpoint: candidates) + for (Replica replica : newCandidates.selected()) { - int versionIdx = msgVersionIdx(MessagingService.instance().getVersion(endpoint)); + int versionIdx = msgVersionIdx(MessagingService.instance().getVersion(replica.endpoint())); Mutation mutation = versionedMutations[versionIdx]; if (mutation == null) { - mutation = BlockingReadRepairs.createRepairMutation(update, consistency, endpoint, true); + mutation = BlockingReadRepairs.createRepairMutation(update, replicaLayout.consistencyLevel(), replica.endpoint(), true); versionedMutations[versionIdx] = mutation; } if (mutation == null) { // the mutation is too large to send. - ReadRepairDiagnostics.speculatedWriteOversized(this, endpoint); + ReadRepairDiagnostics.speculatedWriteOversized(this, replica.endpoint()); continue; } - Tracing.trace("Sending speculative read-repair-mutation to {}", endpoint); - sendRR(mutation.createMessage(MessagingService.Verb.READ_REPAIR), endpoint); - ReadRepairDiagnostics.speculatedWrite(this, endpoint, mutation); + Tracing.trace("Sending speculative read-repair-mutation to {}", replica); + sendRR(mutation.createMessage(MessagingService.Verb.READ_REPAIR), replica.endpoint()); + ReadRepairDiagnostics.speculatedWrite(this, replica.endpoint(), mutation); } } Keyspace getKeyspace() { - return keyspace; + return replicaLayout.keyspace(); } DecoratedKey getKey() @@ -249,13 +249,6 @@ public class BlockingPartitionRepair extends AbstractFuture implements I ConsistencyLevel getConsistency() { - return consistency; + return replicaLayout.consistencyLevel(); } - - @VisibleForTesting - protected Iterable getCandidateEndpoints() - { - return BlockingReadRepairs.getCandidateEndpoints(keyspace, key.getToken(), consistency); - } - } diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java index e46372e938..402aed05b6 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java @@ -23,18 +23,19 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.locator.Endpoints; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.codahale.metrics.Meter; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.metrics.ReadRepairMetrics; import org.apache.cassandra.tracing.Tracing; @@ -43,20 +44,22 @@ import org.apache.cassandra.tracing.Tracing; * updates have been written to nodes needing correction. Breaks write * atomicity in some situations */ -public class BlockingReadRepair extends AbstractReadRepair +public class BlockingReadRepair, L extends ReplicaLayout> extends AbstractReadRepair { private static final Logger logger = LoggerFactory.getLogger(BlockingReadRepair.class); protected final Queue repairs = new ConcurrentLinkedQueue<>(); + private final int blockFor; - public BlockingReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + BlockingReadRepair(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - super(command, queryStartNanoTime, consistency); + super(command, replicaLayout, queryStartNanoTime); + this.blockFor = replicaLayout.consistencyLevel().blockFor(cfs.keyspace); } - public UnfilteredPartitionIterators.MergeListener getMergeListener(InetAddressAndPort[] endpoints) + public UnfilteredPartitionIterators.MergeListener getMergeListener(L replicaLayout) { - return new PartitionIteratorMergeListener(endpoints, command, consistency, this); + return new PartitionIteratorMergeListener(replicaLayout, command, this.replicaLayout.consistencyLevel(), this); } @Override @@ -70,7 +73,7 @@ public class BlockingReadRepair extends AbstractReadRepair { for (BlockingPartitionRepair repair: repairs) { - repair.maybeSendAdditionalWrites(cfs.sampleLatencyNanos, TimeUnit.NANOSECONDS); + repair.maybeSendAdditionalWrites(cfs.transientWriteLatencyNanos, TimeUnit.NANOSECONDS); } } @@ -88,20 +91,20 @@ public class BlockingReadRepair extends AbstractReadRepair if (timedOut) { // We got all responses, but timed out while repairing - int blockFor = consistency.blockFor(cfs.keyspace); + int blockFor = replicaLayout.consistencyLevel().blockFor(cfs.keyspace); if (Tracing.isTracing()) Tracing.trace("Timed out while read-repairing after receiving all {} data and digest responses", blockFor); else logger.debug("Timeout while read-repairing after receiving all {} data and digest responses", blockFor); - throw new ReadTimeoutException(consistency, blockFor-1, blockFor, true); + throw new ReadTimeoutException(replicaLayout.consistencyLevel(), blockFor - 1, blockFor, true); } } @Override - public void repairPartition(DecoratedKey key, Map mutations, InetAddressAndPort[] destinations) + public void repairPartition(DecoratedKey partitionKey, Map mutations, L replicaLayout) { - BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair(cfs.keyspace, key, consistency, mutations, consistency.blockFor(cfs.keyspace), destinations); + BlockingPartitionRepair blockingRepair = new BlockingPartitionRepair<>(partitionKey, mutations, blockFor, replicaLayout); blockingRepair.sendInitialRepairs(); repairs.add(blockingRepair); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java index e5f71791b4..ceb1765690 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java @@ -18,10 +18,6 @@ package org.apache.cassandra.service.reads.repair; -import java.util.List; - -import com.google.common.collect.Iterables; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,13 +27,10 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.tracing.Tracing; public class BlockingReadRepairs @@ -47,18 +40,6 @@ public class BlockingReadRepairs private static final boolean DROP_OVERSIZED_READ_REPAIR_MUTATIONS = Boolean.getBoolean("cassandra.drop_oversized_readrepair_mutations"); - /** - * Returns all of the endpoints that are replicas for the given key. If the consistency level is datacenter - * local, only the endpoints in the local dc will be returned. - */ - static Iterable getCandidateEndpoints(Keyspace keyspace, Token token, ConsistencyLevel consistency) - { - List endpoints = StorageProxy.getLiveSortedEndpoints(keyspace, token); - return consistency.isDatacenterLocal() && keyspace.getReplicationStrategy() instanceof NetworkTopologyStrategy - ? Iterables.filter(endpoints, ConsistencyLevel::isLocal) - : endpoints; - } - /** * Create a read repair mutation from the given update, if the mutation is not larger than the maximum * mutation size, otherwise return null. Or, if we're configured to be strict, throw an exception. diff --git a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java index a43e3ebd55..4af4a92645 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/NoopReadRepair.java @@ -18,7 +18,6 @@ package org.apache.cassandra.service.reads.repair; -import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -27,24 +26,28 @@ import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.service.reads.DigestResolver; /** * Bypasses the read repair path for short read protection and testing */ -public class NoopReadRepair implements ReadRepair +public class NoopReadRepair, L extends ReplicaLayout> implements ReadRepair { public static final NoopReadRepair instance = new NoopReadRepair(); private NoopReadRepair() {} - public UnfilteredPartitionIterators.MergeListener getMergeListener(InetAddressAndPort[] endpoints) + @Override + public UnfilteredPartitionIterators.MergeListener getMergeListener(L replicas) { return UnfilteredPartitionIterators.MergeListener.NOOP; } - public void startRepair(DigestResolver digestResolver, List allEndpoints, List contactedEndpoints, Consumer resultConsumer) + @Override + public void startRepair(DigestResolver digestResolver, Consumer resultConsumer) { resultConsumer.accept(digestResolver.getData()); } @@ -72,7 +75,7 @@ public class NoopReadRepair implements ReadRepair } @Override - public void repairPartition(DecoratedKey key, Map mutations, InetAddressAndPort[] destinations) + public void repairPartition(DecoratedKey partitionKey, Map mutations, L replicaLayout) { } diff --git a/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java index 6cf761af0d..4cae3aec06 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java +++ b/src/java/org/apache/cassandra/service/reads/repair/PartitionIteratorMergeListener.java @@ -28,18 +28,18 @@ import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterators; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; public class PartitionIteratorMergeListener implements UnfilteredPartitionIterators.MergeListener { - private final InetAddressAndPort[] sources; + private final ReplicaLayout replicaLayout; private final ReadCommand command; private final ConsistencyLevel consistency; private final ReadRepair readRepair; - public PartitionIteratorMergeListener(InetAddressAndPort[] sources, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair) + public PartitionIteratorMergeListener(ReplicaLayout replicaLayout, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair) { - this.sources = sources; + this.replicaLayout = replicaLayout; this.command = command; this.consistency = consistency; this.readRepair = readRepair; @@ -47,10 +47,10 @@ public class PartitionIteratorMergeListener implements UnfilteredPartitionIterat public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List versions) { - return new RowIteratorMergeListener(partitionKey, columns(versions), isReversed(versions), sources, command, consistency, readRepair); + return new RowIteratorMergeListener(partitionKey, columns(versions), isReversed(versions), replicaLayout, command, consistency, readRepair); } - private RegularAndStaticColumns columns(List versions) + protected RegularAndStaticColumns columns(List versions) { Columns statics = Columns.NONE; Columns regulars = Columns.NONE; @@ -66,7 +66,7 @@ public class PartitionIteratorMergeListener implements UnfilteredPartitionIterat return new RegularAndStaticColumns(statics, regulars); } - private boolean isReversed(List versions) + protected boolean isReversed(List versions) { for (UnfilteredRowIterator iter : versions) { diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java index d994b2301b..c13e2d6e87 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java @@ -21,27 +21,28 @@ package org.apache.cassandra.service.reads.repair; import java.util.Map; import com.codahale.metrics.Meter; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.metrics.ReadRepairMetrics; /** * Only performs the collection of data responses and reconciliation of them, doesn't send repair mutations * to replicas. This preserves write atomicity, but doesn't provide monotonic quorum reads */ -public class ReadOnlyReadRepair extends AbstractReadRepair +public class ReadOnlyReadRepair, L extends ReplicaLayout> extends AbstractReadRepair { - public ReadOnlyReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + ReadOnlyReadRepair(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - super(command, queryStartNanoTime, consistency); + super(command, replicaLayout, queryStartNanoTime); } @Override - public UnfilteredPartitionIterators.MergeListener getMergeListener(InetAddressAndPort[] endpoints) + public UnfilteredPartitionIterators.MergeListener getMergeListener(L replicaLayout) { return UnfilteredPartitionIterators.MergeListener.NOOP; } @@ -59,7 +60,7 @@ public class ReadOnlyReadRepair extends AbstractReadRepair } @Override - public void repairPartition(DecoratedKey key, Map mutations, InetAddressAndPort[] destinations) + public void repairPartition(DecoratedKey partitionKey, Map mutations, L replicaLayout) { throw new UnsupportedOperationException("ReadOnlyReadRepair shouldn't be trying to repair partitions"); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java index 97f0f679ce..168f003384 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java @@ -17,44 +17,45 @@ */ package org.apache.cassandra.service.reads.repair; -import java.util.List; import java.util.Map; import java.util.function.Consumer; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.locator.Endpoints; + import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.service.reads.DigestResolver; -public interface ReadRepair +public interface ReadRepair, L extends ReplicaLayout> { public interface Factory { - ReadRepair create(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency); + , L extends ReplicaLayout> ReadRepair create(ReadCommand command, L replicaLayout, long queryStartNanoTime); + } + + static , L extends ReplicaLayout> ReadRepair create(ReadCommand command, L replicaPlan, long queryStartNanoTime) + { + return command.metadata().params.readRepair.create(command, replicaPlan, queryStartNanoTime); } /** * Used by DataResolver to generate corrections as the partition iterator is consumed */ - UnfilteredPartitionIterators.MergeListener getMergeListener(InetAddressAndPort[] endpoints); + UnfilteredPartitionIterators.MergeListener getMergeListener(L replicaLayout); /** * Called when the digests from the initial read don't match. Reads may block on the * repair started by this method. * @param digestResolver supplied so we can get the original data response - * @param allEndpoints all available replicas for this read - * @param contactedEndpoints the replicas we actually sent requests to * @param resultConsumer hook for the repair to set it's result on completion */ - public void startRepair(DigestResolver digestResolver, - List allEndpoints, - List contactedEndpoints, - Consumer resultConsumer); + public void startRepair(DigestResolver digestResolver, Consumer resultConsumer); /** * Block on the reads (or timeout) sent out in {@link ReadRepair#startRepair} @@ -80,18 +81,14 @@ public interface ReadRepair */ public void maybeSendAdditionalWrites(); - /** - * Hook for the merge listener to start repairs on individual partitions. - */ - void repairPartition(DecoratedKey key, Map mutations, InetAddressAndPort[] destinations); - /** * Block on any mutations (or timeout) we sent out to repair replicas in {@link ReadRepair#repairPartition} */ public void awaitWrites(); - static ReadRepair create(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) - { - return command.metadata().params.readRepair.create(command, queryStartNanoTime, consistency); - } + /** + * Repairs a partition _after_ receiving data responses. This method receives replica list, since + * we will block repair only on the replicas that have responded. + */ + void repairPartition(DecoratedKey partitionKey, Map mutations, L replicaLayout); } diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java index 1117822c55..6eff3955dc 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairDiagnostics.java @@ -18,6 +18,7 @@ package org.apache.cassandra.service.reads.repair; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -38,8 +39,8 @@ final class ReadRepairDiagnostics { } - static void startRepair(AbstractReadRepair readRepair, List endpointDestinations, - DigestResolver digestResolver, List allEndpoints) + static void startRepair(AbstractReadRepair readRepair, Collection endpointDestinations, + DigestResolver digestResolver, Collection allEndpoints) { if (service.isEnabled(ReadRepairEvent.class, ReadRepairEventType.START_REPAIR)) service.publish(new ReadRepairEvent(ReadRepairEventType.START_REPAIR, diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java index 152f7e62bd..9e1436255f 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairEvent.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service.reads.repair; import java.io.Serializable; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -48,9 +49,9 @@ final class ReadRepairEvent extends DiagnosticEvent private final ConsistencyLevel consistency; private final SpeculativeRetryPolicy.Kind speculativeRetry; @VisibleForTesting - final List destinations; + final Collection destinations; @VisibleForTesting - final List allEndpoints; + final Collection allEndpoints; @Nullable private final DigestResolverDebugResult[] digestsByEndpoint; @@ -60,13 +61,13 @@ final class ReadRepairEvent extends DiagnosticEvent SPECULATED_READ } - ReadRepairEvent(ReadRepairEventType type, AbstractReadRepair readRepair, List destinations, - List allEndpoints, DigestResolver digestResolver) + ReadRepairEvent(ReadRepairEventType type, AbstractReadRepair readRepair, Collection destinations, + Collection allEndpoints, DigestResolver digestResolver) { this.keyspace = readRepair.cfs.keyspace; this.tableName = readRepair.cfs.getTableName(); this.cqlCommand = readRepair.command.toCQLString(); - this.consistency = readRepair.consistency; + this.consistency = readRepair.replicaLayout.consistencyLevel(); this.speculativeRetry = readRepair.cfs.metadata().params.speculativeRetry.kind(); this.destinations = destinations; this.allEndpoints = allEndpoints; diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java index 594563358d..28c0e9e669 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java @@ -18,26 +18,25 @@ package org.apache.cassandra.service.reads.repair; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.ReplicaLayout; public enum ReadRepairStrategy implements ReadRepair.Factory { NONE { - @Override - public ReadRepair create(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public , L extends ReplicaLayout> ReadRepair create(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - return new ReadOnlyReadRepair(command, queryStartNanoTime, consistency); + return new ReadOnlyReadRepair<>(command, replicaLayout, queryStartNanoTime); } }, BLOCKING { - @Override - public ReadRepair create(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public , L extends ReplicaLayout> ReadRepair create(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - return new BlockingReadRepair(command, queryStartNanoTime, consistency); + return new BlockingReadRepair<>(command, replicaLayout, queryStartNanoTime); } }; diff --git a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java index cb6707d8e1..b0c019a3a9 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java +++ b/src/java/org/apache/cassandra/service/reads/repair/RowIteratorMergeListener.java @@ -21,6 +21,7 @@ package org.apache.cassandra.service.reads.repair; import java.util.Arrays; import java.util.Map; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.cassandra.db.Clustering; @@ -43,7 +44,9 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowDiffListener; import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.db.rows.UnfilteredRowIterators; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.schema.ColumnMetadata; public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeListener @@ -51,14 +54,14 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis private final DecoratedKey partitionKey; private final RegularAndStaticColumns columns; private final boolean isReversed; - private final InetAddressAndPort[] sources; private final ReadCommand command; private final ConsistencyLevel consistency; private final PartitionUpdate.Builder[] repairs; - + private final Replica[] sources; private final Row.Builder[] currentRows; private final RowDiffListener diffListener; + private final ReplicaLayout layout; // The partition level deletion for the merge row. private DeletionTime partitionLevelDeletion; @@ -71,16 +74,21 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis private final ReadRepair readRepair; - public RowIteratorMergeListener(DecoratedKey partitionKey, RegularAndStaticColumns columns, boolean isReversed, InetAddressAndPort[] sources, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair) + public RowIteratorMergeListener(DecoratedKey partitionKey, RegularAndStaticColumns columns, boolean isReversed, ReplicaLayout layout, ReadCommand command, ConsistencyLevel consistency, ReadRepair readRepair) { this.partitionKey = partitionKey; this.columns = columns; this.isReversed = isReversed; - this.sources = sources; - repairs = new PartitionUpdate.Builder[sources.length]; - currentRows = new Row.Builder[sources.length]; - sourceDeletionTime = new DeletionTime[sources.length]; - markerToRepair = new ClusteringBound[sources.length]; + Endpoints sources = layout.selected(); + this.sources = new Replica[sources.size()]; + for (int i = 0; i < sources.size(); i++) + this.sources[i] = sources.get(i); + + this.layout = layout; + repairs = new PartitionUpdate.Builder[sources.size()]; + currentRows = new Row.Builder[sources.size()]; + sourceDeletionTime = new DeletionTime[sources.size()]; + markerToRepair = new ClusteringBound[sources.size()]; this.command = command; this.consistency = consistency; this.readRepair = readRepair; @@ -89,25 +97,25 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis { public void onPrimaryKeyLivenessInfo(int i, Clustering clustering, LivenessInfo merged, LivenessInfo original) { - if (merged != null && !merged.equals(original)) + if (merged != null && !merged.equals(original) && !isTransient(i)) currentRow(i, clustering).addPrimaryKeyLivenessInfo(merged); } public void onDeletion(int i, Clustering clustering, Row.Deletion merged, Row.Deletion original) { - if (merged != null && !merged.equals(original)) + if (merged != null && !merged.equals(original) && !isTransient(i)) currentRow(i, clustering).addRowDeletion(merged); } public void onComplexDeletion(int i, Clustering clustering, ColumnMetadata column, DeletionTime merged, DeletionTime original) { - if (merged != null && !merged.equals(original)) + if (merged != null && !merged.equals(original) && !isTransient(i)) currentRow(i, clustering).addComplexDeletion(column, merged); } public void onCell(int i, Clustering clustering, Cell merged, Cell original) { - if (merged != null && !merged.equals(original) && isQueried(merged)) + if (merged != null && !merged.equals(original) && isQueried(merged) && !isTransient(i)) currentRow(i, clustering).addCell(merged); } @@ -126,6 +134,11 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis }; } + private boolean isTransient(int i) + { + return sources[i].isTransient(); + } + private PartitionUpdate.Builder update(int i) { if (repairs[i] == null) @@ -159,6 +172,9 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis this.partitionLevelDeletion = mergedDeletion; for (int i = 0; i < versions.length; i++) { + if (isTransient(i)) + continue; + if (mergedDeletion.supersedes(versions[i])) update(i).addPartitionDeletion(mergedDeletion); } @@ -193,6 +209,9 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis for (int i = 0; i < versions.length; i++) { + if (isTransient(i)) + continue; + RangeTombstoneMarker marker = versions[i]; // Update what the source now thinks is the current deletion @@ -245,12 +264,12 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis if (!marker.isBoundary() && marker.isOpen(isReversed)) // (1) { assert currentDeletion.equals(marker.openDeletionTime(isReversed)) - : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata())); + : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata())); } else // (2) { assert marker.isClose(isReversed) && currentDeletion.equals(marker.closeDeletionTime(isReversed)) - : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata())); + : String.format("currentDeletion=%s, marker=%s", currentDeletion, marker.toString(command.metadata())); } // and so unless it's a boundary whose opening deletion time is still equal to the current @@ -306,13 +325,14 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis public void close() { - Map mutations = null; + Map mutations = null; for (int i = 0; i < repairs.length; i++) { if (repairs[i] == null) continue; - Mutation mutation = BlockingReadRepairs.createRepairMutation(repairs[i].build(), consistency, sources[i], false); + Preconditions.checkState(!isTransient(i), "cannot read repair transient replicas"); + Mutation mutation = BlockingReadRepairs.createRepairMutation(repairs[i].build(), consistency, sources[i].endpoint(), false); if (mutation == null) continue; @@ -324,7 +344,7 @@ public class RowIteratorMergeListener implements UnfilteredRowIterators.MergeLis if (mutations != null) { - readRepair.repairPartition(partitionKey, mutations, sources); + readRepair.repairPartition(partitionKey, mutations, layout); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java b/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java index 609d2a0e58..38c25dcb86 100644 --- a/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java +++ b/src/java/org/apache/cassandra/streaming/DefaultConnectionFactory.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,8 +44,10 @@ public class DefaultConnectionFactory implements StreamConnectionFactory private static final int DEFAULT_CHANNEL_BUFFER_SIZE = 1 << 22; - private static final long MAX_WAIT_TIME_NANOS = TimeUnit.SECONDS.toNanos(30); - private static final int MAX_CONNECT_ATTEMPTS = 3; + @VisibleForTesting + public static long MAX_WAIT_TIME_NANOS = TimeUnit.SECONDS.toNanos(30); + @VisibleForTesting + public static int MAX_CONNECT_ATTEMPTS = 3; @Override public Channel createConnection(OutboundConnectionIdentifier connectionId, int protocolVersion) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java index b56f165c22..2f6deb5392 100644 --- a/src/java/org/apache/cassandra/streaming/StreamPlan.java +++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java @@ -19,11 +19,14 @@ package org.apache.cassandra.streaming; import java.util.*; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.utils.UUIDGen; +import static com.google.common.collect.Iterables.all; import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; /** @@ -69,12 +72,13 @@ public class StreamPlan * * @param from endpoint address to fetch data from. * @param keyspace name of keyspace - * @param ranges ranges to fetch + * @param fullRanges ranges to fetch that from provides the full version of + * @param transientRanges ranges to fetch that from provides only transient data of * @return this object for chaining */ - public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, Collection> ranges) + public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges) { - return requestRanges(from, keyspace, ranges, EMPTY_COLUMN_FAMILIES); + return requestRanges(from, keyspace, fullRanges, transientRanges, EMPTY_COLUMN_FAMILIES); } /** @@ -82,14 +86,20 @@ public class StreamPlan * * @param from endpoint address to fetch data from. * @param keyspace name of keyspace - * @param ranges ranges to fetch + * @param fullRanges ranges to fetch that from provides the full data for + * @param transientRanges ranges to fetch that from provides only transient data for * @param columnFamilies specific column families * @return this object for chaining */ - public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, Collection> ranges, String... columnFamilies) + public StreamPlan requestRanges(InetAddressAndPort from, String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges, String... columnFamilies) { + //It should either be a dummy address for repair or if it's a bootstrap/move/rebuild it should be this node + assert all(fullRanges, Replica::isLocal) || all(fullRanges, range -> range.endpoint().getHostAddress(true).equals("0.0.0.0:0")) : + fullRanges.toString(); + assert all(transientRanges, Replica::isLocal) || all(transientRanges, range -> range.endpoint().getHostAddress(true).equals("0.0.0.0:0")) : + transientRanges.toString(); StreamSession session = coordinator.getOrCreateNextSession(from); - session.addStreamRequest(keyspace, ranges, Arrays.asList(columnFamilies)); + session.addStreamRequest(keyspace, fullRanges, transientRanges, Arrays.asList(columnFamilies)); return this; } @@ -98,14 +108,14 @@ public class StreamPlan * * @param to endpoint address of receiver * @param keyspace name of keyspace - * @param ranges ranges to send + * @param replicas ranges to send * @param columnFamilies specific column families * @return this object for chaining */ - public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, Collection> ranges, String... columnFamilies) + public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, RangesAtEndpoint replicas, String... columnFamilies) { StreamSession session = coordinator.getOrCreateNextSession(to); - session.addTransferRanges(keyspace, ranges, Arrays.asList(columnFamilies), flushBeforeTransfer); + session.addTransferRanges(keyspace, replicas, Arrays.asList(columnFamilies), flushBeforeTransfer); return this; } @@ -182,4 +192,10 @@ public class StreamPlan { return flushBeforeTransfer; } + + @VisibleForTesting + public StreamCoordinator getCoordinator() + { + return coordinator; + } } diff --git a/src/java/org/apache/cassandra/streaming/StreamRequest.java b/src/java/org/apache/cassandra/streaming/StreamRequest.java index 4a3761e75d..f37268f200 100644 --- a/src/java/org/apache/cassandra/streaming/StreamRequest.java +++ b/src/java/org/apache/cassandra/streaming/StreamRequest.java @@ -29,6 +29,10 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.net.CompactEndpointSerializationHelper; import org.apache.cassandra.net.MessagingService; public class StreamRequest @@ -36,12 +40,23 @@ public class StreamRequest public static final IVersionedSerializer serializer = new StreamRequestSerializer(); public final String keyspace; - public final Collection> ranges; + //Full replicas and transient replicas are split based on the transient status of the remote we are fetching + //from. We preserve this distinction so on completion we can log to a system table whether we got the data transiently + //or fully from some remote. This is an important distinction for resumable bootstrap. The Replicas in these collections + //are local replicas (or dummy if this is triggered by repair) and don't encode the necessary information about + //what the remote provided. + public final RangesAtEndpoint full; + public final RangesAtEndpoint transientReplicas; public final Collection columnFamilies = new HashSet<>(); - public StreamRequest(String keyspace, Collection> ranges, Collection columnFamilies) + + public StreamRequest(String keyspace, RangesAtEndpoint full, RangesAtEndpoint transientReplicas, Collection columnFamilies) { this.keyspace = keyspace; - this.ranges = ranges; + if (!full.endpoint().equals(transientReplicas.endpoint())) + throw new IllegalStateException("Mismatching endpoints: " + full + ", " + transientReplicas); + + this.full = full; + this.transientReplicas = transientReplicas; this.columnFamilies.addAll(columnFamilies); } @@ -50,49 +65,82 @@ public class StreamRequest public void serialize(StreamRequest request, DataOutputPlus out, int version) throws IOException { out.writeUTF(request.keyspace); - out.writeInt(request.ranges.size()); - for (Range range : request.ranges) - { - MessagingService.validatePartitioner(range); - Token.serializer.serialize(range.left, out, version); - Token.serializer.serialize(range.right, out, version); - } out.writeInt(request.columnFamilies.size()); + + CompactEndpointSerializationHelper.streamingInstance.serialize(request.full.endpoint(), out, version); + serializeReplicas(request.full, out, version); + serializeReplicas(request.transientReplicas, out, version); for (String cf : request.columnFamilies) out.writeUTF(cf); } + private void serializeReplicas(RangesAtEndpoint replicas, DataOutputPlus out, int version) throws IOException + { + out.writeInt(replicas.size()); + + for (Replica replica : replicas) + { + MessagingService.validatePartitioner(replica.range()); + Token.serializer.serialize(replica.range().left, out, version); + Token.serializer.serialize(replica.range().right, out, version); + } + } + public StreamRequest deserialize(DataInputPlus in, int version) throws IOException { String keyspace = in.readUTF(); - int rangeCount = in.readInt(); - List> ranges = new ArrayList<>(rangeCount); - for (int i = 0; i < rangeCount; i++) - { - Token left = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version); - Token right = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version); - ranges.add(new Range<>(left, right)); - } int cfCount = in.readInt(); + InetAddressAndPort endpoint = CompactEndpointSerializationHelper.streamingInstance.deserialize(in, version); + + RangesAtEndpoint full = deserializeReplicas(in, version, endpoint, true); + RangesAtEndpoint transientReplicas = deserializeReplicas(in, version, endpoint, false); List columnFamilies = new ArrayList<>(cfCount); for (int i = 0; i < cfCount; i++) columnFamilies.add(in.readUTF()); - return new StreamRequest(keyspace, ranges, columnFamilies); + return new StreamRequest(keyspace, full, transientReplicas, columnFamilies); + } + + RangesAtEndpoint deserializeReplicas(DataInputPlus in, int version, InetAddressAndPort endpoint, boolean isFull) throws IOException + { + int replicaCount = in.readInt(); + + RangesAtEndpoint.Builder replicas = RangesAtEndpoint.builder(endpoint, replicaCount); + for (int i = 0; i < replicaCount; i++) + { + //TODO, super need to review the usage of streaming vs not streaming endpoint serialization helper + //to make sure I'm not using the wrong one some of the time, like do repair messages use the + //streaming version? + Token left = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version); + Token right = Token.serializer.deserialize(in, MessagingService.globalPartitioner(), version); + replicas.add(new Replica(endpoint, new Range<>(left, right), isFull)); + } + return replicas.build(); } public long serializedSize(StreamRequest request, int version) { int size = TypeSizes.sizeof(request.keyspace); - size += TypeSizes.sizeof(request.ranges.size()); - for (Range range : request.ranges) - { - size += Token.serializer.serializedSize(range.left, version); - size += Token.serializer.serializedSize(range.right, version); - } size += TypeSizes.sizeof(request.columnFamilies.size()); + size += CompactEndpointSerializationHelper.streamingInstance.serializedSize(request.full.endpoint(), version); + size += replicasSerializedSize(request.transientReplicas, version); + size += replicasSerializedSize(request.full, version); for (String cf : request.columnFamilies) size += TypeSizes.sizeof(cf); return size; } + + private long replicasSerializedSize(RangesAtEndpoint replicas, int version) + { + long size = 0; + size += TypeSizes.sizeof(replicas.size()); + + for (Replica replica : replicas) + { + size += Token.serializer.serializedSize(replica.range().left, version); + size += Token.serializer.serializedSize(replica.range().right, version); + } + return size; + } + } } diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index 393cd24389..ec80772c86 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -29,6 +29,7 @@ import com.google.common.util.concurrent.Futures; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +41,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.*; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.StreamingMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.async.OutboundConnectionIdentifier; @@ -49,6 +51,8 @@ import org.apache.cassandra.streaming.messages.*; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; +import static com.google.common.collect.Iterables.all; + /** * Handles the streaming a one or more streams to and from a specific remote node. * @@ -243,7 +247,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber public StreamReceiver getAggregator(TableId tableId) { - assert receivers.containsKey(tableId); + assert receivers.containsKey(tableId) : "Missing tableId " + tableId; return receivers.get(tableId).getReceiver(); } @@ -297,38 +301,52 @@ public class StreamSession implements IEndpointStateChangeSubscriber * Request data fetch task to this session. * * @param keyspace Requesting keyspace - * @param ranges Ranges to retrieve data + * @param fullRanges Ranges to retrieve data that will return full data from the source + * @param transientRanges Ranges to retrieve data that will return transient data from the source * @param columnFamilies ColumnFamily names. Can be empty if requesting all CF under the keyspace. */ - public void addStreamRequest(String keyspace, Collection> ranges, Collection columnFamilies) + public void addStreamRequest(String keyspace, RangesAtEndpoint fullRanges, RangesAtEndpoint transientRanges, Collection columnFamilies) { - requests.add(new StreamRequest(keyspace, ranges, columnFamilies)); + //It should either be a dummy address for repair or if it's a bootstrap/move/rebuild it should be this node + assert all(fullRanges, Replica::isLocal) || all(fullRanges, range -> range.endpoint().getHostAddress(true).equals("0.0.0.0:0")) : fullRanges.toString(); + assert all(transientRanges, Replica::isLocal) || all(transientRanges, range -> range.endpoint().getHostAddress(true).equals("0.0.0.0:0")) : transientRanges.toString(); + requests.add(new StreamRequest(keyspace, fullRanges, transientRanges, columnFamilies)); } /** * Set up transfer for specific keyspace/ranges/CFs * * @param keyspace Transfer keyspace - * @param ranges Transfer ranges + * @param replicas Transfer ranges * @param columnFamilies Transfer ColumnFamilies * @param flushTables flush tables? */ - synchronized void addTransferRanges(String keyspace, Collection> ranges, Collection columnFamilies, boolean flushTables) + synchronized void addTransferRanges(String keyspace, RangesAtEndpoint replicas, Collection columnFamilies, boolean flushTables) { failIfFinished(); Collection stores = getColumnFamilyStores(keyspace, columnFamilies); if (flushTables) flushSSTables(stores); - List> normalizedRanges = Range.normalize(ranges); - List streams = getOutgoingStreamsForRanges(normalizedRanges, stores, pendingRepair, previewKind); + //Was it safe to remove this normalize, sorting seems not to matter, merging? Maybe we should have? + //Do we need to unwrap here also or is that just making it worse? + //Range and if it's transient + RangesAtEndpoint.Builder unwrappedRanges = RangesAtEndpoint.builder(replicas.endpoint(), replicas.size()); + for (Replica replica : replicas) + { + for (Range unwrapped : replica.range().unwrap()) + { + unwrappedRanges.add(new Replica(replica.endpoint(), unwrapped, replica.isFull())); + } + } + List streams = getOutgoingStreamsForRanges(unwrappedRanges.build(), stores, pendingRepair, previewKind); addTransferStreams(streams); Set> toBeUpdated = transferredRangesPerKeyspace.get(keyspace); if (toBeUpdated == null) { toBeUpdated = new HashSet<>(); } - toBeUpdated.addAll(ranges); + toBeUpdated.addAll(replicas.ranges()); transferredRangesPerKeyspace.put(keyspace, toBeUpdated); } @@ -355,14 +373,14 @@ public class StreamSession implements IEndpointStateChangeSubscriber } @VisibleForTesting - public List getOutgoingStreamsForRanges(Collection> ranges, Collection stores, UUID pendingRepair, PreviewKind previewKind) + public List getOutgoingStreamsForRanges(RangesAtEndpoint replicas, Collection stores, UUID pendingRepair, PreviewKind previewKind) { List streams = new ArrayList<>(); try { for (ColumnFamilyStore cfs: stores) { - streams.addAll(cfs.getStreamManager().createOutgoingStreams(this, ranges, pendingRepair, previewKind)); + streams.addAll(cfs.getStreamManager().createOutgoingStreams(this, replicas, pendingRepair, previewKind)); } } catch (Throwable t) @@ -561,7 +579,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber { for (StreamRequest request : requests) - addTransferRanges(request.keyspace, request.ranges, request.columnFamilies, true); // always flush on stream request + addTransferRanges(request.keyspace, RangesAtEndpoint.concat(request.full, request.transientReplicas), request.columnFamilies, true); // always flush on stream request for (StreamSummary summary : summaries) prepareReceiving(summary); @@ -812,4 +830,16 @@ public class StreamSession implements IEndpointStateChangeSubscriber } maybeCompleted(); } + + @VisibleForTesting + public int getNumRequests() + { + return requests.size(); + } + + @VisibleForTesting + public int getNumTransfers() + { + return transferredRangesPerKeyspace.size(); + } } diff --git a/src/java/org/apache/cassandra/streaming/TableStreamManager.java b/src/java/org/apache/cassandra/streaming/TableStreamManager.java index 11512e9b8e..d97fabc16a 100644 --- a/src/java/org/apache/cassandra/streaming/TableStreamManager.java +++ b/src/java/org/apache/cassandra/streaming/TableStreamManager.java @@ -21,8 +21,7 @@ package org.apache.cassandra.streaming; import java.util.Collection; import java.util.UUID; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.streaming.messages.StreamMessageHeader; /** @@ -46,12 +45,12 @@ public interface TableStreamManager /** * Returns a collection of {@link OutgoingStream}s that contains the data selected by the - * given ranges, pendingRepair, and preview. + * given replicas, pendingRepair, and preview. * * There aren't any requirements on how data is divided between the outgoing streams */ Collection createOutgoingStreams(StreamSession session, - Collection> ranges, + RangesAtEndpoint replicas, UUID pendingRepair, PreviewKind previewKind); } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 03b8af0822..54187d10b2 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -816,6 +816,11 @@ public class NodeProbe implements AutoCloseable return ssProxy.getNaturalEndpoints(keyspace, cf, key); } + public List getReplicas(String keyspace, String cf, String key) + { + return ssProxy.getReplicas(keyspace, cf, key); + } + public List getSSTables(String keyspace, String cf, String key, boolean hexFormat) { ColumnFamilyStoreMBean cfsProxy = getCfsProxy(keyspace, cf); diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index c2193d4a2a..1d09b0f7d4 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -190,6 +190,8 @@ public class NodeTool ReloadSslCertificates.class, EnableAuditLog.class, DisableAuditLog.class, + GetReplicas.class, + DisableAuditLog.class, EnableOldProtocolVersions.class, DisableOldProtocolVersions.class ); diff --git a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java index 8056ff8cf3..31d80facc6 100644 --- a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java +++ b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java @@ -88,11 +88,11 @@ public class SSTableRepairedAtSetter if (setIsRepaired) { FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath()); - descriptor.getMetadataSerializer().mutateRepaired(descriptor, f.toMillis(), null); + descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null, false); } else { - descriptor.getMetadataSerializer().mutateRepaired(descriptor, 0, null); + descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, 0, null, false); } } } diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetReplicas.java b/src/java/org/apache/cassandra/tools/nodetool/GetReplicas.java new file mode 100644 index 0000000000..4c401fc4e5 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/nodetool/GetReplicas.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools.nodetool; + +import java.util.ArrayList; +import java.util.List; + +import io.airlift.airline.Arguments; +import io.airlift.airline.Command; +import org.apache.cassandra.tools.NodeProbe; +import org.apache.cassandra.tools.NodeTool; + +import static com.google.common.base.Preconditions.checkArgument; + +@Command(name = "getreplicas", description = "Print replicas for a given key") +public class GetReplicas extends NodeTool.NodeToolCmd +{ + @Arguments(usage = " ", description = "The keyspace, the table, and the partition key for which we need to find replicas") + private List args = new ArrayList<>(); + + @Override + public void execute(NodeProbe probe) + { + checkArgument(args.size() == 3, "getreplicas requires keyspace, table and partition key arguments"); + String ks = args.get(0); + String table = args.get(1); + String key = args.get(2); + + System.out.println(probe.getReplicas(ks, table, key)); + } +} diff --git a/src/java/org/apache/cassandra/tracing/TraceState.java b/src/java/org/apache/cassandra/tracing/TraceState.java index a53846cd7a..1e0813cb11 100644 --- a/src/java/org/apache/cassandra/tracing/TraceState.java +++ b/src/java/org/apache/cassandra/tracing/TraceState.java @@ -161,7 +161,7 @@ public abstract class TraceState implements ProgressEventNotifier trace(MessageFormatter.format(format, arg1, arg2).getMessage()); } - public void trace(String format, Object[] args) + public void trace(String format, Object... args) { trace(MessageFormatter.arrayFormat(format, args).getMessage()); } diff --git a/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java b/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java index 84af41c6d3..8e0b19ff64 100644 --- a/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java +++ b/src/java/org/apache/cassandra/transport/messages/ErrorMessage.java @@ -68,7 +68,7 @@ public class ErrorMessage extends Message.Response ConsistencyLevel cl = CBUtil.readConsistencyLevel(body); int required = body.readInt(); int alive = body.readInt(); - te = new UnavailableException(cl, required, alive); + te = UnavailableException.create(cl, required, alive); } break; case OVERLOADED: diff --git a/src/java/org/apache/cassandra/utils/Pair.java b/src/java/org/apache/cassandra/utils/Pair.java index ea8b8fc03f..cb0952947f 100644 --- a/src/java/org/apache/cassandra/utils/Pair.java +++ b/src/java/org/apache/cassandra/utils/Pair.java @@ -53,6 +53,18 @@ public class Pair return "(" + left + "," + right + ")"; } + //For functional interfaces + public T1 left() + { + return left; + } + + //For functional interfaces + public T2 right() + { + return right; + } + public static Pair create(X x, Y y) { return new Pair(x, y); diff --git a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java index e80faca806..0c097a6504 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java +++ b/src/java/org/apache/cassandra/utils/concurrent/Accumulator.java @@ -18,6 +18,8 @@ */ package org.apache.cassandra.utils.concurrent; +import java.util.AbstractCollection; +import java.util.Collection; import java.util.Iterator; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; @@ -27,7 +29,7 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; * * @param */ -public class Accumulator implements Iterable +public class Accumulator { private volatile int nextIndex; private volatile int presentCount; @@ -105,7 +107,7 @@ public class Accumulator implements Iterable return values.length; } - public Iterator iterator() + private Iterator iterator(int count) { return new Iterator() { @@ -113,7 +115,7 @@ public class Accumulator implements Iterable public boolean hasNext() { - return p < presentCount; + return p < count; } public E next() @@ -135,4 +137,23 @@ public class Accumulator implements Iterable throw new IndexOutOfBoundsException(); return (E) values[i]; } + + public Collection snapshot() + { + int count = presentCount; + return new AbstractCollection() + { + @Override + public Iterator iterator() + { + return Accumulator.this.iterator(count); + } + + @Override + public int size() + { + return count; + } + }; + } } diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db index ceaa5a3b35be388650d827072b29e64465f0769a..8fad34fe9e11d06e2af4ba6060fefbc095229905 100644 GIT binary patch delta 58 lcmWF!pP*#g&%nUI2E<5!`7(sVz8OMu*FtFiNC+)#2mrG81n&R< delta 58 lcmWF!pP*zqm4ShQ4TzBd^92ZpeLIBao(-Y-8zHoC3;?&s1!n*N diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db index 69687207681ea7af8d7dbf24f6ac0c74944d9cd7..ae35335fbf9aa78f860a9433538853f387fa6b93 100644 GIT binary patch literal 5214 zcmd6r{Zmxe8OQGe3$kjirosBs_JG}HsFU$7%L1!a?zOS0OfZshGBINlvn&Lb7k8I; zsIse#2_dbe=0!}#MkR2yj@2b&6O#b@LR&Ctiyg{U`Khix6Rj4+g@H<>)jcwXb;rX*VTJ`HNpChs$f;1A<)+BX??D|yrHS3 zs@msoZm6uPEN^J8sI6}H?ktCis#+^N9!PI6Z@m?h>hpK)TNiySTe3C^k-P>QO*DNV z%^!YopY-BHF_;JQMBTMG`os;%+PuX17b~A!RVduGPl`Sr1vAMnq@D+10*ho7Jjc=3 z^#-T;qS4oeo&{#l;3o7eG<&u$Mb9F$XLuSt-Dc0O_rSJwTU=~ zsj+{H6n!-SizE?IK(D?nu)bG1Z7sa&APYxwJ8iI7XNc}WA2-YkZ!f}WHHVW^ta|4R zt3C*`YNVP~zjw3hPxDxHX%dsnPkw@Y()mmii74Nt8Sa1#-`v(|GcvgTEF1f82dWXC zI&*i6$({Fq20wv1*MJzyJQCJpK`wmB2p$k(Sz)5B9R`<4jIR*;X6G};OhPONrAwdK zemly;MvmC^jqDrdS%=(D=~fyM4~;S37GsVGO636>4+l_+xegf)vf$pss&zg_k&U|= z^-sydrW_-0St!j^f8phNV)FlrVMEvJhT>nh9R|bq48w0)(69y#*UNxaN8aB$Zpj>A zd|JcTk!RyxQ#yf&;^FYzz&GNy^zy&LvhrW#uQ>WhnP?3hvv8KY5E!uv?Vv@dKMj5O z^t6LOG#}e~`7vt%M@&%1Iww^A+tLp*vr4fiVSShqucEH%SB-1e<1bp z*R%vm@moi4(*iJ}6@Xa(6jK0XVf%5qIH&3$3n%Cy4O9GO;nW$NRU3=+GK!r2$lC;! zLK{3+jfXI)$j|~`kQ??AeK_W3hK1keFf$3`aAJXfZKjWJ(hSuCAPYNsXomXm#V8X? zCdG}g=~2qROs+)v7A*&1sx8Xv841^(Wx|B%O|1HdgH<<{vg#Ix$+@)TW~OO!D)zGA zZKug;!{nV0flTR&oGsg=R+&o>o$1IAU}qJ+uo%SfB#EJno~%cA>88m$9siQoL#N z#vr$TBd>OQSy&Yb8H@^T6YvdQLw(mdRN65QA7xcnfK@-;#;P8PRj=l;`S_ zGE_0UjS_tyD3xuQ z6yDtmy&BMdx(u(?jfc=hc|n@u_uLrfR&#wLS6`T!r?=m1YDF7uI>`(CV5H1!Qg%w4aug85$?70 z7ODb1AjWTgPM4&&-E}!QqssVzn2?~*8HeuS8?5@_B&!~cv1-c+R&D(StDbB#DLBYX zFW>HAAcy?^bHPmcjuJN5>p-PaJ|HBjhVUw{m87cW>Pvsms@|Pia3uzzh>k}{^d-9bcqQ| zH{x9s;|TAhPk%q+ow1~Mv~CbqX-7czc0BYj0Unt5fI{v z8!2G-zKq6#S*~N09-9cMQ?Ng#{1~85^gxsbI$(T}GNgC%#6?OG!}+Ajk=DqMo-t#7 zWfRV)wel&7k#>xyoH(YbYs*+QZK7xz)?;}7{c~3RYl2l@yl0~Pi}xAIWud><4EfAa zHukHRP-(4Pp%|&z4G5#D7|uqFr9i9Se7*v&eqt8CI)HE80||05nPaWc6r%$ z*-Z=y&8V?4X^1h2(YhC#R)xf9CkiVpunW71=}bqGv8}c3gmjuI?KqX0MsA{feRl;D1HP5uWY0TJ!P?Y=!nB$6AX3Bg(6a zFVHOtX@Nj(ZGA)M?!6tZ+KN3LO%=7S`gTucXM4l0UAx;oEzO>$3U75qRb72+MMJxL zPvf4w&6V}F-d%0=O=RCEy;&qq?*tG*ad?douGsF^lS|{DGcsXXw)qxTOL+Q@#oT1gDu^ zHeErn?TDa>O-?h(nMG-#Aw}mp7b@+^$}KafEJ{}HJcLR|l2U5#vbBl*Ycc!YMb{Q% z#Ahx9EN&@(kU39oab7;y1dEjVDM8hVov^<5&$9KYnjpP=Bo|_fe)bkl2!kRyc^w^z z7_#)+kl3ahm%282wD7WYt%PvaP0k9^;d`9UEG-I{4@8WhK{EUh3`zxwXTa#=QvnO4 zsjS8{gEUl{P4OcX45w0Cj8yvJO=_X(NLLTPfRTfP2aHiNa_XV0ZZcq~FQsuykgjy~Kw2~_5jo`Z zp;^Mrlaz)IbxB|s>-N!~Z3258+J7%&v_6-b^@RWop8;%DfGxri1vfbj$+LF`y!{WG zj*GWUZ;S6?VR4wKnVgAGbJKB?V9N7@3#*W@#VlDrTZjU2<3eQ{5G{`Uucy@9w3G4A zK^=Y^m_q*TsmeCdWx_cO>+q6Xw)yg$jO&wIXt)Z%hMm4*u^!kIun;$H7O7y?q|ZkL z1KZJSn=+C>@BVc37S7YYeipsUQAkVy-ou|?ahjn3 zsN7KJs5xN6&B6GWN#=Dw$H^!p)g}fQ`9V6G9pLsL9kr!lXl?V4J~Foz_MjR$NJe)K zmBK60hg$|KdGk5k-p1%Fyh%r2>kYu3RFRa)+b&k5RQgHy>qDXrtV~Wh{1&4Y$Wp)G zW}r_-FLpg;h^6q*TF#1LQu<>)yMPrbk+Ce@>gv(Qf@Eyb3j?q`txAxwObnzEN5&p- z5_pX&`Xu~KzeO)FbS!Vi56dyco0Qf3+#Pc+Fg5bWT$j%CqX9Omy~nW^=?;GMgtFQo zUnT=np&VlggUVL?wkr7H2Ysy`QRhCKKerr4k^48xupwI;e)PmGhPmFOttURJ6L@|H=^y+2E9oD9WMhK<7t%l|UW7Re{;}KXN$^uTUOgnXN%`SaEvgW(&~e`^ zup$*~2Feq&5%L4Qh^XyZJppvS#%L}~Qc8WBiJK1O*^{LgJ1}Tbl9XHpqCMhL3QI)7 z$N%Nzh^NkoB_Qdfodi1*(b9oWi(rq^L5$o#lmluF`AA+Lk9?$>fch=}hK^l1xm^OCm>yPd;=#t4>X-VyVe0hLx_c94i@soc&d7xSF^%(6w za{82!wk*Am1Zk8@gSTJeA2n<>(vr!9zy&7pY0i@Vi0e@ymdO{nEJcm@@s*c4co1iy zkU~+?nUALUpqv43)_)*XGuWI`fnP~~|mtr$tBbF>w z${G3517-8bpSXW>7E+qO9&%KZLii`X%{VRiFtGxRfS|9n!fujm3yd_-`(4~|QsF%9 z<_qYx&(m&i!u_~}LSl&e#O_?Qb@V4%BWUZ*eqx_1x!F$~hNdMZdOhr`qccXcm&uzU zoUD;QF|ZL$wHcDj)4cV`cHWwv<*hFRh83ifX%5aLllKS@8hvqc)gio%Xg!rq7OhC? z^U>e%U37B&6~50y>_?AbZ1s#!CbzM^uQkfaD&C}%&3ySGlfIk9-(Ng`Q6SIh|%1B|?_|!G_g0 zF3nwv=}!%kn8o*o5g6=@68~dXK``?v}a_zWY343d0I|Fk^DDs4p6-&^X`lwq{cd*dW!Ls- z4v9IUIq(IvLqDalb8tA$%>Pus^W%~K@-byO^qS<$c@(jof8qbw3C{T!{y(h79c`hI zGXfsF#?VrMr_R{*B@}beiI4p)f*V&SJ~n-pCmtgFvzH;Ip6{iNn|43|wcX2P=4pnM z+V5p)cIgw%!eFU(9+1hG4g-uJjSJbBpVt55sWVCDy@!&_Ra^xLlK8jgT*jjzmi-D3 zKZ&nr7m>snUnW0JH)<1ze~`-hMNI|J(h+e(vH+|TgM zz%KU@d_TBL7rvWV~xFtwf#O*r>15_Aof$>7m5HUN2(7d&H$Vmy}x8< cfG6+siiz8M7CBx{&2?q?qn>He@1D8we^v?zl>h($ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 index f1c192bebe..8a92f3c583 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 @@ -1 +1 @@ -4004129384 \ No newline at end of file +2977407251 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db index af16195ea0be93fdb23534655ecee71229bdd460..d50fdeb4e209041c5469b14cf84433c739f95a91 100644 GIT binary patch delta 116 zcmex(j`QO=&JBi6jLh2;EEw}S8CkbqaAYjxWMMscjd`=)1)vD?=6N@ZK$PHPpg8lv zYuuaj3v)PGfB!xBntAerDl5k7?Fnxf1*_Q@7>y3`GQc^;hlN|=9FrqbTo4Wb$5$z1 delta 109 zcmex(j`QO=&JBi6+v6=53pg3swqI~$ECkU7j!Z?IjI5jI-2_vDkAb4B+c!P}s$yX} o_?mI@gDNY=#_b7j7zL}@7#NKX@v^}=#)pNw;T)4AQUVYT05R<(yZ`_I diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db index 970e3851b31eed4f38584731795342a38c3a251b..734186497e79a96bedaf7886b935bc46979e17b5 100644 GIT binary patch delta 131 zcmdmPzQcTiI;*C5N#v)EI^2SWsaCoLfec(PHZZUjXFDDj{s5*=dkNM5|NkF^8KA(v ze-ZQlFFq!lGX$q`8bP!%IGhE_G6E42AIb(9$*^$Y*XBRL3X|VV{xDfzx`k!&o3aT2 D!$dBT delta 130 zcmdmCzTJF+I;%!Pac0U!9d1Fxgx?x-Eg86;sWGr7T@_b2D*>kU<~Y><|NkF^8KA(v z|AlDLVqKBV8G_R|q1qU1w*h4tfe2y@m=9%xj8v@qEcnN%E7Q$aVzPmB3(JN-*Sr92 C^elh? diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt index bb800f8592..b03b28372b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Digest.crc32 Filter.db -CompressionInfo.db +Digest.crc32 Index.db -Summary.db -Data.db TOC.txt +Summary.db Statistics.db +CompressionInfo.db +Data.db diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db index f5ad4d0cb1dce7207245e462142422e7f53df4dc..f0a1cfb59e84a0e2e737bd0d14f1554ff7674839 100644 GIT binary patch delta 50 hcmebGpP;0Bn1O+T6^M}l%TWl2vkpS@Swm={>i~;p1jqmY delta 50 hcmebGpP;0Bl7WGN6^M}l%YF!lvj9T#`9Nr4ApnZT1P1^B diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db index 721771665b5874f7f603543df52a98c1caf55f0d..b487fe88edf6a5289f2ebea0d7a7ae6873599409 100644 GIT binary patch literal 5759 zcmaKw0aR3F7RTR{QQqjvLub_)3?G6uGm^ex7+@d^bv+7Zh--(dCv zl>>bm*_HHxR#|O?L$hY;_CD?pEk4*1s(l){IjL8T@^NiX^drjrthY}y-Woex3}%59 z`?$9JOJrML+ozF|dz7Fzupo~O3LD5$UV$jv3@#K!mg0?M*_w-(l)*7&S!8(?S&mN~ z98<0*OEof)(gw#=TqjFy8)EE(Vx+2Qqc-MHv{7`XP^GmaHfw47opl{r!PKz4Ln~R2 zE0+2KRI;ZO>lm9X*uixxkpd2L$5Z6jt8gqaiEgbJ@O8=uX2DU{(ykr0bdF^8D5rRs zuaA=JqhUz+==xO+pOGC}+bO^61OBDww02mZMqT7s6QKEQRY!Q;5BCxJY$@Y)s)*1( zW-{oEozR!qN?q$yfzVgjMn#|Z0GV!L6Dc{Ew{)d1PpiM%mnXSIqgvjZOWj&QpS&kS zZVoxhUzPW1BGbLY?DbLVg10TS$4KlXz7wTH^LsPoy9T0$26bmgX1n$fvB?|5Ol}`N zDMv7S@1K+t$_9R8%J4Cm%~|RRfQ`#XY|OOj@xgr^a=W1~Pe}JB=~=f}GeQk^lwu1M zhtMel2sO^=#;59<79XlldaFK4NI3J*mzrUft61+JP_|)aeIdgh^kRjCIfqVbdB#_6 zw^Xh|@>#@Q(FGRXn~~XWtEK!Gl&Xn2WqSoyFyt1MgjRJ*Gm1(O{5Xr|DCkKRW&PqD ziYyqY(uEFOS}?y4@jnll>E+rtpU26@?``PSMBqwW=a5Al zOJ013R^HYMt9gGl-F7CFn!U1)Ob2lrB^w(B(~eikba**TDMO-mmyyYl1ykyfs2aMW z<|AkaDQ!s9UJ?fL`+At{L!$PZVbV1tecWxI+3(*iJ~tdSP>Kh~z?N`=WFy)rTP;IE z>%NEC*xHR%gSzCD^V)mC)>iaFtF0##4Rgxx^wRbpx1jMl6j5@@jh%$r(UD@v>pd5t z(^U*|?IYA#%AgZl37uQYpw?nSe>X9x-A?FIG=omZ5xP9UpiTkE|5EB)aDAk3waAdkCn5M@II(b_BFv5xZ=}IaM$31)?(>)QNsi3#Q z6jUE5?CRwx$mHEJBU^G61*HS6S(1g1P5<&kpaD^;F92PG5kg3KMX1zq?xT`xRlt8C zENj!VA#6J)=7hwmp6CX9ws609cTCQ9=#O;^)MrD{Q2g%nke+br3K zL={`ev=xmgj-0efoX-dItfEAo)x&7o-j83cmaYh#c9mEg;U7V^GW;hPcedeImo=OB zj}Y}dIq+!GsZG?cv#WYFna$QDCQy-V_r`A8v$_F$hUDm_7D9WfnMSHOM5uNngSNj) z=)iIYRh1Dsl*OQ(`Go3GSg4ca>RE&uP>zcBn1OVE>D3IAawJtLYsY{l93{o&7l$?D z&F_JY$B=hel%_(n1>@Y(L8A0lFKzS@Qdc*6yR%O-K@16!(**~^DOQl)-3JuoVbtg7 zR$ArEQU)EvM~ORzv@aHklBrP3qzuT0>u0jhxcoh%k$?bCcVni?nqO?>QF5ulz>rQ(B)A^Ngdrr?+gY!7FRz zV(jI2WP0wyha{M7Etn-$7U!jtvA8xo=ZYy8=rN_sEC*H zMpTgUM4K1TFORAA=k=>=W@%WzQg>7HFA$%M@Xtoy%MVr{gZMd11NxO#$@^!Ek5l_E zz)MEjMk!~_#BcT?ah77bJ+`s%e2hw;C)*am*|flfSvqagL}=&rB`Zmf>V2 z8?zV{JFkaK1x+xe45{bQHZrZKW%_Ml1EJLwEILA{q=-R_Ut`)x&f5qF)0$koP}K2e z*>W)BT03?P&O!KU{!cd*J-T6#<}CXyJkhJnY*dgtqBMZs$Y>r>_vgeVns5c z&HW5oZ9rmD>O*PjC(6H(A(!AB1oO5YtQVYVr01^YX$_TE^VE!`=jlK5Bd3GL4U z3eH2Bp30@f4q6zu%HOcZy<-+R9Hxh3D6!opAe{r>2CUxHM1v+Y(zn6YBjGA)#wz1n z;PHiF>!(|ZTK%|NcufU|AB-&?z8}!g4;Y?(Jj2W0juP~---3p<$D_@s1DJdhH1J5H zxyLhllu@fQ&yB6;c!-Z}3+`)W4J$r0_;~*G!+EWc@Lu;V4WAaT3J`k0$9Y;OBYl0h z>o`06c=q*TH?yn7qh-aXH37bci#4qwO>s&e*R*O!;SdA5H2ehz@=nvd2rvx9((cjB1d*ZrlR%wTcC?;@jkuldJsIDu18#rl*7Bj?$m66lqF< zHd%wp^hx9YQKoD3C$Qjy(Ebwdw_|Lc?hY*3mPnH6y<|J+GEcj8b$Xqdo&(DW1{ROvozXqdUI zbl)9xpOlOY)bt!tvJbZsPYpDoWMh^h)wELMr3@+O8ro4X-|NRxsf<5WdYvPazXzdd zOcX7Jf05~DBifh__~+Mb2Yh=Tv8KodR&f5^0%jaljd=CpRp-KBpVrliNsdvM?+z*J zi#SlVf$k~2siWQujXL*suv_}eDBTJ4N1Z^^IWjvU@`+8vnS@#7EMk(LWuwesx&(A* z)vA!*AY2>+obUj(=QAY?A3FeUn%a#@f`S6)@jgJ~bQBqq9=wb9Ddl~v%y41X7%%EG zpS3P1W#VJH!qqtBj!DbrU>r-u9H&0SI^r?bOg)>yOl=QrrMo|S<}-Trpr#gw*Kx|z z7a=S2Tzvhh4%T!+^P|-RaLTE1l>1{SEBxs4G*%#adsjQYgvIcq%hR}x#N_n+SXFpP rOy0|m6-L1^{Mz!kvSKlwbY{VkQJoX>VuiM@bJ1Mjhv%&ae>diT1fhU1 literal 5888 zcmZ{oeOy#k8prRzl{usC429VY8(tV*=H40L4g)wdqx_Q z&U5a0zV|toAY{Z&LS`(Vn3$-5|0jq@qPop!Ofn^>q|P;4tkVe?nW(>VJFOy!7(!M- z>?8hmZPB)cq5IkE^RE8*(zC?0)gRv(`DM*nvYe2CcYD$W68r2gH&mY|I^K4BNqJ1v z_LB<2OR{ZA#}vMRAvSV@zS>K?k?SVq*2wGQT?&;)@N*L#?>B&-HEM69mz-^in5ObC z1cG{rj?X*1Bw|ixdPypI#OGBH0c9}3jZr+}Ee5ZYc?9nTQ*)gWG#px~$1}C=j!7)S*z>Q*p8lA!Q02 zYm*vc(aVlQEYlI+Nb^cB(OEVOvhK-TklQXBu)}`hNGf}19CpSWrK)g~Y`fpy#&y0@ z<6G-`NJiD|+spm3W+l5P9k26wH>i`L!Su~gtd&7>e$CDvt-#X0F;SwlQ}v3fI+*u$8?I9){nF>-wY)Z^HXV z!PCxqGqKb=EK1bn(^ZhmC#S3U1=`qdSt$qFSEU@4!q3@#+tC_CNB?qhIn|b+^{tKV zc!F4ln5cV+WfHrXJ+*L<6nknVS_+OAqn5CJw8L@k%N(Ywr zsdN(WXMEZ+iNq3gHPu!q$_=~SU|gct zQf&pI+Gxo_u~F5QtR*r+mK1vx&)2?EuOhmC+7*WAm@_+0sB}Knw!}?1?m1b5EU|I< zyTerU;)$Pfd~jOz+xJ;7GL1}oT?>g z;u^rkLpUDJhcjr3wT^noU|gzcFrq@(h;S%4M*)G4N*0tMl`OjA`YwjbM$E(MFp?rV z!3gzYO&GGq=8wa_sewM*L-F}6hEq(<4e(*#0u3?MkuYKo^mz|7#MHOIh$YZxz<06_ ztKk@b%K&&(mD47!;ia7i32Ay0wXDe)=<*9Bg4Ca%G4k-Sv+{JPO-szz?t@Co#GmsX z2m?7>>WA14A9rkAPpI3WkXvJa%@OJbL>hj{t#LC)q~hK13o()TQf*l#*34qjfpa1J zLd+kFrFL6#K?`4zs1-`Mr;@>Dg{_jZ?)l3&Xqt8NvxI`>dUn8ak?z?dheBWv>zHv+ z3AOYSWalT0oq7!l)*QFRBEgQGZ7RZsl?c}2rw-fmu6H?N;&ynK3%=};G5iQ$6%k`+ zJ6iDa4jP3l?9|ySWyl)$L8Boj_uW7HXuf)ylfB)zjr={hE6Mi2HU&HwhFW)+bW6& z4%eGMtkTKNy8Cn2s)0lNDd^qoGkr*AlZ4!cQ_~eyNFGLSeFerziApCrbG-*F_z6{= zsp6gORQs7KMC%9@$t&m%Eo`!=;)|4c1^r5(6ThIr<*@A_=d1iC&b8Q^*Fs@V0FK=0q*h zGr1Bp^;ZFu5c5G^1nCV3g)}eVMvF&N|6BEG3Z@)hq6i3zdV zVWSYU%P5quojq2Ht?ILjMXQf8vDN-ziPfoLkm&Ye$iNQ>JL`d8NY@pLv*O#KzL}3@ zK#~0LScYIneKQ{_lo{&Bz~!B(s~|GWXP3)(I~(gHfFm?>>2jH*8*wZ~SENzPV)bU#2*st->=ZH;FU!*~c%rr5w6+kUEMSb+vLO==H3Mlh8 zML?%M!&r5gJeHn$;~=I}ayb-1bJ|ooIiIQ82`MeD+8Kzafsp3_g!*s)S0>;I?iS9M z8{SjO8U5;~GN>f44-L458<_n(Hoo>epxw_X8#@qD$tg59D>3IRRXvxEM?Q2*O8R^FftXXUZlUnUA#KE*14!@;Vh0gi*Q|!o5mDm?f8PQ9n}XfYlM{l}7f2a^D4>6Gy$P2%I-wkIx!6Cs63kk}B>e7Nm6T7eLtau5{t9g8 zWCI21woiu}b;8c+%R~~nPE=h&`h?f3F7alZtFsXvm}Y-dEtS-i2CIZm4Xz$IgGg=b zNcDrQg>yMK;zM`nKY3Kr0c@6;^O|+K5X!T=miP)pf^ZA z?t!tWT!0dANaE-Nk<0A{SEjQ9HgJ+dcJ@mrmU@bCp~DIUh}x9wXd51K=D<*jBoyY* z7=!?{M8Z(&l%&SN9x-P}U`Wbr0=~2j$;a^6_a!hkmDph|c<5@TP(3zoh{^|Uh9$bm zU}+yVq%(qPqmJ$^r$t+TV)@Q#oh6B0ttlhedY5I1yh4Mf)` z>ZCI+PRAik{auP6Xe`@>9%|^(d0B`_{DasY!SV`lLl?Qg(y5px!iO8$ScGFNMQ)Om zv|ObYP&JQ=3$|0$^=ZhMSWpN462qF$AgX4qIFp^--iduTirtEx-Cc(#dCu6Z3|a>t zR{R|m=gOraSth|EUhV8`1MIilm{_XrP>^fY0(RKFt02a0FI^y~30q)i8?$iWic#3a zZDC?-#SJw>pMW&g;-@R1pa!JK?TrBmP=~$C6;X9Hf$6^o{UK8CcH{~8vQzJG0TL!Q zY!OMw{<{fp>E84M%3yqlV&;?XWcnH3nNqQ?+`OT%k sla)7@N?R+}D>s&+q=fZ#m0>}Fmp|9e4!HcaZptSn|75bT;HDY>2O6HLKmY&$ diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 index 4f1391a4d5..ca286e0954 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -4072239034 \ No newline at end of file +2759187708 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db index 6dd3da6e4b42860c181c683c815fd4181242f9bc..c981a226039e4a75a5fcc064c73c9928daf7b7e5 100644 GIT binary patch delta 115 zcmex(j`QO=&JBi6jGwnBSTN>uGVa}e!I80$lg0JmjnA9)E&xSNZ=QFvh?6mRv*2T( z`0ax?es9h%%;jW>J@}@2@`EZXM)~atZx{uu*%%m&4!-b!bBqt&DTH%O4quoB;Q#=2 CFfhCT delta 109 zcmex(j`QO=&JBi6+v6=53pg3~ZNK2iSjfrfw!OfSsfd&D?B;nlfmG;b!N)++yW2ND u0;*z3I{2n?@`EZXM%C>JZx{uu*%%m&4!-b%bBqt&DTi}R4qsRR;Q#<^nkzg2 diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db index 3a0e63f8843af9c3389ad05d85ab1689be8e46d0..33fccc9c84bd3494eb0e6921db5e278b899ab053 100644 GIT binary patch delta 131 zcmX?Le$ae^I;*C5N#v)EI^2SWsaCoLfec(PHZZUjXFDDjzX7IC`~Il^|NlP-GeCj; zq_cmwiI#V4&Jdi&X#~;6;H3kUWdtH5K9mhIlHt%1(~v*G3X|VVzcE=~x|e0*thZhO D`j#!= delta 130 zcmX?Te!zT!I;%!Pac0U!9d1Fxgx?x-Eg86;sWGr7T@_b&umDW!&2g;%|NlP-GeCj; zWGks%Jwg?mGX$q`LbWlt^8sZUfe2y@m=9%xjJ*Gq&Fqg;SEgISg2@Kby)3cTLhAv0 C@GE5i diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt index bb800f8592..b03b28372b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Digest.crc32 Filter.db -CompressionInfo.db +Digest.crc32 Index.db -Summary.db -Data.db TOC.txt +Summary.db Statistics.db +CompressionInfo.db +Data.db diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db index c665dfb00ed7fa366a970445c79ee9c88ea43385..11219d037a8da6ad934880d1c2b6d003f1aece36 100644 GIT binary patch delta 49 zcma!yn4qI`aODYQ25BQ@2Ae}AFM%v$AnUM=qB4Vwi86!g>(|N*0SZZpRgAYpJr4r_ Dc*G9} delta 50 zcma!uoS>t7#CL)+gS3${gUxZ{13;EBkahC-3n0rxnZfi-*9T>W0EMK)D#puhPk#Xb Dg`p7U diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 index c6c24a74bb..985d6dcf36 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 @@ -1 +1 @@ -3772296151 \ No newline at end of file +462858821 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db index 67414306d8d7e68d980aaf7df76a86b6686e6f4c..3c68ac568f566f72c9a93b6dda378bcc0f309fe9 100644 GIT binary patch delta 144 zcmZ3XvQlM&I_u%JF1zG5>TnAhrdsJ11Tt{F*ucPAob7mQ9|M>^weofS|Ns9%m;nmx z?+e?kUT$_^a-pE~hGIZPj6j6Mhq6H?Gt3QO^!^j9F!_xgP@Tp^ QJ+aA;1f5vwUoUzL0RH?g*8l(j delta 143 zcmZ3fvO;BoI_sfZ&RrcFb+`o$6Mk#VwPfIWrpCaUbX8m-y8uk3+p=^-JmmGu4{y24Iy1p)8V9=PT QCpP)9pcBi9JMZ5C0NIx`aR2}S diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt index bb800f8592..b03b28372b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Digest.crc32 Filter.db -CompressionInfo.db +Digest.crc32 Index.db -Summary.db -Data.db TOC.txt +Summary.db Statistics.db +CompressionInfo.db +Data.db diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db index d9fe576f958c8d841cca8931877e933a2d7480e3..620cdf260e5aa91c847989cb8ad33ba3d9380dae 100644 GIT binary patch delta 117 zcmeBS>|&fClQ(<&+j4_B?tfcn+N%@>C-O3|7H2yiX9>_?$Yfw-WMDM3V#wFxXE-=Z z(}scF{6zW*8wMF88wUGBnFb(%lV%ry0>(gr!#Yhi3>@nJIBXbXO>7wCA5<`K95}W1 O0s{mvGRAp$+yDSHKOrvw delta 119 zcmeBT>|vZBQ!tmI@0)MCx4A(80=4ONvHrS(6R=K%a{NK&Xi{WbsW(7 P<-q^}jEt!-^nU;VrfVVe diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 index de7baed426..bc5f671e01 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -4035692752 \ No newline at end of file +3987542254 \ No newline at end of file diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db index e9556d1b7cd6e1c4a9c52b8fb8cbf1666083023f..689bec8f1a850b0d03e73337f9420ad8ee44691d 100644 GIT binary patch delta 144 zcmdm}vPorvI_u%JF1zG5>TnAhrdsJ11Tt{F*ucPAob7n*>;f=-YVC#k|NsAkFas3W z-#P9Za3J^FlY0btOd+}%Y`lPq7=Z|h4`qW)W)Q1668a}tVe%W_1q=)t R6ZOOGa8b2LQtkF;f5l delta 143 zcmdm_vQcG%I_sfZ&RrcFb+`o$6Mk#VwPfIWrpCaUbX8oT=>eG5`Q=gn|NnmwW`F|w zyQa>J+pis&Tqr0#xj~R~a*rU72}C!8r4~>TBM>3+p=^-JjlxI1{&DKcbYptJz@RZv PPi*pIL2s6S`t{NPm+moL diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt index bb800f8592..b03b28372b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ -Digest.crc32 Filter.db -CompressionInfo.db +Digest.crc32 Index.db -Summary.db -Data.db TOC.txt +Summary.db Statistics.db +CompressionInfo.db +Data.db diff --git a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java index a5025a36ad..94a3bd3569 100644 --- a/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java +++ b/test/long/org/apache/cassandra/locator/DynamicEndpointSnitchLongTest.java @@ -54,19 +54,21 @@ public class DynamicEndpointSnitchLongTest DynamicEndpointSnitch dsnitch = new DynamicEndpointSnitch(ss, String.valueOf(ss.hashCode())); InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); - List hosts = new ArrayList<>(); + EndpointsForRange.Builder replicasBuilder = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE); // We want a big list of hosts so sorting takes time, making it much more likely to reproduce the // problem we're looking for. for (int i = 0; i < 100; i++) for (int j = 0; j < 256; j++) - hosts.add(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, (byte)i, (byte)j})); + replicasBuilder.add(ReplicaUtils.full(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, (byte)i, (byte)j}))); - ScoreUpdater updater = new ScoreUpdater(dsnitch, hosts); + EndpointsForRange replicas = replicasBuilder.build(); + + ScoreUpdater updater = new ScoreUpdater(dsnitch, replicas); updater.start(); - List result = null; + EndpointsForRange result = replicas; for (int i = 0; i < ITERATIONS; i++) - result = dsnitch.getSortedListByProximity(self, hosts); + result = dsnitch.sortedByProximity(self, result); updater.stopped = true; updater.join(); @@ -84,10 +86,10 @@ public class DynamicEndpointSnitchLongTest public volatile boolean stopped; private final DynamicEndpointSnitch dsnitch; - private final List hosts; + private final EndpointsForRange hosts; private final Random random = new Random(); - public ScoreUpdater(DynamicEndpointSnitch dsnitch, List hosts) + public ScoreUpdater(DynamicEndpointSnitch dsnitch, EndpointsForRange hosts) { this.dsnitch = dsnitch; this.hosts = hosts; @@ -97,9 +99,9 @@ public class DynamicEndpointSnitchLongTest { while (!stopped) { - InetAddressAndPort host = hosts.get(random.nextInt(hosts.size())); + Replica host = hosts.get(random.nextInt(hosts.size())); int score = random.nextInt(SCORE_RANGE); - dsnitch.receiveTiming(host, score); + dsnitch.receiveTiming(host.endpoint(), score); } } } diff --git a/test/long/org/apache/cassandra/streaming/LongStreamingTest.java b/test/long/org/apache/cassandra/streaming/LongStreamingTest.java index 01e67f0d32..e37045aead 100644 --- a/test/long/org/apache/cassandra/streaming/LongStreamingTest.java +++ b/test/long/org/apache/cassandra/streaming/LongStreamingTest.java @@ -33,11 +33,10 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.io.sstable.CQLSSTableWriter; import org.apache.cassandra.io.sstable.SSTableLoader; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadataRef; @@ -121,8 +120,8 @@ public class LongStreamingTest private String ks; public void init(String keyspace) { - for (Range range : StorageService.instance.getLocalRanges(KS)) - addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort()); + for (Replica range : StorageService.instance.getLocalReplicas(KS)) + addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort()); this.ks = keyspace; } @@ -148,8 +147,8 @@ public class LongStreamingTest private String ks; public void init(String keyspace) { - for (Range range : StorageService.instance.getLocalRanges(KS)) - addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort()); + for (Replica range : StorageService.instance.getLocalReplicas(KS)) + addRangeForEndpoint(range.range(), FBUtilities.getBroadcastAddressAndPort()); this.ks = keyspace; } diff --git a/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java b/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java index 68cfd7eb8b..73a2b71356 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/PendingRangesBench.java @@ -21,18 +21,22 @@ package org.apache.cassandra.test.microbench; import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.PendingRangeMaps; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaUtils; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import java.net.UnknownHostException; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; @@ -50,7 +54,7 @@ public class PendingRangesBench PendingRangeMaps pendingRangeMaps; int maxToken = 256 * 100; - Multimap, InetAddressAndPort> oldPendingRanges; + Multimap, Replica> oldPendingRanges; private Range genRange(String left, String right) { @@ -63,15 +67,17 @@ public class PendingRangesBench pendingRangeMaps = new PendingRangeMaps(); oldPendingRanges = HashMultimap.create(); - InetAddressAndPort[] addresses = { InetAddressAndPort.getByName("127.0.0.1"), InetAddressAndPort.getByName("127.0.0.2")}; + List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), + InetAddressAndPort.getByName("127.0.0.2")); for (int i = 0; i < maxToken; i++) { for (int j = 0; j < ThreadLocalRandom.current().nextInt(2); j ++) { Range range = genRange(Integer.toString(i * 10 + 5), Integer.toString(i * 10 + 15)); - pendingRangeMaps.addPendingRange(range, addresses[j]); - oldPendingRanges.put(range, addresses[j]); + Replica replica = Replica.fullReplica(endpoints.get(j), range); + pendingRangeMaps.addPendingRange(range, replica); + oldPendingRanges.put(range, replica); } } @@ -79,8 +85,9 @@ public class PendingRangesBench for (int j = 0; j < ThreadLocalRandom.current().nextInt(2); j ++) { Range range = genRange(Integer.toString(maxToken * 10 + 5), Integer.toString(5)); - pendingRangeMaps.addPendingRange(range, addresses[j]); - oldPendingRanges.put(range, addresses[j]); + Replica replica = Replica.fullReplica(endpoints.get(j), range); + pendingRangeMaps.addPendingRange(range, replica); + oldPendingRanges.put(range, replica); } } @@ -97,13 +104,13 @@ public class PendingRangesBench { int randomToken = ThreadLocalRandom.current().nextInt(maxToken * 10 + 5); Token searchToken = new RandomPartitioner.BigIntegerToken(Integer.toString(randomToken)); - Set endpoints = new HashSet<>(); - for (Map.Entry, Collection> entry : oldPendingRanges.asMap().entrySet()) + Set replicas = new HashSet<>(); + for (Map.Entry, Collection> entry : oldPendingRanges.asMap().entrySet()) { if (entry.getKey().contains(searchToken)) - endpoints.addAll(entry.getValue()); + replicas.addAll(entry.getValue()); } - bh.consume(endpoints); + bh.consume(replicas); } } diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 1201efaaca..bc2c19c962 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -32,9 +32,11 @@ import java.util.function.Supplier; import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +75,7 @@ import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; public class Util @@ -446,6 +449,14 @@ public class Util } } + public static void consume(UnfilteredPartitionIterator iterator) + { + while (iterator.hasNext()) + { + consume(iterator.next()); + } + } + public static int size(PartitionIterator iter) { int size = 0; @@ -478,6 +489,15 @@ public class Util && Iterators.elementsEqual(a, b); } + public static boolean sameContent(RowIterator a, RowIterator b) + { + return Objects.equals(a.metadata(), b.metadata()) + && Objects.equals(a.isReverseOrder(), b.isReverseOrder()) + && Objects.equals(a.partitionKey(), b.partitionKey()) + && Objects.equals(a.staticRow(), b.staticRow()) + && Iterators.elementsEqual(a, b); + } + public static boolean sameContent(Mutation a, Mutation b) { if (!a.key().equals(b.key()) || !a.getTableIds().equals(b.getTableIds())) diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 07ab3dc9e1..782e3b10ef 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -104,6 +104,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.io.util.DataOutputPlus", "org.apache.cassandra.io.util.DiskOptimizationStrategy", "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy", + "org.apache.cassandra.locator.Replica", "org.apache.cassandra.locator.SimpleSeedProvider", "org.apache.cassandra.locator.SeedProvider", "org.apache.cassandra.net.BackPressureStrategy", @@ -134,7 +135,9 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer", "org.apache.cassandra.ConsoleAppenderBeanInfo", "org.apache.cassandra.ConsoleAppenderCustomizer", - "org.apache.cassandra.locator.InetAddressAndPort" + "org.apache.cassandra.locator.InetAddressAndPort", + "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement", + "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement" }; static final Set checkedClasses = new HashSet<>(Arrays.asList(validClasses)); diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 4a1a3655cf..2fbbc28190 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -49,6 +49,7 @@ import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.*; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.functions.FunctionName; @@ -148,7 +149,7 @@ public abstract class CQLTester { @Override public String getRack(InetAddressAndPort endpoint) { return RACK1; } @Override public String getDatacenter(InetAddressAndPort endpoint) { return DATA_CENTER; } - @Override public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) { return 0; } + @Override public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } }); try diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java index 184c5ad398..37605d6695 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.locator.AbstractEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.SchemaKeyspace; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.schema.*; @@ -527,7 +528,7 @@ public class CreateTest extends CQLTester public String getDatacenter(InetAddressAndPort endpoint) { return "us-east-1"; } @Override - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) { return 0; } + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } }); // this forces the dc above to be added to the list of known datacenters (fixes static init problem diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java index a096c781ad..46c0afd938 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTest.java @@ -107,7 +107,6 @@ public class CleanupTest SchemaLoader.compositeIndexCFMD(KEYSPACE2, CF_INDEXED2, true)); } - /* @Test public void testCleanup() throws ExecutionException, InterruptedException { @@ -116,7 +115,6 @@ public class CleanupTest Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - UnfilteredPartitionIterator iter; // insert data and verify we get it back w/ range query fillCF(cfs, "val", LOOPS); @@ -124,8 +122,7 @@ public class CleanupTest // record max timestamps of the sstables pre-cleanup List expectedMaxTimestamps = getMaxTimestampList(cfs); - iter = Util.getRangeSlice(cfs); - assertEquals(LOOPS, Iterators.size(iter)); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); // with one token in the ring, owned by the local node, cleanup should be a no-op CompactionManager.instance.performCleanup(cfs, 2); @@ -134,10 +131,8 @@ public class CleanupTest assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs)); // check data is still there - iter = Util.getRangeSlice(cfs); - assertEquals(LOOPS, Iterators.size(iter)); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); } - */ @Test public void testCleanupWithIndexes() throws IOException, ExecutionException, InterruptedException diff --git a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java new file mode 100644 index 0000000000..9789183dc1 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.partitions.FilteredPartition; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.PendingRangeCalculatorService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class CleanupTransientTest +{ + private static final IPartitioner partitioner = RandomPartitioner.instance; + private static IPartitioner oldPartitioner; + + public static final int LOOPS = 200; + public static final String KEYSPACE1 = "CleanupTest1"; + public static final String CF_INDEXED1 = "Indexed1"; + public static final String CF_STANDARD1 = "Standard1"; + + public static final String KEYSPACE2 = "CleanupTestMultiDc"; + public static final String CF_INDEXED2 = "Indexed2"; + public static final String CF_STANDARD2 = "Standard2"; + + public static final ByteBuffer COLUMN = ByteBufferUtil.bytes("birthdate"); + public static final ByteBuffer VALUE = ByteBuffer.allocate(8); + static + { + VALUE.putLong(20101229); + VALUE.flip(); + } + + @BeforeClass + public static void setup() throws Exception + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + oldPartitioner = StorageService.instance.setPartitionerUnsafe(partitioner); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, + KeyspaceParams.simple("2/1"), + SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1), + SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true)); + + StorageService ss = StorageService.instance; + final int RING_SIZE = 2; + + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + ArrayList endpointTokens = new ArrayList<>(); + ArrayList keyTokens = new ArrayList<>(); + List hosts = new ArrayList<>(); + List hostIds = new ArrayList<>(); + + endpointTokens.add(RandomPartitioner.MINIMUM); + endpointTokens.add(RandomPartitioner.instance.midpoint(RandomPartitioner.MINIMUM, new RandomPartitioner.BigIntegerToken(RandomPartitioner.MAXIMUM))); + + Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, RING_SIZE); + PendingRangeCalculatorService.instance.blockUntilFinished(); + + + DatabaseDescriptor.setEndpointSnitch(new AbstractNetworkTopologySnitch() + { + @Override + public String getRack(InetAddressAndPort endpoint) + { + return "RC1"; + } + + @Override + public String getDatacenter(InetAddressAndPort endpoint) + { + return "DC1"; + } + }); + } + + @Test + public void testCleanup() throws Exception + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); + + + // insert data and verify we get it back w/ range query + fillCF(cfs, "val", LOOPS); + + // record max timestamps of the sstables pre-cleanup + List expectedMaxTimestamps = getMaxTimestampList(cfs); + + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); + + // with two tokens RF=2/1 and the sstable not repaired this should do nothing + CompactionManager.instance.performCleanup(cfs, 2); + + // ensure max timestamp of the sstables are retained post-cleanup + assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs)); + + // check data is still there + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); + + //Get an exact count of how many partitions are in the fully replicated range and should + //be retained + int fullCount = 0; + RangesAtEndpoint localRanges = StorageService.instance.getLocalReplicas(keyspace.getName()).filter(Replica::isFull); + for (FilteredPartition partition : Util.getAll(Util.cmd(cfs).build())) + { + Token token = partition.partitionKey().getToken(); + for (Replica r : localRanges) + { + if (r.range().contains(token)) + fullCount++; + } + } + + SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false); + sstable.reloadSSTableMetadata(); + + // This should remove approximately 50% of the data, specifically whatever was transiently replicated + CompactionManager.instance.performCleanup(cfs, 2); + + // ensure max timestamp of the sstables are retained post-cleanup + assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs)); + + // check less data is there, all transient data should be gone since the table was repaired + assertEquals(fullCount, Util.getAll(Util.cmd(cfs).build()).size()); + } + + protected void fillCF(ColumnFamilyStore cfs, String colName, int rowsPerSSTable) + { + CompactionManager.instance.disableAutoCompaction(); + + for (int i = 0; i < rowsPerSSTable; i++) + { + String key = String.valueOf(i); + // create a row and update the birthdate value, test that the index query fetches the new version + new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBufferUtil.bytes(key)) + .clustering(COLUMN) + .add(colName, VALUE) + .build() + .applyUnsafe(); + } + + cfs.forceBlockingFlush(); + } + + protected List getMaxTimestampList(ColumnFamilyStore cfs) + { + List list = new LinkedList(); + for (SSTableReader sstable : cfs.getLiveSSTables()) + list.add(sstable.getMaxTimestamp()); + return list; + } +} diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java index 66bbff33bd..5ceb233a56 100644 --- a/test/unit/org/apache/cassandra/db/ImportTest.java +++ b/test/unit/org/apache/cassandra/db/ImportTest.java @@ -174,7 +174,7 @@ public class ImportTest extends CQLTester Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); for (SSTableReader sstable : sstables) - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, 111, null); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 111, null, false); File backupdir = moveToBackupDir(sstables); assertEquals(0, execute("select * from %s").size()); diff --git a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java index e01088db27..a864786dbc 100644 --- a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java +++ b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java @@ -308,7 +308,7 @@ public class RepairedDataTombstonesTest extends CQLTester public static void repair(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException { - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, 1, null); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false); sstable.reloadSSTableMetadata(); cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); } diff --git a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java index 706b27423d..1ac5440d39 100644 --- a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java +++ b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java @@ -70,6 +70,12 @@ public class RowUpdateBuilder this.updateBuilder.nowInSec(localDeletionTime); } + public RowUpdateBuilder timestamp(long ts) + { + updateBuilder.timestamp(ts); + return this; + } + private Row.SimpleBuilder rowBuilder() { // Normally, rowBuilder is created by the call to clustering(), but we allow skipping that call for an empty diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index 36af54f12e..b58909b434 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -615,7 +615,7 @@ public class ScrubTest { SerializationHeader header = new SerializationHeader(true, metadata.get(), metadata.get().regularAndStaticColumns(), EncodingStats.NO_STATS); MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(0); - return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, null, metadata, collector, header, txn), txn); + return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, null, false, metadata, collector, header, txn), txn); } private static class TestMultiWriter extends SimpleSSTableMultiWriter @@ -631,10 +631,10 @@ public class ScrubTest */ private static class TestWriter extends BigTableWriter { - TestWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, TableMetadataRef metadata, + TestWriter(Descriptor descriptor, long keyCount, long repairedAt, UUID pendingRepair, boolean isTransient, TableMetadataRef metadata, MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn) { - super(descriptor, keyCount, repairedAt, pendingRepair, metadata, collector, header, Collections.emptySet(), txn); + super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, Collections.emptySet(), txn); } @Override diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java index 1c051f50d0..a14db00b27 100644 --- a/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java +++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java @@ -34,6 +34,8 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.utils.UUIDGen; import static org.junit.Assert.assertEquals; @@ -189,4 +191,28 @@ public class SystemKeyspaceMigrator40Test extends CQLTester } assertEquals(1, rowCount); } + + @Test + public void testMigrateAvailableRanges() throws Throwable + { + Range testRange = new Range<>(DatabaseDescriptor.getPartitioner().getRandomToken(), DatabaseDescriptor.getPartitioner().getRandomToken()); + String insert = String.format("INSERT INTO %s (" + + "keyspace_name, " + + "ranges) " + + " values ( ?, ? )", + SystemKeyspaceMigrator40.legacyAvailableRangesName); + execute(insert, + "foo", + ImmutableSet.of(SystemKeyspace.rangeToBytes(testRange))); + SystemKeyspaceMigrator40.migrate(); + + int rowCount = 0; + for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.availableRangesName))) + { + rowCount++; + assertEquals("foo", row.getString("keyspace_name")); + assertEquals(ImmutableSet.of(testRange), SystemKeyspace.rawRangesToRangeSet(row.getSet("full_ranges", BytesType.instance), DatabaseDescriptor.getPartitioner())); + } + assertEquals(1, rowCount); + } } diff --git a/test/unit/org/apache/cassandra/db/TableCQLHelperTest.java b/test/unit/org/apache/cassandra/db/TableCQLHelperTest.java index 32fa4e4ffa..fff567bbce 100644 --- a/test/unit/org/apache/cassandra/db/TableCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/TableCQLHelperTest.java @@ -26,6 +26,7 @@ import java.util.*; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; +import org.apache.cassandra.service.reads.NeverSpeculativeRetryPolicy; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -255,6 +256,7 @@ public class TableCQLHelperTest extends CQLTester .maxIndexInterval(7) .memtableFlushPeriod(8) .speculativeRetry(AlwaysSpeculativeRetryPolicy.INSTANCE) + .speculativeWriteThreshold(NeverSpeculativeRetryPolicy.INSTANCE) .extensions(ImmutableMap.of("ext1", ByteBuffer.wrap("val1".getBytes()))) .recordColumnDrop(ColumnMetadata.regularColumn(keyspace, table, "reg1", AsciiType.instance), FBUtilities.timestampMicros()); @@ -272,6 +274,7 @@ public class TableCQLHelperTest extends CQLTester "\tAND max_index_interval = 7\n" + "\tAND memtable_flush_period_in_ms = 8\n" + "\tAND speculative_retry = 'ALWAYS'\n" + + "\tAND speculative_write_threshold = 'NEVER'\n" + "\tAND comment = 'comment'\n" + "\tAND caching = { 'keys': 'ALL', 'rows_per_partition': 'NONE' }\n" + "\tAND compaction = { 'max_threshold': '32', 'min_threshold': '4', 'sstable_size_in_mb': '1', 'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy' }\n" + diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java index 0632274026..df2acb4fb1 100644 --- a/test/unit/org/apache/cassandra/db/VerifyTest.java +++ b/test/unit/org/apache/cassandra/db/VerifyTest.java @@ -421,7 +421,7 @@ public class VerifyTest // make the sstable repaired: SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, System.currentTimeMillis(), sstable.getSSTableMetadata().pendingRepair); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), sstable.getPendingRepair(), sstable.isTransient()); sstable.reloadSSTableMetadata(); // break the sstable: @@ -487,7 +487,7 @@ public class VerifyTest fillCF(cfs, 2); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, 1, sstable.getSSTableMetadata().pendingRepair); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, sstable.getPendingRepair(), sstable.isTransient()); sstable.reloadSSTableMetadata(); cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); assertTrue(sstable.isRepaired()); diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java index a3202482b4..4d628940a7 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java @@ -109,17 +109,16 @@ public class AbstractPendingRepairTest extends AbstractRepairTest SSTableReader sstable = diff.iterator().next(); if (orphan) { - Iterables.any(csm.getUnrepaired(), s -> s.getSSTables().contains(sstable)); - csm.getUnrepaired().forEach(s -> s.removeSSTable(sstable)); + csm.getUnrepairedUnsafe().allStrategies().forEach(acs -> acs.removeSSTable(sstable)); } return sstable; } - protected static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair) + protected static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient) { try { - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, repairedAt, pendingRepair); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient); sstable.reloadSSTableMetadata(); } catch (IOException e) @@ -130,11 +129,11 @@ public class AbstractPendingRepairTest extends AbstractRepairTest protected static void mutateRepaired(SSTableReader sstable, long repairedAt) { - mutateRepaired(sstable, repairedAt, ActiveRepairService.NO_PENDING_REPAIR); + mutateRepaired(sstable, repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false); } - protected static void mutateRepaired(SSTableReader sstable, UUID pendingRepair) + protected static void mutateRepaired(SSTableReader sstable, UUID pendingRepair, boolean isTransient) { - mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair); + mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, isTransient); } } diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index 366c18e2f0..f514ea65d0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.function.Predicate; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -39,6 +40,8 @@ import org.junit.Test; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; @@ -50,7 +53,6 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.KeyspaceParams; @@ -58,6 +60,7 @@ import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; import org.apache.cassandra.utils.concurrent.Refs; import org.apache.cassandra.UpdateBuilder; @@ -77,16 +80,21 @@ public class AntiCompactionTest { private static final String KEYSPACE1 = "AntiCompactionTest"; private static final String CF = "AntiCompactionTest"; + private static final Collection> NO_RANGES = Collections.emptyList(); + private static TableMetadata metadata; private static ColumnFamilyStore cfs; + private static InetAddressAndPort local; + @BeforeClass - public static void defineSchema() throws ConfigurationException + public static void defineSchema() throws Throwable { SchemaLoader.prepareServer(); metadata = SchemaLoader.standardCFMD(KEYSPACE1, CF).build(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), metadata); cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); + local = InetAddressAndPort.getByName("127.0.0.1"); } @After @@ -97,56 +105,86 @@ public class AntiCompactionTest store.truncateBlocking(); } - private void registerParentRepairSession(UUID sessionID, Collection> ranges, long repairedAt, UUID pendingRepair) throws IOException + private void registerParentRepairSession(UUID sessionID, Iterable> ranges, long repairedAt, UUID pendingRepair) throws IOException { ActiveRepairService.instance.registerParentRepairSession(sessionID, InetAddressAndPort.getByName("10.0.0.1"), - Lists.newArrayList(cfs), ranges, + Lists.newArrayList(cfs), ImmutableSet.copyOf(ranges), pendingRepair != null || repairedAt != UNREPAIRED_SSTABLE, repairedAt, true, PreviewKind.NONE); } - private void antiCompactOne(long repairedAt, UUID pendingRepair) throws Exception + private static RangesAtEndpoint atEndpoint(Collection> full, Collection> trans) { - assert repairedAt != UNREPAIRED_SSTABLE || pendingRepair != null; + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : full) + builder.add(new Replica(local, range, true)); - ColumnFamilyStore store = prepareColumnFamilyStore(); - Collection sstables = getUnrepairedSSTables(store); - assertEquals(store.getLiveSSTables().size(), sstables.size()); - Range range = new Range(new BytesToken("0".getBytes()), new BytesToken("4".getBytes())); - List> ranges = Arrays.asList(range); + for (Range range : trans) + builder.add(new Replica(local, range, false)); - int repairedKeys = 0; + return builder.build(); + } + + private static Collection> range(int l, int r) + { + return Collections.singleton(new Range<>(new BytesToken(Integer.toString(l).getBytes()), new BytesToken(Integer.toString(r).getBytes()))); + } + + private static class SSTableStats + { + int numLiveSSTables = 0; int pendingKeys = 0; - int nonRepairedKeys = 0; + int transKeys = 0; + int unrepairedKeys = 0; + } + + private SSTableStats antiCompactRanges(ColumnFamilyStore store, RangesAtEndpoint ranges) throws IOException + { + UUID sessionID = UUID.randomUUID(); + Collection sstables = getUnrepairedSSTables(store); try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); Refs refs = Refs.ref(sstables)) { if (txn == null) throw new IllegalStateException(); - UUID parentRepairSession = pendingRepair == null ? UUID.randomUUID() : pendingRepair; - registerParentRepairSession(parentRepairSession, ranges, repairedAt, pendingRepair); - CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, repairedAt, pendingRepair, parentRepairSession); + registerParentRepairSession(sessionID, ranges.ranges(), FBUtilities.nowInSeconds(), sessionID); + CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, sessionID); } - assertEquals(2, store.getLiveSSTables().size()); + SSTableStats stats = new SSTableStats(); + stats.numLiveSSTables = store.getLiveSSTables().size(); + + Predicate fullContains = t -> Iterables.any(ranges.fullRanges(), r -> r.contains(t)); + Predicate transContains = t -> Iterables.any(ranges.transientRanges(), r -> r.contains(t)); for (SSTableReader sstable : store.getLiveSSTables()) { + assertFalse(sstable.isRepaired()); + assertEquals(sstable.isPendingRepair() ? sessionID : NO_PENDING_REPAIR, sstable.getPendingRepair()); try (ISSTableScanner scanner = sstable.getScanner()) { while (scanner.hasNext()) { UnfilteredRowIterator row = scanner.next(); - if (sstable.isRepaired() || sstable.isPendingRepair()) + Token token = row.partitionKey().getToken(); + if (sstable.isPendingRepair() && !sstable.isTransient()) { - assertTrue(range.contains(row.partitionKey().getToken())); - repairedKeys += sstable.isRepaired() ? 1 : 0; - pendingKeys += sstable.isPendingRepair() ? 1 : 0; + assertTrue(fullContains.test(token)); + assertFalse(transContains.test(token)); + stats.pendingKeys++; + } + else if (sstable.isPendingRepair() && sstable.isTransient()) + { + + assertTrue(transContains.test(token)); + assertFalse(fullContains.test(token)); + stats.transKeys++; } else { - assertFalse(range.contains(row.partitionKey().getToken())); - nonRepairedKeys++; + assertFalse(fullContains.test(token)); + assertFalse(transContains.test(token)); + stats.unrepairedKeys++; } } } @@ -157,21 +195,40 @@ public class AntiCompactionTest assertEquals(1, sstable.selfRef().globalCount()); } assertEquals(0, store.getTracker().getCompacting().size()); - assertEquals(repairedKeys, repairedAt != UNREPAIRED_SSTABLE ? 4 : 0); - assertEquals(pendingKeys, pendingRepair != NO_PENDING_REPAIR ? 4 : 0); - assertEquals(nonRepairedKeys, 6); + return stats; } @Test - public void antiCompactOneRepairedAt() throws Exception + public void antiCompactOneFull() throws Exception { - antiCompactOne(1000, NO_PENDING_REPAIR); + ColumnFamilyStore store = prepareColumnFamilyStore(); + SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), NO_RANGES)); + assertEquals(2, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 4); + assertEquals(stats.transKeys, 0); + assertEquals(stats.unrepairedKeys, 6); } @Test - public void antiCompactOnePendingRepair() throws Exception + public void antiCompactOneMixed() throws Exception { - antiCompactOne(UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID()); + ColumnFamilyStore store = prepareColumnFamilyStore(); + SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8))); + assertEquals(3, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 4); + assertEquals(stats.transKeys, 4); + assertEquals(stats.unrepairedKeys, 2); + } + + @Test + public void antiCompactOneTransOnly() throws Exception + { + ColumnFamilyStore store = prepareColumnFamilyStore(); + SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4))); + assertEquals(2, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 0); + assertEquals(stats.transKeys, 4); + assertEquals(stats.unrepairedKeys, 6); } @Test @@ -190,7 +247,7 @@ public class AntiCompactionTest try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); Refs refs = Refs.ref(sstables)) { - CompactionManager.instance.performAnticompaction(cfs, ranges, refs, txn, 12345, NO_PENDING_REPAIR, parentRepairSession); + CompactionManager.instance.performAnticompaction(cfs, atEndpoint(ranges, NO_RANGES), refs, txn, parentRepairSession); } long sum = 0; long rows = 0; @@ -208,7 +265,7 @@ public class AntiCompactionTest File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { for (int i = 0; i < count; i++) { @@ -240,7 +297,7 @@ public class AntiCompactionTest } @Test - public void antiCompactTen() throws InterruptedException, IOException + public void antiCompactTenFull() throws InterruptedException, IOException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF); @@ -250,56 +307,59 @@ public class AntiCompactionTest { generateSStable(store,Integer.toString(table)); } - Collection sstables = getUnrepairedSSTables(store); - assertEquals(store.getLiveSSTables().size(), sstables.size()); - - Range range = new Range(new BytesToken("0".getBytes()), new BytesToken("4".getBytes())); - List> ranges = Arrays.asList(range); - - long repairedAt = 1000; - UUID parentRepairSession = UUID.randomUUID(); - registerParentRepairSession(parentRepairSession, ranges, repairedAt, null); - try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); - Refs refs = Refs.ref(sstables)) - { - CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, repairedAt, NO_PENDING_REPAIR, parentRepairSession); - } + SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), NO_RANGES)); /* Anticompaction will be anti-compacting 10 SSTables but will be doing this two at a time so there will be no net change in the number of sstables */ - assertEquals(10, store.getLiveSSTables().size()); - int repairedKeys = 0; - int nonRepairedKeys = 0; - for (SSTableReader sstable : store.getLiveSSTables()) - { - try (ISSTableScanner scanner = sstable.getScanner()) - { - while (scanner.hasNext()) - { - try (UnfilteredRowIterator row = scanner.next()) - { - if (sstable.isRepaired()) - { - assertTrue(range.contains(row.partitionKey().getToken())); - assertEquals(repairedAt, sstable.getSSTableMetadata().repairedAt); - repairedKeys++; - } - else - { - assertFalse(range.contains(row.partitionKey().getToken())); - assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt); - nonRepairedKeys++; - } - } - } - } - } - assertEquals(repairedKeys, 40); - assertEquals(nonRepairedKeys, 60); + assertEquals(10, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 40); + assertEquals(stats.transKeys, 0); + assertEquals(stats.unrepairedKeys, 60); } - private void shouldMutate(long repairedAt, UUID pendingRepair) throws InterruptedException, IOException + @Test + public void antiCompactTenTrans() throws InterruptedException, IOException + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF); + store.disableAutoCompaction(); + + for (int table = 0; table < 10; table++) + { + generateSStable(store,Integer.toString(table)); + } + SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4))); + /* + Anticompaction will be anti-compacting 10 SSTables but will be doing this two at a time + so there will be no net change in the number of sstables + */ + assertEquals(10, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 0); + assertEquals(stats.transKeys, 40); + assertEquals(stats.unrepairedKeys, 60); + } + + @Test + public void antiCompactTenMixed() throws InterruptedException, IOException + { + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF); + store.disableAutoCompaction(); + + for (int table = 0; table < 10; table++) + { + generateSStable(store,Integer.toString(table)); + } + SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8))); + assertEquals(15, stats.numLiveSSTables); + assertEquals(stats.pendingKeys, 40); + assertEquals(stats.transKeys, 40); + assertEquals(stats.unrepairedKeys, 20); + } + + @Test + public void shouldMutatePendingRepair() throws InterruptedException, IOException { ColumnFamilyStore store = prepareColumnFamilyStore(); Collection sstables = getUnrepairedSSTables(store); @@ -307,34 +367,22 @@ public class AntiCompactionTest // the sstables start at "0".getBytes() = 48, we need to include that first token, with "/".getBytes() = 47 Range range = new Range(new BytesToken("/".getBytes()), new BytesToken("9999".getBytes())); List> ranges = Arrays.asList(range); - UUID parentRepairSession = pendingRepair == null ? UUID.randomUUID() : pendingRepair; - registerParentRepairSession(parentRepairSession, ranges, repairedAt, pendingRepair); + UUID pendingRepair = UUID.randomUUID(); + registerParentRepairSession(pendingRepair, ranges, UNREPAIRED_SSTABLE, pendingRepair); try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); Refs refs = Refs.ref(sstables)) { - CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, repairedAt, pendingRepair, parentRepairSession); + CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, pendingRepair); } assertThat(store.getLiveSSTables().size(), is(1)); - assertThat(Iterables.get(store.getLiveSSTables(), 0).isRepaired(), is(repairedAt != UNREPAIRED_SSTABLE)); - assertThat(Iterables.get(store.getLiveSSTables(), 0).isPendingRepair(), is(pendingRepair != NO_PENDING_REPAIR)); + assertThat(Iterables.get(store.getLiveSSTables(), 0).isRepaired(), is(false)); + assertThat(Iterables.get(store.getLiveSSTables(), 0).isPendingRepair(), is(true)); assertThat(Iterables.get(store.getLiveSSTables(), 0).selfRef().globalCount(), is(1)); assertThat(store.getTracker().getCompacting().size(), is(0)); } - @Test - public void shouldMutateRepairedAt() throws InterruptedException, IOException - { - shouldMutate(1, NO_PENDING_REPAIR); - } - - @Test - public void shouldMutatePendingRepair() throws InterruptedException, IOException - { - shouldMutate(UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID()); - } - @Test public void shouldSkipAntiCompactionForNonIntersectingRange() throws InterruptedException, IOException { @@ -358,7 +406,7 @@ public class AntiCompactionTest try (LifecycleTransaction txn = store.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); Refs refs = Refs.ref(sstables)) { - CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, 1, NO_PENDING_REPAIR, parentRepairSession); + CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, parentRepairSession); } catch (IllegalStateException e) { @@ -428,7 +476,7 @@ public class AntiCompactionTest Assert.assertFalse(refs.isEmpty()); try { - CompactionManager.instance.performAnticompaction(store, ranges, refs, txn, 1, missingRepairSession, missingRepairSession); + CompactionManager.instance.performAnticompaction(store, atEndpoint(ranges, NO_RANGES), refs, txn, missingRepairSession); Assert.fail("expected RuntimeException"); } catch (RuntimeException e) @@ -484,8 +532,7 @@ public class AntiCompactionTest Range r = new Range<>(t(9), t(100)); // sstable is not intersecting and should not be included - Iterator sstableIterator = sstables.iterator(); - CompactionManager.findSSTablesToAnticompact(sstableIterator, Collections.singletonList(r), UUID.randomUUID()); + CompactionManager.validateSSTableBoundsForAnticompaction(UUID.randomUUID(), sstables, atEndpoint(Collections.singletonList(r), NO_RANGES)); } @Test(expected = IllegalStateException.class) @@ -500,8 +547,7 @@ public class AntiCompactionTest Range r = new Range<>(t(10), t(11)); // no sstable included, throw - Iterator sstableIterator = sstables.iterator(); - CompactionManager.findSSTablesToAnticompact(sstableIterator, Collections.singletonList(r), UUID.randomUUID()); + CompactionManager.validateSSTableBoundsForAnticompaction(UUID.randomUUID(), sstables, atEndpoint(Collections.singletonList(r), NO_RANGES)); } @Test diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java index c7f1ae88ac..267c2e416e 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.db.compaction; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -42,29 +41,34 @@ import org.apache.cassandra.utils.FBUtilities; public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingRepairTest { - private static boolean strategiesContain(Collection strategies, SSTableReader sstable) + private boolean transientContains(SSTableReader sstable) { - return Iterables.any(strategies, strategy -> strategy.getSSTables().contains(sstable)); - } - - private boolean pendingContains(UUID id, SSTableReader sstable) - { - return Iterables.any(csm.getPendingRepairManagers(), p -> p.get(id) != null && p.get(id).getSSTables().contains(sstable)); + return csm.getTransientRepairsUnsafe().containsSSTable(sstable); } private boolean pendingContains(SSTableReader sstable) { - return Iterables.any(csm.getPendingRepairManagers(), p -> strategiesContain(p.getStrategies(), sstable)); + return csm.getPendingRepairsUnsafe().containsSSTable(sstable); } private boolean repairedContains(SSTableReader sstable) { - return strategiesContain(csm.getRepaired(), sstable); + return csm.getRepairedUnsafe().containsSSTable(sstable); } private boolean unrepairedContains(SSTableReader sstable) { - return strategiesContain(csm.getUnrepaired(), sstable); + return csm.getUnrepairedUnsafe().containsSSTable(sstable); + } + + private boolean hasPendingStrategiesFor(UUID sessionID) + { + return !Iterables.isEmpty(csm.getPendingRepairsUnsafe().getStrategiesFor(sessionID)); + } + + private boolean hasTransientStrategiesFor(UUID sessionID) + { + return !Iterables.isEmpty(csm.getTransientRepairsUnsafe().getStrategiesFor(sessionID)); } /** @@ -75,23 +79,25 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR { UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); - Assert.assertTrue(csm.pendingRepairs().isEmpty()); + Assert.assertTrue(Iterables.isEmpty(csm.getPendingRepairsUnsafe().allStrategies())); SSTableReader sstable = makeSSTable(true); Assert.assertFalse(sstable.isRepaired()); Assert.assertFalse(sstable.isPendingRepair()); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); Assert.assertFalse(sstable.isRepaired()); Assert.assertTrue(sstable.isPendingRepair()); - csm.getForPendingRepair(repairID).forEach(Assert::assertNull); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); // add the sstable csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); Assert.assertFalse(repairedContains(sstable)); Assert.assertFalse(unrepairedContains(sstable)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNotNull); - Assert.assertTrue(pendingContains(repairID, sstable)); + Assert.assertTrue(pendingContains(sstable)); + Assert.assertTrue(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); } @Test @@ -101,16 +107,17 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable1 = makeSSTable(true); - mutateRepaired(sstable1, repairID); + mutateRepaired(sstable1, repairID, false); SSTableReader sstable2 = makeSSTable(true); - mutateRepaired(sstable2, repairID); + mutateRepaired(sstable2, repairID, false); Assert.assertFalse(repairedContains(sstable1)); Assert.assertFalse(unrepairedContains(sstable1)); Assert.assertFalse(repairedContains(sstable2)); Assert.assertFalse(unrepairedContains(sstable2)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNull); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); // add only SSTableListChangedNotification notification; @@ -119,13 +126,14 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR OperationType.COMPACTION); csm.handleNotification(notification, cfs.getTracker()); - csm.getForPendingRepair(repairID).forEach(Assert::assertNotNull); Assert.assertFalse(repairedContains(sstable1)); Assert.assertFalse(unrepairedContains(sstable1)); - Assert.assertTrue(pendingContains(repairID, sstable1)); + Assert.assertTrue(pendingContains(sstable1)); Assert.assertFalse(repairedContains(sstable2)); Assert.assertFalse(unrepairedContains(sstable2)); - Assert.assertFalse(pendingContains(repairID, sstable2)); + Assert.assertFalse(pendingContains(sstable2)); + Assert.assertTrue(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); // remove and add notification = new SSTableListChangedNotification(Collections.singleton(sstable2), @@ -135,10 +143,10 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR Assert.assertFalse(repairedContains(sstable1)); Assert.assertFalse(unrepairedContains(sstable1)); - Assert.assertFalse(pendingContains(repairID, sstable1)); + Assert.assertFalse(pendingContains(sstable1)); Assert.assertFalse(repairedContains(sstable2)); Assert.assertFalse(unrepairedContains(sstable2)); - Assert.assertTrue(pendingContains(repairID, sstable2)); + Assert.assertTrue(pendingContains(sstable2)); } @Test @@ -151,18 +159,20 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR SSTableReader sstable = makeSSTable(false); Assert.assertTrue(unrepairedContains(sstable)); Assert.assertFalse(repairedContains(sstable)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNull); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); SSTableRepairStatusChanged notification; // change to pending repaired - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); notification = new SSTableRepairStatusChanged(Collections.singleton(sstable)); csm.handleNotification(notification, cfs.getTracker()); Assert.assertFalse(unrepairedContains(sstable)); Assert.assertFalse(repairedContains(sstable)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNotNull); - Assert.assertTrue(pendingContains(repairID, sstable)); + Assert.assertTrue(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); + Assert.assertTrue(pendingContains(sstable)); // change to repaired mutateRepaired(sstable, System.currentTimeMillis()); @@ -170,7 +180,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR csm.handleNotification(notification, cfs.getTracker()); Assert.assertFalse(unrepairedContains(sstable)); Assert.assertTrue(repairedContains(sstable)); - Assert.assertFalse(pendingContains(repairID, sstable)); + Assert.assertFalse(pendingContains(sstable)); } @Test @@ -180,14 +190,14 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); - Assert.assertTrue(pendingContains(repairID, sstable)); + Assert.assertTrue(pendingContains(sstable)); // delete sstable SSTableDeletingNotification notification = new SSTableDeletingNotification(sstable); csm.handleNotification(notification, cfs.getTracker()); - Assert.assertFalse(pendingContains(repairID, sstable)); + Assert.assertFalse(pendingContains(sstable)); Assert.assertFalse(unrepairedContains(sstable)); Assert.assertFalse(repairedContains(sstable)); } @@ -209,7 +219,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR Assert.assertTrue(strategies.get(2).isEmpty()); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); strategies = csm.getStrategies(); @@ -227,11 +237,12 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); LocalSessionAccessor.finalizeUnsafe(repairID); - csm.getForPendingRepair(repairID).forEach(Assert::assertNotNull); - Assert.assertNotNull(pendingContains(repairID, sstable)); + Assert.assertTrue(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); + Assert.assertTrue(pendingContains(sstable)); Assert.assertTrue(sstable.isPendingRepair()); Assert.assertFalse(sstable.isRepaired()); @@ -245,7 +256,9 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR Assert.assertTrue(repairedContains(sstable)); Assert.assertFalse(unrepairedContains(sstable)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNull); + Assert.assertFalse(pendingContains(sstable)); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); // sstable should have pendingRepair cleared, and repairedAt set correctly long expectedRepairedAt = ActiveRepairService.instance.getParentRepairSession(repairID).repairedAt; @@ -264,12 +277,13 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); LocalSessionAccessor.failUnsafe(repairID); - csm.getForPendingRepair(repairID).forEach(Assert::assertNotNull); - Assert.assertNotNull(pendingContains(repairID, sstable)); + Assert.assertTrue(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); + Assert.assertTrue(pendingContains(sstable)); Assert.assertTrue(sstable.isPendingRepair()); Assert.assertFalse(sstable.isRepaired()); @@ -283,11 +297,78 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR Assert.assertFalse(repairedContains(sstable)); Assert.assertTrue(unrepairedContains(sstable)); - csm.getForPendingRepair(repairID).forEach(Assert::assertNull); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); // sstable should have pendingRepair cleared, and repairedAt set correctly Assert.assertFalse(sstable.isPendingRepair()); Assert.assertFalse(sstable.isRepaired()); Assert.assertEquals(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getSSTableMetadata().repairedAt); } + + @Test + public void finalizedSessionTransientCleanup() + { + Assert.assertTrue(cfs.getLiveSSTables().isEmpty()); + UUID repairID = registerSession(cfs, true, true); + LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); + SSTableReader sstable = makeSSTable(true); + mutateRepaired(sstable, repairID, true); + csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); + LocalSessionAccessor.finalizeUnsafe(repairID); + + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertTrue(hasTransientStrategiesFor(repairID)); + Assert.assertTrue(transientContains(sstable)); + Assert.assertFalse(pendingContains(sstable)); + Assert.assertFalse(repairedContains(sstable)); + Assert.assertFalse(unrepairedContains(sstable)); + + cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task + AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds()); + Assert.assertNotNull(compactionTask); + Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass()); + + // run the compaction + compactionTask.execute(null); + + Assert.assertTrue(cfs.getLiveSSTables().isEmpty()); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); + } + + @Test + public void failedSessionTransientCleanup() + { + Assert.assertTrue(cfs.getLiveSSTables().isEmpty()); + UUID repairID = registerSession(cfs, true, true); + LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); + SSTableReader sstable = makeSSTable(true); + mutateRepaired(sstable, repairID, true); + csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker()); + LocalSessionAccessor.failUnsafe(repairID); + + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertTrue(hasTransientStrategiesFor(repairID)); + Assert.assertTrue(transientContains(sstable)); + Assert.assertFalse(pendingContains(sstable)); + Assert.assertFalse(repairedContains(sstable)); + Assert.assertFalse(unrepairedContains(sstable)); + + cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task + AbstractCompactionTask compactionTask = csm.getNextBackgroundTask(FBUtilities.nowInSeconds()); + Assert.assertNotNull(compactionTask); + Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass()); + + // run the compaction + compactionTask.execute(null); + + Assert.assertFalse(cfs.getLiveSSTables().isEmpty()); + Assert.assertFalse(hasPendingStrategiesFor(repairID)); + Assert.assertFalse(hasTransientStrategiesFor(repairID)); + Assert.assertFalse(transientContains(sstable)); + Assert.assertFalse(pendingContains(sstable)); + Assert.assertFalse(repairedContains(sstable)); + Assert.assertTrue(unrepairedContains(sstable)); + } } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java index eeaaf5b76f..73e6852eb1 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java @@ -129,12 +129,12 @@ public class CompactionStrategyManagerTest if (i % 3 == 0) { //make 1 third of sstables repaired - cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null); + cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null, false); } else if (i % 3 == 1) { //make 1 third of sstables pending repair - cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, UUIDGen.getTimeUUID()); + cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, UUIDGen.getTimeUUID(), false); } previousSSTables = currentSSTables; } @@ -272,19 +272,19 @@ public class CompactionStrategyManagerTest DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false); } - private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, Class expectedType) + private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, boolean isTransient, Class expectedType) { ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX); CompactionStrategyManager csm = cfs.getCompactionStrategyManager(); - AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair); + AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair, isTransient); assertNotNull(holder); assertSame(expectedType, holder.getClass()); int matches = 0; for (AbstractStrategyHolder other : csm.getHolders()) { - if (other.managesRepairedGroup(isRepaired, isPendingRepair)) + if (other.managesRepairedGroup(isRepaired, isPendingRepair, isTransient)) { assertSame("holder assignment should be mutually exclusive", holder, other); matches++; @@ -293,13 +293,13 @@ public class CompactionStrategyManagerTest assertEquals(1, matches); } - private static void assertInvalieHolderConfig(boolean isRepaired, boolean isPendingRepair) + private static void assertInvalieHolderConfig(boolean isRepaired, boolean isPendingRepair, boolean isTransient) { ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX); CompactionStrategyManager csm = cfs.getCompactionStrategyManager(); try { - csm.getHolder(isRepaired, isPendingRepair); + csm.getHolder(isRepaired, isPendingRepair, isTransient); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) @@ -315,10 +315,14 @@ public class CompactionStrategyManagerTest @Test public void testMutualExclusiveHolderClassification() throws Exception { - assertHolderExclusivity(false, false, CompactionStrategyHolder.class); - assertHolderExclusivity(true, false, CompactionStrategyHolder.class); - assertHolderExclusivity(false, true, PendingRepairHolder.class); - assertInvalieHolderConfig(true, true); + assertHolderExclusivity(false, false, false, CompactionStrategyHolder.class); + assertHolderExclusivity(true, false, false, CompactionStrategyHolder.class); + assertHolderExclusivity(false, true, false, PendingRepairHolder.class); + assertHolderExclusivity(false, true, true, PendingRepairHolder.class); + assertInvalieHolderConfig(true, true, false); + assertInvalieHolderConfig(true, true, true); + assertInvalieHolderConfig(false, false, true); + assertInvalieHolderConfig(true, false, true); } PartitionPosition forKey(int key) @@ -337,20 +341,23 @@ public class CompactionStrategyManagerTest ColumnFamilyStore cfs = createJBODMockCFS(numDir); Keyspace.open(cfs.keyspace.getName()).getColumnFamilyStore(cfs.name).disableAutoCompaction(); assertTrue(cfs.getLiveSSTables().isEmpty()); - List unrepaired = new ArrayList<>(); + List transientRepairs = new ArrayList<>(); List pendingRepair = new ArrayList<>(); + List unrepaired = new ArrayList<>(); List repaired = new ArrayList<>(); for (int i = 0; i < numDir; i++) { int key = 100 * i; - unrepaired.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++)); + transientRepairs.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++)); pendingRepair.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++)); + unrepaired.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++)); repaired.add(createSSTableWithKey(cfs.keyspace.getName(), cfs.name, key++)); } - cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, UUID.randomUUID()); - cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null); + cfs.getCompactionStrategyManager().mutateRepaired(transientRepairs, 0, UUID.randomUUID(), true); + cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, UUID.randomUUID(), false); + cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null, false); DiskBoundaries boundaries = new DiskBoundaries(cfs.getDirectories().getWriteableLocations(), Lists.newArrayList(forKey(100), forKey(200), forKey(300)), @@ -358,7 +365,7 @@ public class CompactionStrategyManagerTest CompactionStrategyManager csm = new CompactionStrategyManager(cfs, () -> boundaries, true); - List grouped = csm.groupSSTables(Iterables.concat(repaired, pendingRepair, unrepaired)); + List grouped = csm.groupSSTables(Iterables.concat( transientRepairs, pendingRepair, repaired, unrepaired)); for (int x=0; x toCompact = new ArrayList<>(sstables); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java index c91d2fed83..857fa32998 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java @@ -411,7 +411,7 @@ public class CompactionsCQLTest extends CQLTester cfs.forceBlockingFlush(); } assertEquals(50, cfs.getLiveSSTables().size()); - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepaired().get(0); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); AbstractCompactionTask act = lcs.getNextBackgroundTask(0); // we should be compacting all 50 sstables: assertEquals(50, act.transaction.originals().size()); @@ -445,7 +445,7 @@ public class CompactionsCQLTest extends CQLTester // mark the L1 sstable as compacting to make sure we trigger STCS in L0: LifecycleTransaction txn = cfs.getTracker().tryModify(l1sstable, OperationType.COMPACTION); - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepaired().get(0); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); AbstractCompactionTask act = lcs.getNextBackgroundTask(0); // note that max_threshold is 60 (more than the amount of L0 sstables), but MAX_COMPACTING_L0 is 32, which means we will trigger STCS with at most max_threshold sstables assertEquals(50, act.transaction.originals().size()); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 9ebe326c09..23e88fe719 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -364,7 +364,7 @@ public class LeveledCompactionStrategyTest SSTableReader sstable1 = unrepaired.manifest.generations[2].get(0); SSTableReader sstable2 = unrepaired.manifest.generations[1].get(0); - sstable1.descriptor.getMetadataSerializer().mutateRepaired(sstable1.descriptor, System.currentTimeMillis(), null); + sstable1.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable1.descriptor, System.currentTimeMillis(), null, false); sstable1.reloadSSTableMetadata(); assertTrue(sstable1.isRepaired()); diff --git a/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java index 2b88c9c164..d83e0633ea 100644 --- a/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java @@ -45,7 +45,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); @@ -63,7 +63,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); LocalSessionAccessor.finalizeUnsafe(repairID); @@ -82,7 +82,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); LocalSessionAccessor.failUnsafe(repairID); @@ -94,7 +94,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest public void needsCleanupNoSession() { UUID fakeID = UUIDGen.getTimeUUID(); - PendingRepairManager prm = new PendingRepairManager(cfs, null); + PendingRepairManager prm = new PendingRepairManager(cfs, null, false); Assert.assertTrue(prm.canCleanup(fakeID)); } @@ -106,7 +106,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); @@ -122,7 +122,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); Assert.assertNotNull(prm.get(repairID)); @@ -140,13 +140,13 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); LocalSessionAccessor.finalizeUnsafe(repairID); @@ -184,7 +184,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); Assert.assertNotNull(prm.get(repairID)); @@ -202,7 +202,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest UUID repairID = registerSession(cfs, true, true); LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertNotNull(prm.get(repairID)); Assert.assertNotNull(prm.get(repairID)); @@ -225,7 +225,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest PendingRepairManager prm = csm.getPendingRepairManagers().get(0); UUID repairId = registerSession(cfs, true, true); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairId); + mutateRepaired(sstable, repairId, false); prm.addSSTable(sstable); List tasks = csm.getUserDefinedTasks(Collections.singleton(sstable), 100); try @@ -247,8 +247,8 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest SSTableReader sstable = makeSSTable(true); SSTableReader sstable2 = makeSSTable(true); - mutateRepaired(sstable, repairId); - mutateRepaired(sstable2, repairId2); + mutateRepaired(sstable, repairId, false); + mutateRepaired(sstable2, repairId2, false); prm.addSSTable(sstable); prm.addSSTable(sstable2); List tasks = csm.getUserDefinedTasks(Lists.newArrayList(sstable, sstable2), 100); @@ -296,7 +296,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest Assert.assertFalse(prm.hasDataForSession(repairID)); SSTableReader sstable = makeSSTable(true); - mutateRepaired(sstable, repairID); + mutateRepaired(sstable, repairID, false); prm.addSSTable(sstable); Assert.assertTrue(prm.hasDataForSession(repairID)); } diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java index 6428ab70bf..1292b7e521 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java @@ -56,7 +56,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester assertEquals(1, cfs.getLiveSSTables().size()); cfs.getLiveSSTables().forEach(s -> assertEquals(2, s.getSSTableLevel())); // make sure compaction strategy is notified: - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepaired().iterator().next(); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); for (int i = 0; i < lcs.manifest.getLevelCount(); i++) { if (i == 2) @@ -98,7 +98,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester cfs.forceBlockingFlush(); } // now we have a bunch of data in L0, first compaction will be a normal one, containing all sstables: - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepaired().get(0); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); AbstractCompactionTask act = lcs.getNextBackgroundTask(0); act.execute(null); @@ -148,7 +148,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester assertEquals(1, cfs.getLiveSSTables().size()); for (SSTableReader sst : cfs.getLiveSSTables()) assertEquals(0, sst.getSSTableMetadata().sstableLevel); - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepaired().iterator().next(); + LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); assertEquals(1, lcs.getLevelSize(0)); assertTrue(cfs.getTracker().getCompacting().isEmpty()); } diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index 5694e861a3..e5ff1388e9 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -1183,7 +1183,7 @@ public class LogTransactionTest extends AbstractTransactionalTest SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, header) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java index 5cda2ad195..b7b7d4ac6f 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java @@ -164,6 +164,7 @@ public class RealTransactionsTest extends SchemaLoader 0, 0, null, + false, 0, SerializationHeader.make(cfs.metadata(), txn.originals()), cfs.indexManager.listIndexes(), diff --git a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java index 76ebfd8b37..3b29cc5b50 100644 --- a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java +++ b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java @@ -119,11 +119,11 @@ public class CompactionManagerGetSSTablesForValidationTest Iterator iter = cfs.getLiveSSTables().iterator(); repaired = iter.next(); - repaired.descriptor.getMetadataSerializer().mutateRepaired(repaired.descriptor, System.currentTimeMillis(), null); + repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, System.currentTimeMillis(), null, false); repaired.reloadSSTableMetadata(); pendingRepair = iter.next(); - pendingRepair.descriptor.getMetadataSerializer().mutateRepaired(pendingRepair.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sessionID); + pendingRepair.descriptor.getMetadataSerializer().mutateRepairMetadata(pendingRepair.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE, sessionID, false); pendingRepair.reloadSSTableMetadata(); unrepaired = iter.next(); diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java index 447d50444d..374a7602a1 100644 --- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db.repair; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -42,6 +43,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Schema; @@ -64,6 +67,9 @@ public class PendingAntiCompactionTest { private static final Logger logger = LoggerFactory.getLogger(PendingAntiCompactionTest.class); private static final Collection> FULL_RANGE; + private static final Collection> NO_RANGES = Collections.emptyList(); + private static InetAddressAndPort local; + static { DatabaseDescriptor.daemonInitialization(); @@ -77,9 +83,10 @@ public class PendingAntiCompactionTest private ColumnFamilyStore cfs; @BeforeClass - public static void setupClass() + public static void setupClass() throws Throwable { SchemaLoader.prepareServer(); + local = InetAddressAndPort.getByName("127.0.0.1"); } @Before @@ -89,6 +96,7 @@ public class PendingAntiCompactionTest cfm = CreateTableStatement.parse(String.format("CREATE TABLE %s.%s (k INT PRIMARY KEY, v INT)", ks, tbl), ks).build(); SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1), cfm); cfs = Schema.instance.getColumnFamilyStoreInstance(cfm.id); + } private void makeSSTables(int num) @@ -105,7 +113,7 @@ public class PendingAntiCompactionTest private static class InstrumentedAcquisitionCallback extends PendingAntiCompaction.AcquisitionCallback { - public InstrumentedAcquisitionCallback(UUID parentRepairSession, Collection> ranges) + public InstrumentedAcquisitionCallback(UUID parentRepairSession, RangesAtEndpoint ranges) { super(parentRepairSession, ranges); } @@ -155,7 +163,7 @@ public class PendingAntiCompactionTest ExecutorService executor = Executors.newSingleThreadExecutor(); try { - pac = new PendingAntiCompaction(sessionID, tables, ranges, executor); + pac = new PendingAntiCompaction(sessionID, tables, atEndpoint(ranges, NO_RANGES), executor); pac.run().get(); } finally @@ -217,7 +225,7 @@ public class PendingAntiCompactionTest Assert.assertTrue(repaired.intersects(FULL_RANGE)); Assert.assertTrue(unrepaired.intersects(FULL_RANGE)); - repaired.descriptor.getMetadataSerializer().mutateRepaired(repaired.descriptor, 1, null); + repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, 1, null, false); repaired.reloadSSTableMetadata(); PendingAntiCompaction.AcquisitionCallable acquisitionCallable = new PendingAntiCompaction.AcquisitionCallable(cfs, FULL_RANGE, UUIDGen.getTimeUUID()); @@ -243,7 +251,7 @@ public class PendingAntiCompactionTest Assert.assertTrue(repaired.intersects(FULL_RANGE)); Assert.assertTrue(unrepaired.intersects(FULL_RANGE)); - repaired.descriptor.getMetadataSerializer().mutateRepaired(repaired.descriptor, 0, UUIDGen.getTimeUUID()); + repaired.descriptor.getMetadataSerializer().mutateRepairMetadata(repaired.descriptor, 0, UUIDGen.getTimeUUID(), false); repaired.reloadSSTableMetadata(); Assert.assertTrue(repaired.isPendingRepair()); @@ -284,7 +292,7 @@ public class PendingAntiCompactionTest PendingAntiCompaction.AcquireResult result = acquisitionCallable.call(); Assert.assertNotNull(result); - InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), FULL_RANGE); + InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, NO_RANGES)); Assert.assertTrue(cb.submittedCompactions.isEmpty()); cb.apply(Lists.newArrayList(result)); @@ -308,7 +316,7 @@ public class PendingAntiCompactionTest Assert.assertNotNull(result); Assert.assertEquals(Transactional.AbstractTransactional.State.IN_PROGRESS, result.txn.state()); - InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), FULL_RANGE); + InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, Collections.emptyList())); Assert.assertTrue(cb.submittedCompactions.isEmpty()); cb.apply(Lists.newArrayList(result, null)); @@ -333,7 +341,7 @@ public class PendingAntiCompactionTest ColumnFamilyStore cfs2 = Schema.instance.getColumnFamilyStoreInstance(Schema.instance.getTableMetadata("system", "peers").id); PendingAntiCompaction.AcquireResult fakeResult = new PendingAntiCompaction.AcquireResult(cfs2, null, null); - InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), FULL_RANGE); + InstrumentedAcquisitionCallback cb = new InstrumentedAcquisitionCallback(UUIDGen.getTimeUUID(), atEndpoint(FULL_RANGE, NO_RANGES)); Assert.assertTrue(cb.submittedCompactions.isEmpty()); cb.apply(Lists.newArrayList(result, fakeResult)); @@ -359,8 +367,19 @@ public class PendingAntiCompactionTest true,0, true, PreviewKind.NONE); - CompactionManager.instance.performAnticompaction(result.cfs, FULL_RANGE, result.refs, result.txn, - ActiveRepairService.UNREPAIRED_SSTABLE, sessionID, sessionID); + CompactionManager.instance.performAnticompaction(result.cfs, atEndpoint(FULL_RANGE, NO_RANGES), result.refs, result.txn, sessionID); } + + private static RangesAtEndpoint atEndpoint(Collection> full, Collection> trans) + { + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : full) + builder.add(new Replica(local, range, true)); + + for (Range range : trans) + builder.add(new Replica(local, range, false)); + + return builder.build(); + } } diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java index 8256ac6548..5e443463d5 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java @@ -114,6 +114,7 @@ public class CassandraOutgoingFileTest List> requestedRanges = Arrays.asList(new Range<>(store.getPartitioner().getMinimumToken(), getTokenAtIndex(4)), new Range<>(getTokenAtIndex(2), getTokenAtIndex(6)), new Range<>(getTokenAtIndex(5), sstable.last.getToken())); + requestedRanges = Range.normalize(requestedRanges); CassandraOutgoingFile cof = new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.ref(), sstable.getPositionsForRanges(requestedRanges), diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java index 86018af2c0..b597bfeb5a 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java @@ -33,6 +33,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -118,10 +120,10 @@ public class CassandraStreamManagerTest return Iterables.getOnlyElement(diff); } - private static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair) throws IOException + private static void mutateRepaired(SSTableReader sstable, long repairedAt, UUID pendingRepair, boolean isTransient) throws IOException { Descriptor descriptor = sstable.descriptor; - descriptor.getMetadataSerializer().mutateRepaired(descriptor, repairedAt, pendingRepair); + descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, repairedAt, pendingRepair, isTransient); sstable.reloadSSTableMetadata(); } @@ -141,7 +143,7 @@ public class CassandraStreamManagerTest private Set getReadersForRange(Range range) { Collection streams = cfs.getStreamManager().createOutgoingStreams(session(NO_PENDING_REPAIR), - Collections.singleton(range), + RangesAtEndpoint.toDummyList(Collections.singleton(range)), NO_PENDING_REPAIR, PreviewKind.NONE); return sstablesFromStreams(streams); @@ -151,7 +153,7 @@ public class CassandraStreamManagerTest { IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); Collection> ranges = Lists.newArrayList(new Range(partitioner.getMinimumToken(), partitioner.getMinimumToken())); - Collection streams = cfs.getStreamManager().createOutgoingStreams(session(pendingRepair), ranges, pendingRepair, PreviewKind.NONE); + Collection streams = cfs.getStreamManager().createOutgoingStreams(session(pendingRepair), RangesAtEndpoint.toDummyList(ranges), pendingRepair, PreviewKind.NONE); return sstablesFromStreams(streams); } @@ -167,9 +169,9 @@ public class CassandraStreamManagerTest UUID pendingRepair = UUIDGen.getTimeUUID(); long repairedAt = System.currentTimeMillis(); - mutateRepaired(sstable2, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair); - mutateRepaired(sstable3, ActiveRepairService.UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID()); - mutateRepaired(sstable4, repairedAt, NO_PENDING_REPAIR); + mutateRepaired(sstable2, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, false); + mutateRepaired(sstable3, ActiveRepairService.UNREPAIRED_SSTABLE, UUIDGen.getTimeUUID(), false); + mutateRepaired(sstable4, repairedAt, NO_PENDING_REPAIR, false); diff --git a/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java b/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java new file mode 100644 index 0000000000..2a6cb65a39 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/streaming/StreamRequestTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.streaming; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.streaming.StreamRequest; + +public class StreamRequestTest +{ + private static InetAddressAndPort local; + private final String ks = "keyspace"; + private final int version = MessagingService.current_version; + + @BeforeClass + public static void setUp() throws Throwable + { + DatabaseDescriptor.daemonInitialization(); + local = InetAddressAndPort.getByName("127.0.0.1"); + } + + @Test + public void serializationRoundTrip() throws Throwable + { + StreamRequest orig = new StreamRequest(ks, + atEndpoint(Arrays.asList(range(1, 2), range(3, 4), range(5, 6)), + Collections.emptyList()), + atEndpoint(Collections.emptyList(), + Arrays.asList(range(5, 6), range(7, 8))), + Arrays.asList("a", "b", "c")); + + int expectedSize = (int) StreamRequest.serializer.serializedSize(orig, version); + try (DataOutputBuffer out = new DataOutputBuffer(expectedSize)) + { + StreamRequest.serializer.serialize(orig, out, version); + Assert.assertEquals(expectedSize, out.buffer().limit()); + try (DataInputBuffer in = new DataInputBuffer(out.buffer(), false)) + { + StreamRequest decoded = StreamRequest.serializer.deserialize(in, version); + + Assert.assertEquals(orig.keyspace, decoded.keyspace); + Assert.assertEquals(orig.full, decoded.full); + Assert.assertEquals(orig.transientReplicas, decoded.transientReplicas); + Assert.assertEquals(orig.columnFamilies, decoded.columnFamilies); + } + } + } + + private static RangesAtEndpoint atEndpoint(Collection> full, Collection> trans) + { + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : full) + builder.add(new Replica(local, range, true)); + + for (Range range : trans) + builder.add(new Replica(local, range, false)); + + return builder.build(); + } + + private static Range range(int l, int r) + { + return new Range<>(new ByteOrderedPartitioner.BytesToken(Integer.toString(l).getBytes()), + new ByteOrderedPartitioner.BytesToken(Integer.toString(r).getBytes())); + } +} diff --git a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java index ec0f6b1d74..7eebef75a5 100644 --- a/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java +++ b/test/unit/org/apache/cassandra/db/view/ViewUtilsTest.java @@ -28,6 +28,7 @@ import org.junit.Test; import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; @@ -76,12 +77,12 @@ public class ViewUtilsTest KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap)); Schema.instance.load(meta); - Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", - new StringToken("CA"), - new StringToken("BB")); + Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", + new StringToken("CA"), + new StringToken("BB")); Assert.assertTrue(naturalEndpoint.isPresent()); - Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.2"), naturalEndpoint.get()); + Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.2"), naturalEndpoint.get().endpoint()); } @@ -109,12 +110,12 @@ public class ViewUtilsTest KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap)); Schema.instance.load(meta); - Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", - new StringToken("CA"), - new StringToken("BB")); + Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", + new StringToken("CA"), + new StringToken("BB")); Assert.assertTrue(naturalEndpoint.isPresent()); - Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.1"), naturalEndpoint.get()); + Assert.assertEquals(InetAddressAndPort.getByName("127.0.0.1"), naturalEndpoint.get().endpoint()); } @Test @@ -141,9 +142,9 @@ public class ViewUtilsTest KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap)); Schema.instance.load(meta); - Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", - new StringToken("AB"), - new StringToken("BB")); + Optional naturalEndpoint = ViewUtils.getViewNaturalEndpoint("Keyspace1", + new StringToken("AB"), + new StringToken("BB")); Assert.assertFalse(naturalEndpoint.isPresent()); } diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index f11cb62f0b..8ae6853dcf 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -18,18 +18,20 @@ package org.apache.cassandra.dht; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; import java.net.UnknownHostException; import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.UUID; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import org.apache.cassandra.dht.RangeStreamer.FetchReplica; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.junit.AfterClass; @@ -41,6 +43,7 @@ import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.tokenallocator.TokenAllocation; @@ -60,6 +63,7 @@ public class BootStrapperTest { static IPartitioner oldPartitioner; + static Predicate originalAlivePredicate = RangeStreamer.ALIVE_PREDICATE; @BeforeClass public static void setup() throws ConfigurationException { @@ -68,12 +72,14 @@ public class BootStrapperTest SchemaLoader.startGossiper(); SchemaLoader.prepareServer(); SchemaLoader.schemaDefinition("BootStrapperTest"); + RangeStreamer.ALIVE_PREDICATE = Predicates.alwaysTrue(); } @AfterClass public static void tearDown() { DatabaseDescriptor.setPartitionerUnsafe(oldPartitioner); + RangeStreamer.ALIVE_PREDICATE = originalAlivePredicate; } @Test @@ -82,7 +88,7 @@ public class BootStrapperTest final int[] clusterSizes = new int[] { 1, 3, 5, 10, 100}; for (String keyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) { - int replicationFactor = Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor(); + int replicationFactor = Keyspace.open(keyspaceName).getReplicationStrategy().getReplicationFactor().allReplicas; for (int clusterSize : clusterSizes) if (clusterSize >= replicationFactor) testSourceTargetComputation(keyspaceName, clusterSize, replicationFactor); @@ -115,21 +121,25 @@ public class BootStrapperTest public void forceConviction(InetAddressAndPort ep) { throw new UnsupportedOperationException(); } }; s.addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(mockFailureDetector)); + assertNotNull(Keyspace.open(keyspaceName)); s.addRanges(keyspaceName, Keyspace.open(keyspaceName).getReplicationStrategy().getPendingAddressRanges(tmd, myToken, myEndpoint)); - Collection>>> toFetch = s.toFetch().get(keyspaceName); + + Collection> toFetch = s.toFetch().get(keyspaceName); // Check we get get RF new ranges in total - Set> ranges = new HashSet<>(); - for (Map.Entry>> e : toFetch) - ranges.addAll(e.getValue()); - - assertEquals(replicationFactor, ranges.size()); + long rangesCount = toFetch.stream() + .map(Multimap::values) + .flatMap(Collection::stream) + .map(f -> f.remote) + .map(Replica::range) + .count(); + assertEquals(replicationFactor, rangesCount); // there isn't any point in testing the size of these collections for any specific size. When a random partitioner // is used, they will vary. - assert toFetch.iterator().next().getValue().size() > 0; - assert !toFetch.iterator().next().getKey().equals(myEndpoint); + assert toFetch.stream().map(Multimap::values).flatMap(Collection::stream).count() > 0; + assert toFetch.stream().map(Multimap::keySet).map(Collection::stream).noneMatch(myEndpoint::equals); return s; } diff --git a/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java index 78e87c1b96..07d63772fc 100644 --- a/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java +++ b/test/unit/org/apache/cassandra/dht/RangeFetchMapCalculatorTest.java @@ -25,8 +25,9 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; -import com.google.common.collect.HashMultimap; +import com.google.common.base.Predicate; import com.google.common.collect.Multimap; +import org.apache.cassandra.locator.EndpointsByRange; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -34,6 +35,8 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.locator.AbstractNetworkTopologySnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaUtils; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -78,14 +81,14 @@ public class RangeFetchMapCalculatorTest @Test public void testWithSingleSource() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.4"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.5"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -95,14 +98,14 @@ public class RangeFetchMapCalculatorTest @Test public void testWithNonOverlappingSource() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.2"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.4"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.5", "127.0.0.6"); addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.7", "127.0.0.8"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.9", "127.0.0.10"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -112,12 +115,12 @@ public class RangeFetchMapCalculatorTest @Test public void testWithRFThreeReplacement() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.2"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2", "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3", "127.0.0.4"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -128,14 +131,14 @@ public class RangeFetchMapCalculatorTest @Test public void testForMultipleRoundsComputation() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.3"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.3", "127.0.0.2"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -150,14 +153,14 @@ public class RangeFetchMapCalculatorTest @Test public void testForMultipleRoundsComputationWithLocalHost() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1", "127.0.0.2"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -170,14 +173,14 @@ public class RangeFetchMapCalculatorTest @Test public void testForEmptyGraph() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 31, 40, "127.0.0.1"); addNonTrivialRangeAndSources(rangesWithSources, 41, 50, "127.0.0.1"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> map = calculator.getRangeFetchMap(); //All ranges map to local host so we will not stream anything. assertTrue(map.isEmpty()); @@ -186,31 +189,28 @@ public class RangeFetchMapCalculatorTest @Test public void testWithNoSourceWithLocal() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.5"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3"); //Return false for all except 127.0.0.5 - final RangeStreamer.ISourceFilter filter = new RangeStreamer.ISourceFilter() + final Predicate filter = replica -> { - public boolean shouldInclude(InetAddressAndPort endpoint) + try { - try - { - if (endpoint.equals(InetAddressAndPort.getByName("127.0.0.5"))) - return false; - else - return true; - } - catch (UnknownHostException e) - { + if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.5"))) + return false; + else return true; - } + } + catch (UnknownHostException e) + { + return true; } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, Arrays.asList(filter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Arrays.asList(filter), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); @@ -225,32 +225,26 @@ public class RangeFetchMapCalculatorTest @Test (expected = IllegalStateException.class) public void testWithNoLiveSource() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.5"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.2"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.3"); - final RangeStreamer.ISourceFilter allDeadFilter = new RangeStreamer.ISourceFilter() - { - public boolean shouldInclude(InetAddressAndPort endpoint) - { - return false; - } - }; + final Predicate allDeadFilter = replica -> false; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, Arrays.asList(allDeadFilter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Arrays.asList(allDeadFilter), "Test"); calculator.getRangeFetchMap(); } @Test public void testForLocalDC() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.1", "127.0.0.3", "127.0.0.53"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.1", "127.0.0.3", "127.0.0.57"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59", "127.0.0.61"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, new ArrayList<>(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), new ArrayList<>(), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); Assert.assertEquals(2, map.asMap().size()); @@ -263,31 +257,28 @@ public class RangeFetchMapCalculatorTest @Test public void testForRemoteDC() throws Exception { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59"); //Reject only 127.0.0.3 and accept everyone else - final RangeStreamer.ISourceFilter localHostFilter = new RangeStreamer.ISourceFilter() + final Predicate localHostFilter = replica -> { - public boolean shouldInclude(InetAddressAndPort endpoint) + try { - try - { - if (endpoint.equals(InetAddressAndPort.getByName("127.0.0.3"))) - return false; - else - return true; - } - catch (UnknownHostException e) - { + if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.3"))) + return false; + else return true; - } + } + catch (UnknownHostException e) + { + return true; } }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, Arrays.asList(localHostFilter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Arrays.asList(localHostFilter), "Test"); Multimap> map = calculator.getRangeFetchMap(); validateRange(rangesWithSources, map); Assert.assertEquals(3, map.asMap().size()); @@ -301,14 +292,14 @@ public class RangeFetchMapCalculatorTest @Test public void testTrivialRanges() throws UnknownHostException { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); // add non-trivial ranges addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55"); addNonTrivialRangeAndSources(rangesWithSources, 21, 30, "127.0.0.2", "127.0.0.59"); // and a trivial one: addTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51"); - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, Collections.emptyList(), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.emptyList(), "Test"); Multimap> optMap = calculator.getRangeFetchMapForNonTrivialRanges(); Multimap> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap); assertTrue(trivialMap.get(InetAddressAndPort.getByName("127.0.0.3")).contains(generateTrivialRange(1,10)) ^ @@ -319,7 +310,7 @@ public class RangeFetchMapCalculatorTest @Test(expected = IllegalStateException.class) public void testNotEnoughEndpointsForTrivialRange() throws UnknownHostException { - Multimap, InetAddressAndPort> rangesWithSources = HashMultimap.create(); + EndpointsByRange.Mutable rangesWithSources = new EndpointsByRange.Mutable(); // add non-trivial ranges addNonTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3", "127.0.0.51"); addNonTrivialRangeAndSources(rangesWithSources, 11, 20, "127.0.0.3", "127.0.0.55"); @@ -327,23 +318,20 @@ public class RangeFetchMapCalculatorTest // and a trivial one: addTrivialRangeAndSources(rangesWithSources, 1, 10, "127.0.0.3"); - RangeStreamer.ISourceFilter filter = new RangeStreamer.ISourceFilter() + Predicate filter = replica -> { - public boolean shouldInclude(InetAddressAndPort endpoint) + try { - try - { - if (endpoint.equals(InetAddressAndPort.getByName("127.0.0.3"))) - return false; - } - catch (UnknownHostException e) - { - throw new RuntimeException(e); - } - return true; + if (replica.endpoint().equals(InetAddressAndPort.getByName("127.0.0.3"))) + return false; } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + return true; }; - RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources, Collections.singleton(filter), "Test"); + RangeFetchMapCalculator calculator = new RangeFetchMapCalculator(rangesWithSources.asImmutableView(), Collections.singleton(filter), "Test"); Multimap> optMap = calculator.getRangeFetchMapForNonTrivialRanges(); Multimap> trivialMap = calculator.getRangeFetchMapForTrivialRanges(optMap); @@ -355,27 +343,29 @@ public class RangeFetchMapCalculatorTest assertTrue(result.containsAll(expected)); } - private void validateRange(Multimap, InetAddressAndPort> rangesWithSources, Multimap> result) + private void validateRange(EndpointsByRange.Mutable rangesWithSources, Multimap> result) { for (Map.Entry> entry : result.entries()) { - assertTrue(rangesWithSources.get(entry.getValue()).contains(entry.getKey())); + assertTrue(rangesWithSources.get(entry.getValue()).endpoints().contains(entry.getKey())); } } - private void addNonTrivialRangeAndSources(Multimap, InetAddressAndPort> rangesWithSources, int left, int right, String... hosts) throws UnknownHostException + private void addNonTrivialRangeAndSources(EndpointsByRange.Mutable rangesWithSources, int left, int right, String... hosts) throws UnknownHostException { for (InetAddressAndPort endpoint : makeAddrs(hosts)) { - rangesWithSources.put(generateNonTrivialRange(left, right), endpoint); + Range range = generateNonTrivialRange(left, right); + rangesWithSources.put(range, Replica.fullReplica(endpoint, range)); } } - private void addTrivialRangeAndSources(Multimap, InetAddressAndPort> rangesWithSources, int left, int right, String... hosts) throws UnknownHostException + private void addTrivialRangeAndSources(EndpointsByRange.Mutable rangesWithSources, int left, int right, String... hosts) throws UnknownHostException { for (InetAddressAndPort endpoint : makeAddrs(hosts)) { - rangesWithSources.put(generateTrivialRange(left, right), endpoint); + Range range = generateTrivialRange(left, right); + rangesWithSources.put(range, Replica.fullReplica(endpoint, range)); } } diff --git a/test/unit/org/apache/cassandra/dht/RangeTest.java b/test/unit/org/apache/cassandra/dht/RangeTest.java index 495979efb3..36a8da1161 100644 --- a/test/unit/org/apache/cassandra/dht/RangeTest.java +++ b/test/unit/org/apache/cassandra/dht/RangeTest.java @@ -642,7 +642,7 @@ public class RangeTest Range.OrderedRangeContainmentChecker checker = new Range.OrderedRangeContainmentChecker(ranges); for (Token t : tokensToTest) { - if (checker.contains(t) != Range.isInRanges(t, ranges)) // avoid running Joiner.on(..) every iteration + if (checker.test(t) != Range.isInRanges(t, ranges)) // avoid running Joiner.on(..) every iteration fail(String.format("This should never flap! If it does, it is a bug (ranges = %s, token = %s)", Joiner.on(",").join(ranges), t)); } } @@ -653,11 +653,11 @@ public class RangeTest { List> ranges = asList(r(Long.MIN_VALUE, Long.MIN_VALUE + 1), r(Long.MAX_VALUE - 1, Long.MAX_VALUE)); Range.OrderedRangeContainmentChecker checker = new Range.OrderedRangeContainmentChecker(ranges); - assertFalse(checker.contains(t(Long.MIN_VALUE))); - assertTrue(checker.contains(t(Long.MIN_VALUE + 1))); - assertFalse(checker.contains(t(0))); - assertFalse(checker.contains(t(Long.MAX_VALUE - 1))); - assertTrue(checker.contains(t(Long.MAX_VALUE))); + assertFalse(checker.test(t(Long.MIN_VALUE))); + assertTrue(checker.test(t(Long.MIN_VALUE + 1))); + assertFalse(checker.test(t(0))); + assertFalse(checker.test(t(Long.MAX_VALUE - 1))); + assertTrue(checker.test(t(Long.MAX_VALUE))); } private static Range r(long left, long right) diff --git a/test/unit/org/apache/cassandra/dht/SplitterTest.java b/test/unit/org/apache/cassandra/dht/SplitterTest.java index 322a57c386..c591499e26 100644 --- a/test/unit/org/apache/cassandra/dht/SplitterTest.java +++ b/test/unit/org/apache/cassandra/dht/SplitterTest.java @@ -62,13 +62,54 @@ public class SplitterTest randomSplitTestVNodes(new Murmur3Partitioner()); } + @Test + public void testWithWeight() + { + List ranges = new ArrayList<>(); + ranges.add(new Splitter.WeightedRange(1.0, t(0, 10))); + ranges.add(new Splitter.WeightedRange(1.0, t(20, 30))); + ranges.add(new Splitter.WeightedRange(0.5, t(40, 60))); + + List ranges2 = new ArrayList<>(); + ranges2.add(new Splitter.WeightedRange(1.0, t(0, 10))); + ranges2.add(new Splitter.WeightedRange(1.0, t(20, 30))); + ranges2.add(new Splitter.WeightedRange(1.0, t(40, 50))); + IPartitioner partitioner = Murmur3Partitioner.instance; + Splitter splitter = partitioner.splitter().get(); + + assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false)); + } + + @Test + public void testWithWeight2() + { + List ranges = new ArrayList<>(); + ranges.add(new Splitter.WeightedRange(0.2, t(0, 10))); + ranges.add(new Splitter.WeightedRange(1.0, t(20, 30))); + ranges.add(new Splitter.WeightedRange(1.0, t(40, 50))); + + List ranges2 = new ArrayList<>(); + ranges2.add(new Splitter.WeightedRange(1.0, t(0, 2))); + ranges2.add(new Splitter.WeightedRange(1.0, t(20, 30))); + ranges2.add(new Splitter.WeightedRange(1.0, t(40, 50))); + IPartitioner partitioner = Murmur3Partitioner.instance; + Splitter splitter = partitioner.splitter().get(); + + assertEquals(splitter.splitOwnedRanges(2, ranges, false), splitter.splitOwnedRanges(2, ranges2, false)); + } + + private Range t(long left, long right) + { + return new Range<>(new Murmur3Partitioner.LongToken(left), new Murmur3Partitioner.LongToken(right)); + } + private static void randomSplitTestNoVNodes(IPartitioner partitioner) { Splitter splitter = getSplitter(partitioner); Random r = new Random(); for (int i = 0; i < 10000; i++) { - List> localRanges = generateLocalRanges(1, r.nextInt(4) + 1, splitter, r, partitioner instanceof RandomPartitioner); + List localRanges = generateLocalRanges(1, r.nextInt(4) + 1, splitter, r, partitioner instanceof RandomPartitioner); List boundaries = splitter.splitOwnedRanges(r.nextInt(9) + 1, localRanges, false); assertTrue("boundaries = " + boundaries + " ranges = " + localRanges, assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, true)); } @@ -84,14 +125,14 @@ public class SplitterTest int numTokens = 172 + r.nextInt(128); int rf = r.nextInt(4) + 2; int parts = r.nextInt(5) + 1; - List> localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner); + List localRanges = generateLocalRanges(numTokens, rf, splitter, r, partitioner instanceof RandomPartitioner); List boundaries = splitter.splitOwnedRanges(parts, localRanges, true); if (!assertRangeSizeEqual(localRanges, boundaries, partitioner, splitter, false)) fail(String.format("Could not split %d tokens with rf=%d into %d parts (localRanges=%s, boundaries=%s)", numTokens, rf, parts, localRanges, boundaries)); } } - private static boolean assertRangeSizeEqual(List> localRanges, List tokens, IPartitioner partitioner, Splitter splitter, boolean splitIndividualRanges) + private static boolean assertRangeSizeEqual(List localRanges, List tokens, IPartitioner partitioner, Splitter splitter, boolean splitIndividualRanges) { Token start = partitioner.getMinimumToken(); List splits = new ArrayList<>(); @@ -119,27 +160,27 @@ public class SplitterTest return allBalanced; } - private static BigInteger sumOwnedBetween(List> localRanges, Token start, Token end, Splitter splitter, boolean splitIndividualRanges) + private static BigInteger sumOwnedBetween(List localRanges, Token start, Token end, Splitter splitter, boolean splitIndividualRanges) { BigInteger sum = BigInteger.ZERO; - for (Range range : localRanges) + for (Splitter.WeightedRange range : localRanges) { if (splitIndividualRanges) { - Set> intersections = new Range<>(start, end).intersectionWith(range); + Set> intersections = new Range<>(start, end).intersectionWith(range.range()); for (Range intersection : intersections) sum = sum.add(splitter.valueForToken(intersection.right).subtract(splitter.valueForToken(intersection.left))); } else { - if (new Range<>(start, end).contains(range.left)) - sum = sum.add(splitter.valueForToken(range.right).subtract(splitter.valueForToken(range.left))); + if (new Range<>(start, end).contains(range.left())) + sum = sum.add(splitter.valueForToken(range.right()).subtract(splitter.valueForToken(range.left()))); } } return sum; } - private static List> generateLocalRanges(int numTokens, int rf, Splitter splitter, Random r, boolean randomPartitioner) + private static List generateLocalRanges(int numTokens, int rf, Splitter splitter, Random r, boolean randomPartitioner) { int localTokens = numTokens * rf; List randomTokens = new ArrayList<>(); @@ -152,11 +193,11 @@ public class SplitterTest Collections.sort(randomTokens); - List> localRanges = new ArrayList<>(localTokens); + List localRanges = new ArrayList<>(localTokens); for (int i = 0; i < randomTokens.size() - 1; i++) { assert randomTokens.get(i).compareTo(randomTokens.get(i + 1)) < 0; - localRanges.add(new Range<>(randomTokens.get(i), randomTokens.get(i + 1))); + localRanges.add(new Splitter.WeightedRange(1.0, new Range<>(randomTokens.get(i), randomTokens.get(i + 1)))); i++; } return localRanges; diff --git a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java index bf71c09ca5..34096a70eb 100644 --- a/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java +++ b/test/unit/org/apache/cassandra/dht/StreamStateStoreTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.dht; import java.util.Collections; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.BeforeClass; import org.junit.Test; @@ -53,7 +54,7 @@ public class StreamStateStoreTest InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, local, new DefaultConnectionFactory(), 0, null, PreviewKind.NONE); - session.addStreamRequest("keyspace1", Collections.singleton(range), Collections.singleton("cf")); + session.addStreamRequest("keyspace1", RangesAtEndpoint.toDummyList(Collections.singleton(range)), RangesAtEndpoint.toDummyList(Collections.emptyList()), Collections.singleton("cf")); StreamStateStore store = new StreamStateStore(); // session complete event that is not completed makes data not available for keyspace/ranges @@ -74,7 +75,7 @@ public class StreamStateStoreTest // add different range within the same keyspace Range range2 = new Range<>(factory.fromString("100"), factory.fromString("200")); session = new StreamSession(StreamOperation.BOOTSTRAP, local, new DefaultConnectionFactory(), 0, null, PreviewKind.NONE); - session.addStreamRequest("keyspace1", Collections.singleton(range2), Collections.singleton("cf")); + session.addStreamRequest("keyspace1", RangesAtEndpoint.toDummyList(Collections.singleton(range2)), RangesAtEndpoint.toDummyList(Collections.emptyList()), Collections.singleton("cf")); session.state(StreamSession.State.COMPLETE); store.handleStreamEvent(new StreamEvent.SessionCompleteEvent(session)); diff --git a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java index 833ee8bcaf..0710945cd8 100644 --- a/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java +++ b/test/unit/org/apache/cassandra/gms/PendingRangeCalculatorServiceTest.java @@ -62,7 +62,7 @@ public class PendingRangeCalculatorServiceTest @BMRule(name = "Block pending range calculation", targetClass = "TokenMetadata", targetMethod = "calculatePendingRanges", - targetLocation = "AT INVOKE org.apache.cassandra.locator.AbstractReplicationStrategy.getAddressRanges", + targetLocation = "AT INVOKE org.apache.cassandra.locator.AbstractReplicationStrategy.getAddressReplicas", action = "org.apache.cassandra.gms.PendingRangeCalculatorServiceTest.calculationLock.lock()") public void testDelayedResponse() throws UnknownHostException, InterruptedException { diff --git a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java index fccb34439e..9e3594bc0f 100644 --- a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java @@ -69,7 +69,7 @@ public class BigTableWriterTest extends AbstractTransactionalTest private TestableBTW(Descriptor desc) { - this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, + this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))); diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java index dbb929dcde..faf46bcb64 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java @@ -649,7 +649,7 @@ public class CQLSSTableWriterTest public void init(String keyspace) { this.keyspace = keyspace; - for (Range range : StorageService.instance.getLocalRanges(ks)) + for (Range range : StorageService.instance.getLocalReplicas(ks).ranges()) addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort()); } diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 044cd9f02d..c4ccf4822d 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -176,21 +176,26 @@ public class LegacySSTableTest { for (SSTableReader sstable : cfs.getLiveSSTables()) { - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, 1234, NO_PENDING_REPAIR); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1234, NO_PENDING_REPAIR, false); sstable.reloadSSTableMetadata(); assertEquals(1234, sstable.getRepairedAt()); if (sstable.descriptor.version.hasPendingRepair()) assertEquals(NO_PENDING_REPAIR, sstable.getPendingRepair()); } + boolean isTransient = false; for (SSTableReader sstable : cfs.getLiveSSTables()) { UUID random = UUID.randomUUID(); - sstable.descriptor.getMetadataSerializer().mutateRepaired(sstable.descriptor, UNREPAIRED_SSTABLE, random); + sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, UNREPAIRED_SSTABLE, random, isTransient); sstable.reloadSSTableMetadata(); assertEquals(UNREPAIRED_SSTABLE, sstable.getRepairedAt()); if (sstable.descriptor.version.hasPendingRepair()) assertEquals(random, sstable.getPendingRepair()); + if (sstable.descriptor.version.hasIsTransient()) + assertEquals(isTransient, sstable.isTransient()); + + isTransient = !isTransient; } } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java index 85091150ee..5d40f8cc62 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java @@ -31,14 +31,13 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.*; import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.KeyspaceParams; @@ -109,8 +108,8 @@ public class SSTableLoaderTest public void init(String keyspace) { this.keyspace = keyspace; - for (Range range : StorageService.instance.getLocalRanges(KEYSPACE1)) - addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort()); + for (Replica replica : StorageService.instance.getLocalReplicas(KEYSPACE1)) + addRangeForEndpoint(replica.range(), FBUtilities.getBroadcastAddressAndPort()); } public TableMetadataRef getTableMetadata(String tableName) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 6412ef49be..7c47c8b9be 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -877,7 +877,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount; for ( ; i < end ; i++) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index 441a3b9df6..731cee2e53 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -219,7 +219,7 @@ public class SSTableUtils TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname); ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); SerializationHeader header = appender.header(); - SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, 0, header); + SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFilename(datafile.getAbsolutePath()), expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header); while (appender.append(writer)) { /* pass */ } Collection readers = writer.finish(true); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java index 5d62cdb323..31d0b8986d 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.io.File; import java.nio.ByteBuffer; +import java.util.UUID; import org.junit.Test; @@ -32,9 +33,12 @@ import org.apache.cassandra.db.rows.*; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.utils.FBUtilities; import static junit.framework.Assert.fail; +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -245,4 +249,60 @@ public class SSTableWriterTest extends SSTableWriterTestBase } } + private static void assertValidRepairMetadata(long repairedAt, UUID pendingRepair, boolean isTransient) + { + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE); + File dir = cfs.getDirectories().getDirectoryForNewSSTables(); + LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM); + + try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient)) + { + // expected + } + catch (IllegalArgumentException e) + { + throw new AssertionError("Unexpected IllegalArgumentException", e); + } + + txn.abort(); + LifecycleTransaction.waitForDeletions(); + } + + private static void assertInvalidRepairMetadata(long repairedAt, UUID pendingRepair, boolean isTransient) + { + Keyspace keyspace = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_SMALL_MAX_VALUE); + File dir = cfs.getDirectories().getDirectoryForNewSSTables(); + LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM); + + try (SSTableWriter writer = getWriter(cfs, dir, txn, repairedAt, pendingRepair, isTransient)) + { + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + // expected + } + + txn.abort(); + LifecycleTransaction.waitForDeletions(); + } + + /** + * It should only be possible to create sstables marked transient that also have a pending repair + */ + @Test + public void testRepairMetadataValidation() + { + assertValidRepairMetadata(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false); + assertValidRepairMetadata(1, NO_PENDING_REPAIR, false); + assertValidRepairMetadata(UNREPAIRED_SSTABLE, UUID.randomUUID(), false); + assertValidRepairMetadata(UNREPAIRED_SSTABLE, UUID.randomUUID(), true); + + assertInvalidRepairMetadata(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, true); + assertInvalidRepairMetadata(1, UUID.randomUUID(), false); + assertInvalidRepairMetadata(1, NO_PENDING_REPAIR, true); + + } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java index d42c49b7b2..962e1a15a8 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java @@ -22,6 +22,7 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -48,6 +49,7 @@ import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SSTableWriterTestBase extends SchemaLoader { @@ -161,10 +163,15 @@ public class SSTableWriterTestBase extends SchemaLoader assertFalse(CompactionManager.instance.submitMaximal(cfs, cfs.gcBefore((int) (System.currentTimeMillis() / 1000)), false).isEmpty()); } - public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn) + public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn, long repairedAt, UUID pendingRepair, boolean isTransient) { Descriptor desc = cfs.newSSTableDescriptor(directory); - return SSTableWriter.create(desc, 0, 0, null, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn); + return SSTableWriter.create(desc, 0, repairedAt, pendingRepair, isTransient, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn); + } + + public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn) + { + return getWriter(cfs, directory, txn, 0, null, false); } public static ByteBuffer random(int i, int size) diff --git a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java index 9aa4e28cdd..aea3b4a7e2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java @@ -96,7 +96,7 @@ public class SSTableFlushObserverTest KS_NAME, CF_NAME, 0, sstableFormat), - 10L, 0L, null, TableMetadataRef.forOfflineTools(cfm), + 10L, 0L, null, false, TableMetadataRef.forOfflineTools(cfm), new MetadataCollector(cfm.comparator).sstableLevel(0), new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS), Collections.singletonList(observer), diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 8ab1511c37..f109d8f7b6 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -99,7 +99,7 @@ public class MetadataSerializerTest String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, SerializationHeader.make(cfm, Collections.emptyList())); + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList())); } @Test diff --git a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java index 202d7f1ea2..bf1d9402dd 100644 --- a/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java +++ b/test/unit/org/apache/cassandra/locator/DynamicEndpointSnitchTest.java @@ -50,6 +50,16 @@ public class DynamicEndpointSnitchTest Thread.sleep(150); } + private static EndpointsForRange full(InetAddressAndPort... endpoints) + { + EndpointsForRange.Builder rlist = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE, endpoints.length); + for (InetAddressAndPort endpoint: endpoints) + { + rlist.add(ReplicaUtils.full(endpoint)); + } + return rlist.build(); + } + @Test public void testSnitch() throws InterruptedException, IOException, ConfigurationException { @@ -66,41 +76,41 @@ public class DynamicEndpointSnitchTest // first, make all hosts equal setScores(dsnitch, 1, hosts, 10, 10, 10); - List order = Arrays.asList(host1, host2, host3); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + EndpointsForRange order = full(host1, host2, host3); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); // make host1 a little worse setScores(dsnitch, 1, hosts, 20, 10, 10); - order = Arrays.asList(host2, host3, host1); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + order = full(host2, host3, host1); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); // make host2 as bad as host1 setScores(dsnitch, 2, hosts, 15, 20, 10); - order = Arrays.asList(host3, host1, host2); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + order = full(host3, host1, host2); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); // make host3 the worst setScores(dsnitch, 3, hosts, 10, 10, 30); - order = Arrays.asList(host1, host2, host3); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + order = full(host1, host2, host3); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); // make host3 equal to the others setScores(dsnitch, 5, hosts, 10, 10, 10); - order = Arrays.asList(host1, host2, host3); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + order = full(host1, host2, host3); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); /// Tests CASSANDRA-6683 improvements // make the scores differ enough from the ideal order that we sort by score; under the old // dynamic snitch behavior (where we only compared neighbors), these wouldn't get sorted setScores(dsnitch, 20, hosts, 10, 70, 20); - order = Arrays.asList(host1, host3, host2); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3))); + order = full(host1, host3, host2); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3))); - order = Arrays.asList(host4, host1, host3, host2); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3, host4))); + order = full(host4, host1, host3, host2); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3, host4))); setScores(dsnitch, 20, hosts, 10, 10, 10); - order = Arrays.asList(host4, host1, host2, host3); - assertEquals(order, dsnitch.getSortedListByProximity(self, Arrays.asList(host1, host2, host3, host4))); + order = full(host4, host1, host2, host3); + assertEquals(order, dsnitch.sortedByProximity(self, full(host1, host2, host3, host4))); } } diff --git a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java index ab6c6cdb70..5f6e26ffa0 100644 --- a/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/NetworkTopologyStrategyTest.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.junit.Assert; @@ -36,12 +37,17 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.TokenMetadata.Topology; import org.apache.cassandra.service.StorageService; +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; + public class NetworkTopologyStrategyTest { private String keyspaceName = "Keyspace1"; @@ -51,6 +57,7 @@ public class NetworkTopologyStrategyTest public static void setupDD() { DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); } @Test @@ -68,13 +75,14 @@ public class NetworkTopologyStrategyTest // Set the localhost to the tokenmetadata. Embedded cassandra way? NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions); - assert strategy.getReplicationFactor("DC1") == 3; - assert strategy.getReplicationFactor("DC2") == 2; - assert strategy.getReplicationFactor("DC3") == 1; + assert strategy.getReplicationFactor("DC1").allReplicas == 3; + assert strategy.getReplicationFactor("DC2").allReplicas == 2; + assert strategy.getReplicationFactor("DC3").allReplicas == 1; // Query for the natural hosts - ArrayList endpoints = strategy.getNaturalEndpoints(new StringToken("123")); - assert 6 == endpoints.size(); - assert 6 == new HashSet<>(endpoints).size(); // ensure uniqueness + EndpointsForToken replicas = strategy.getNaturalReplicasForToken(new StringToken("123")); + assert 6 == replicas.size(); + assert 6 == replicas.endpoints().size(); // ensure uniqueness + assert 6 == new HashSet<>(replicas.byEndpoint().values()).size(); // ensure uniqueness } @Test @@ -92,13 +100,14 @@ public class NetworkTopologyStrategyTest // Set the localhost to the tokenmetadata. Embedded cassandra way? NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions); - assert strategy.getReplicationFactor("DC1") == 3; - assert strategy.getReplicationFactor("DC2") == 3; - assert strategy.getReplicationFactor("DC3") == 0; + assert strategy.getReplicationFactor("DC1").allReplicas == 3; + assert strategy.getReplicationFactor("DC2").allReplicas == 3; + assert strategy.getReplicationFactor("DC3").allReplicas == 0; // Query for the natural hosts - ArrayList endpoints = strategy.getNaturalEndpoints(new StringToken("123")); - assert 6 == endpoints.size(); - assert 6 == new HashSet<>(endpoints).size(); // ensure uniqueness + EndpointsForToken replicas = strategy.getNaturalReplicasForToken(new StringToken("123")); + assert 6 == replicas.size(); + assert 6 == replicas.endpoints().size(); // ensure uniqueness + assert 6 == new HashSet<>(replicas.byEndpoint().values()).size(); // ensure uniqueness } @Test @@ -137,12 +146,13 @@ public class NetworkTopologyStrategyTest for (String testToken : new String[]{"123456", "200000", "000402", "ffffff", "400200"}) { - List endpoints = strategy.calculateNaturalEndpoints(new StringToken(testToken), metadata); - Set epSet = new HashSet<>(endpoints); + EndpointsForRange replicas = strategy.calculateNaturalReplicas(new StringToken(testToken), metadata); + Set endpointSet = replicas.endpoints(); - Assert.assertEquals(totalRF, endpoints.size()); - Assert.assertEquals(totalRF, epSet.size()); - logger.debug("{}: {}", testToken, endpoints); + Assert.assertEquals(totalRF, replicas.size()); + Assert.assertEquals(totalRF, new HashSet<>(replicas.byEndpoint().values()).size()); + Assert.assertEquals(totalRF, endpointSet.size()); + logger.debug("{}: {}", testToken, replicas); } } @@ -209,7 +219,7 @@ public class NetworkTopologyStrategyTest { Token token = Murmur3Partitioner.instance.getRandomToken(rand); List expected = calculateNaturalEndpoints(token, tokenMetadata, datacenters, snitch); - List actual = nts.calculateNaturalEndpoints(token, tokenMetadata); + List actual = new ArrayList<>(nts.calculateNaturalReplicas(token, tokenMetadata).endpoints()); if (endpointsDiffer(expected, actual)) { System.err.println("Endpoints mismatch for token " + token); @@ -373,4 +383,50 @@ public class NetworkTopologyStrategyTest Integer replicas = datacenters.get(dc); return replicas == null ? 0 : replicas; } + + private static Token tk(long t) + { + return new LongToken(t); + } + + private static Range range(long l, long r) + { + return new Range<>(tk(l), tk(r)); + } + + @Test + public void testTransientReplica() throws Exception + { + IEndpointSnitch snitch = new SimpleSnitch(); + DatabaseDescriptor.setEndpointSnitch(snitch); + + List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), + InetAddressAndPort.getByName("127.0.0.2"), + InetAddressAndPort.getByName("127.0.0.3"), + InetAddressAndPort.getByName("127.0.0.4")); + + Multimap tokens = HashMultimap.create(); + tokens.put(endpoints.get(0), tk(100)); + tokens.put(endpoints.get(1), tk(200)); + tokens.put(endpoints.get(2), tk(300)); + tokens.put(endpoints.get(3), tk(400)); + TokenMetadata metadata = new TokenMetadata(); + metadata.updateNormalTokens(tokens); + + Map configOptions = new HashMap(); + configOptions.put(snitch.getDatacenter((InetAddressAndPort) null), "3/1"); + + NetworkTopologyStrategy strategy = new NetworkTopologyStrategy(keyspaceName, metadata, snitch, configOptions); + + Assert.assertEquals(EndpointsForRange.of(fullReplica(endpoints.get(0), range(400, 100)), + fullReplica(endpoints.get(1), range(400, 100)), + transientReplica(endpoints.get(2), range(400, 100))), + strategy.getNaturalReplicasForToken(tk(99))); + + + Assert.assertEquals(EndpointsForRange.of(fullReplica(endpoints.get(1), range(100, 200)), + fullReplica(endpoints.get(2), range(100, 200)), + transientReplica(endpoints.get(3), range(100, 200))), + strategy.getNaturalReplicasForToken(tk(101))); + } } diff --git a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java b/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java index 9c90d571f9..4afeb5aaba 100644 --- a/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/OldNetworkTopologyStrategyTest.java @@ -20,7 +20,6 @@ package org.apache.cassandra.locator; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -39,9 +38,11 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OldNetworkTopologyStrategyTest { + private List keyTokens; private TokenMetadata tmd; private Map> expectedResults; @@ -53,7 +54,7 @@ public class OldNetworkTopologyStrategyTest } @Before - public void init() + public void init() throws Exception { keyTokens = new ArrayList(); tmd = new TokenMetadata(); @@ -160,11 +161,11 @@ public class OldNetworkTopologyStrategyTest { for (Token keyToken : keyTokens) { - List endpoints = strategy.getNaturalEndpoints(keyToken); - for (int j = 0; j < endpoints.size(); j++) + int j = 0; + for (InetAddressAndPort endpoint : strategy.getNaturalReplicasForToken(keyToken).endpoints()) { ArrayList hostsExpected = expectedResults.get(keyToken.toString()); - assertEquals(endpoints.get(j), hostsExpected.get(j)); + assertEquals(endpoint, hostsExpected.get(j++)); } } } @@ -188,7 +189,6 @@ public class OldNetworkTopologyStrategyTest assertEquals(ranges.left.iterator().next().left, tokensAfterMove[movingNodeIdx]); assertEquals(ranges.left.iterator().next().right, tokens[movingNodeIdx]); assertEquals("No data should be fetched", ranges.right.size(), 0); - } @Test @@ -205,7 +205,6 @@ public class OldNetworkTopologyStrategyTest assertEquals("No data should be streamed", ranges.left.size(), 0); assertEquals(ranges.right.iterator().next().left, tokens[movingNodeIdx]); assertEquals(ranges.right.iterator().next().right, tokensAfterMove[movingNodeIdx]); - } @SuppressWarnings("unchecked") @@ -366,16 +365,21 @@ public class OldNetworkTopologyStrategyTest TokenMetadata tokenMetadataAfterMove = initTokenMetadata(tokensAfterMove); AbstractReplicationStrategy strategy = new OldNetworkTopologyStrategy("Keyspace1", tokenMetadataCurrent, endpointSnitch, optsWithRF(2)); - Collection> currentRanges = strategy.getAddressRanges().get(movingNode); - Collection> updatedRanges = strategy.getPendingAddressRanges(tokenMetadataAfterMove, tokensAfterMove[movingNodeIdx], movingNode); + RangesAtEndpoint currentRanges = strategy.getAddressReplicas().get(movingNode); + RangesAtEndpoint updatedRanges = strategy.getPendingAddressRanges(tokenMetadataAfterMove, tokensAfterMove[movingNodeIdx], movingNode); - Pair>, Set>> ranges = StorageService.instance.calculateStreamAndFetchRanges(currentRanges, updatedRanges); - - return ranges; + return asRanges(StorageService.calculateStreamAndFetchRanges(currentRanges, updatedRanges)); } private static Map optsWithRF(int rf) { return Collections.singletonMap("replication_factor", Integer.toString(rf)); } + + public static Pair>, Set>> asRanges(Pair replicas) + { + Set> leftRanges = replicas.left.ranges(); + Set> rightRanges = replicas.right.ranges(); + return Pair.create(leftRanges, rightRanges); + } } diff --git a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java index 56fd181fb2..8e0bc00833 100644 --- a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java +++ b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java @@ -26,7 +26,6 @@ import org.apache.cassandra.dht.Token; import org.junit.Test; import java.net.UnknownHostException; -import java.util.Collection; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -38,17 +37,29 @@ public class PendingRangeMapsTest { return new Range(new BigIntegerToken(left), new BigIntegerToken(right)); } + private static void addPendingRange(PendingRangeMaps pendingRangeMaps, Range range, String endpoint) + { + try + { + pendingRangeMaps.addPendingRange(range, Replica.fullReplica(InetAddressAndPort.getByName(endpoint), range)); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } + @Test public void testPendingEndpoints() throws UnknownHostException { PendingRangeMaps pendingRangeMaps = new PendingRangeMaps(); - pendingRangeMaps.addPendingRange(genRange("5", "15"), InetAddressAndPort.getByName("127.0.0.1")); - pendingRangeMaps.addPendingRange(genRange("15", "25"), InetAddressAndPort.getByName("127.0.0.2")); - pendingRangeMaps.addPendingRange(genRange("25", "35"), InetAddressAndPort.getByName("127.0.0.3")); - pendingRangeMaps.addPendingRange(genRange("35", "45"), InetAddressAndPort.getByName("127.0.0.4")); - pendingRangeMaps.addPendingRange(genRange("45", "55"), InetAddressAndPort.getByName("127.0.0.5")); - pendingRangeMaps.addPendingRange(genRange("45", "65"), InetAddressAndPort.getByName("127.0.0.6")); + addPendingRange(pendingRangeMaps, genRange("5", "15"), "127.0.0.1"); + addPendingRange(pendingRangeMaps, genRange("15", "25"), "127.0.0.2"); + addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.3"); + addPendingRange(pendingRangeMaps, genRange("35", "45"), "127.0.0.4"); + addPendingRange(pendingRangeMaps, genRange("45", "55"), "127.0.0.5"); + addPendingRange(pendingRangeMaps, genRange("45", "65"), "127.0.0.6"); assertEquals(0, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("0")).size()); assertEquals(0, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("5")).size()); @@ -61,8 +72,8 @@ public class PendingRangeMapsTest { assertEquals(2, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("55")).size()); assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("65")).size()); - Collection endpoints = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("15")); - assertTrue(endpoints.contains(InetAddressAndPort.getByName("127.0.0.1"))); + EndpointsForToken replicas = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("15")); + assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1"))); } @Test @@ -70,13 +81,13 @@ public class PendingRangeMapsTest { { PendingRangeMaps pendingRangeMaps = new PendingRangeMaps(); - pendingRangeMaps.addPendingRange(genRange("5", "15"), InetAddressAndPort.getByName("127.0.0.1")); - pendingRangeMaps.addPendingRange(genRange("15", "25"), InetAddressAndPort.getByName("127.0.0.2")); - pendingRangeMaps.addPendingRange(genRange("25", "35"), InetAddressAndPort.getByName("127.0.0.3")); - pendingRangeMaps.addPendingRange(genRange("35", "45"), InetAddressAndPort.getByName("127.0.0.4")); - pendingRangeMaps.addPendingRange(genRange("45", "55"), InetAddressAndPort.getByName("127.0.0.5")); - pendingRangeMaps.addPendingRange(genRange("45", "65"), InetAddressAndPort.getByName("127.0.0.6")); - pendingRangeMaps.addPendingRange(genRange("65", "7"), InetAddressAndPort.getByName("127.0.0.7")); + addPendingRange(pendingRangeMaps, genRange("5", "15"), "127.0.0.1"); + addPendingRange(pendingRangeMaps, genRange("15", "25"), "127.0.0.2"); + addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.3"); + addPendingRange(pendingRangeMaps, genRange("35", "45"), "127.0.0.4"); + addPendingRange(pendingRangeMaps, genRange("45", "55"), "127.0.0.5"); + addPendingRange(pendingRangeMaps, genRange("45", "65"), "127.0.0.6"); + addPendingRange(pendingRangeMaps, genRange("65", "7"), "127.0.0.7"); assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("0")).size()); assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("5")).size()); @@ -90,8 +101,8 @@ public class PendingRangeMapsTest { assertEquals(2, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("55")).size()); assertEquals(1, pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("65")).size()); - Collection endpoints = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("6")); - assertTrue(endpoints.contains(InetAddressAndPort.getByName("127.0.0.1"))); - assertTrue(endpoints.contains(InetAddressAndPort.getByName("127.0.0.7"))); + EndpointsForToken replicas = pendingRangeMaps.pendingEndpointsFor(new BigIntegerToken("6")); + assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1"))); + assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.7"))); } } diff --git a/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java new file mode 100644 index 0000000000..66eff237fc --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/ReplicaCollectionTest.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.base.Predicates; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.ReplicaCollection.Mutable.Conflict; +import org.apache.cassandra.utils.FBUtilities; +import org.junit.Assert; +import org.junit.Test; + +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; + +public class ReplicaCollectionTest +{ + + static final InetAddressAndPort EP1, EP2, EP3, EP4, EP5, BROADCAST_EP, NULL_EP; + static final Range R1, R2, R3, R4, R5, BROADCAST_RANGE, NULL_RANGE; + + static + { + try + { + EP1 = InetAddressAndPort.getByName("127.0.0.1"); + EP2 = InetAddressAndPort.getByName("127.0.0.2"); + EP3 = InetAddressAndPort.getByName("127.0.0.3"); + EP4 = InetAddressAndPort.getByName("127.0.0.4"); + EP5 = InetAddressAndPort.getByName("127.0.0.5"); + BROADCAST_EP = FBUtilities.getBroadcastAddressAndPort(); + NULL_EP = InetAddressAndPort.getByName("127.255.255.255"); + R1 = range(0, 1); + R2 = range(1, 2); + R3 = range(2, 3); + R4 = range(3, 4); + R5 = range(4, 5); + BROADCAST_RANGE = range(10, 11); + NULL_RANGE = range(10000, 10001); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + static Token tk(long t) + { + return new Murmur3Partitioner.LongToken(t); + } + + static Range range(long left, long right) + { + return new Range<>(tk(left), tk(right)); + } + + static class TestCase> + { + final C test; + final List canonicalList; + final Multimap canonicalByEndpoint; + final Multimap, Replica> canonicalByRange; + + TestCase(C test, List canonicalList) + { + this.test = test; + this.canonicalList = canonicalList; + this.canonicalByEndpoint = HashMultimap.create(); + this.canonicalByRange = HashMultimap.create(); + for (Replica replica : canonicalList) + canonicalByEndpoint.put(replica.endpoint(), replica); + for (Replica replica : canonicalList) + canonicalByRange.put(replica.range(), replica); + } + + void testSize() + { + Assert.assertEquals(canonicalList.size(), test.size()); + } + + void testEquals() + { + Assert.assertTrue(Iterables.elementsEqual(canonicalList, test)); + } + + void testEndpoints() + { + // TODO: we should do more exhaustive tests of the collection + Assert.assertEquals(ImmutableSet.copyOf(canonicalByEndpoint.keySet()), ImmutableSet.copyOf(test.endpoints())); + try + { + test.endpoints().add(EP5); + Assert.fail(); + } catch (UnsupportedOperationException e) {} + try + { + test.endpoints().remove(EP5); + Assert.fail(); + } catch (UnsupportedOperationException e) {} + + Assert.assertTrue(test.endpoints().containsAll(canonicalByEndpoint.keySet())); + for (InetAddressAndPort ep : canonicalByEndpoint.keySet()) + Assert.assertTrue(test.endpoints().contains(ep)); + for (InetAddressAndPort ep : ImmutableList.of(EP1, EP2, EP3, EP4, EP5, BROADCAST_EP)) + if (!canonicalByEndpoint.containsKey(ep)) + Assert.assertFalse(test.endpoints().contains(ep)); + } + + public void testOrderOfIteration() + { + Assert.assertEquals(canonicalList, ImmutableList.copyOf(test)); + Assert.assertEquals(canonicalList, test.stream().collect(Collectors.toList())); + Assert.assertEquals(new LinkedHashSet<>(Lists.transform(canonicalList, Replica::endpoint)), test.endpoints()); + } + + void testSelect(int subListDepth, int filterDepth, int sortDepth, int selectDepth) + { + TestCase allMatchZeroCapacity = new TestCase<>(test.select().add(Predicates.alwaysTrue(), 0).get(), Collections.emptyList()); + allMatchZeroCapacity.testAll(subListDepth, filterDepth, sortDepth, selectDepth - 1); + + TestCase noMatchFullCapacity = new TestCase<>(test.select().add(Predicates.alwaysFalse(), canonicalList.size()).get(), Collections.emptyList()); + noMatchFullCapacity.testAll(subListDepth, filterDepth, sortDepth,selectDepth - 1); + + if (canonicalList.size() <= 2) + return; + + List newOrderList = ImmutableList.of(canonicalList.get(2), canonicalList.get(1), canonicalList.get(0)); + TestCase newOrder = new TestCase<>( + test.select() + .add(r -> r == newOrderList.get(0), 3) + .add(r -> r == newOrderList.get(1), 3) + .add(r -> r == newOrderList.get(2), 3) + .get(), newOrderList + ); + newOrder.testAll(subListDepth, filterDepth, sortDepth,selectDepth - 1); + } + + private void assertSubList(C subCollection, int from, int to) + { + Assert.assertTrue(subCollection.isSnapshot); + if (from == to) + { + Assert.assertTrue(subCollection.isEmpty()); + } + else + { + List subList = this.test.list.subList(from, to); + if (test.isSnapshot) + Assert.assertSame(subList.getClass(), subCollection.list.getClass()); + Assert.assertEquals(subList, subCollection.list); + } + } + + void testSubList(int subListDepth, int filterDepth, int sortDepth, int selectDepth) + { + if (test.isSnapshot) + Assert.assertSame(test, test.subList(0, test.size())); + + if (test.isEmpty()) + return; + + TestCase skipFront = new TestCase<>(test.subList(1, test.size()), canonicalList.subList(1, canonicalList.size())); + assertSubList(skipFront.test, 1, canonicalList.size()); + skipFront.testAll(subListDepth - 1, filterDepth, sortDepth, selectDepth); + TestCase skipBack = new TestCase<>(test.subList(0, test.size() - 1), canonicalList.subList(0, canonicalList.size() - 1)); + assertSubList(skipBack.test, 0, canonicalList.size() - 1); + skipBack.testAll(subListDepth - 1, filterDepth, sortDepth, selectDepth); + } + + void testFilter(int subListDepth, int filterDepth, int sortDepth, int selectDepth) + { + if (test.isSnapshot) + Assert.assertSame(test, test.filter(Predicates.alwaysTrue())); + + if (test.isEmpty()) + return; + // remove start + // we recurse on the same subset in testSubList, so just corroborate we have the correct list here + assertSubList(test.filter(r -> r != canonicalList.get(0)), 1, canonicalList.size()); + + if (test.size() <= 1) + return; + // remove end + // we recurse on the same subset in testSubList, so just corroborate we have the correct list here + assertSubList(test.filter(r -> r != canonicalList.get(canonicalList.size() - 1)), 0, canonicalList.size() - 1); + + if (test.size() <= 2) + return; + Predicate removeMiddle = r -> r != canonicalList.get(canonicalList.size() / 2); + TestCase filtered = new TestCase<>(test.filter(removeMiddle), ImmutableList.copyOf(Iterables.filter(canonicalList, removeMiddle::test))); + filtered.testAll(subListDepth, filterDepth - 1, sortDepth, selectDepth); + } + + void testContains() + { + for (Replica replica : canonicalList) + Assert.assertTrue(test.contains(replica)); + Assert.assertFalse(test.contains(fullReplica(NULL_EP, NULL_RANGE))); + } + + void testGet() + { + for (int i = 0 ; i < canonicalList.size() ; ++i) + Assert.assertEquals(canonicalList.get(i), test.get(i)); + } + + void testSort(int subListDepth, int filterDepth, int sortDepth, int selectDepth) + { + final Comparator comparator = (o1, o2) -> + { + boolean f1 = o1 == canonicalList.get(0); + boolean f2 = o2 == canonicalList.get(0); + return f1 == f2 ? 0 : f1 ? 1 : -1; + }; + TestCase sorted = new TestCase<>(test.sorted(comparator), ImmutableList.sortedCopyOf(comparator, canonicalList)); + sorted.testAll(subListDepth, filterDepth, sortDepth - 1, selectDepth); + } + + private void testAll(int subListDepth, int filterDepth, int sortDepth, int selectDepth) + { + testEndpoints(); + testOrderOfIteration(); + testContains(); + testGet(); + testEquals(); + testSize(); + if (subListDepth > 0) + testSubList(subListDepth, filterDepth, sortDepth, selectDepth); + if (filterDepth > 0) + testFilter(subListDepth, filterDepth, sortDepth, selectDepth); + if (sortDepth > 0) + testSort(subListDepth, filterDepth, sortDepth, selectDepth); + if (selectDepth > 0) + testSelect(subListDepth, filterDepth, sortDepth, selectDepth); + } + + public void testAll() + { + testAll(2, 2, 2, 2); + } + } + + static class RangesAtEndpointTestCase extends TestCase + { + RangesAtEndpointTestCase(RangesAtEndpoint test, List canonicalList) + { + super(test, canonicalList); + } + + void testRanges() + { + Assert.assertEquals(ImmutableSet.copyOf(canonicalByRange.keySet()), ImmutableSet.copyOf(test.ranges())); + try + { + test.ranges().add(R5); + Assert.fail(); + } catch (UnsupportedOperationException e) {} + try + { + test.ranges().remove(R5); + Assert.fail(); + } catch (UnsupportedOperationException e) {} + + Assert.assertTrue(test.ranges().containsAll(canonicalByRange.keySet())); + for (Range range : canonicalByRange.keySet()) + Assert.assertTrue(test.ranges().contains(range)); + for (Range range : ImmutableList.of(R1, R2, R3, R4, R5, BROADCAST_RANGE)) + if (!canonicalByRange.containsKey(range)) + Assert.assertFalse(test.ranges().contains(range)); + } + + @Override + public void testOrderOfIteration() + { + super.testOrderOfIteration(); + Assert.assertEquals(new LinkedHashSet<>(Lists.transform(canonicalList, Replica::range)), test.ranges()); + } + + @Override + public void testAll() + { + super.testAll(); + testRanges(); + } + } + + private static final ImmutableList RANGES_AT_ENDPOINT = ImmutableList.of( + fullReplica(EP1, R1), + fullReplica(EP1, R2), + transientReplica(EP1, R3), + fullReplica(EP1, R4), + transientReplica(EP1, R5) + ); + + @Test + public void testRangesAtEndpoint() + { + ImmutableList canonical = RANGES_AT_ENDPOINT; + new RangesAtEndpointTestCase( + RangesAtEndpoint.copyOf(canonical), canonical + ).testAll(); + } + + @Test + public void testMutableRangesAtEndpoint() + { + ImmutableList canonical1 = RANGES_AT_ENDPOINT.subList(0, RANGES_AT_ENDPOINT.size()); + RangesAtEndpoint.Mutable test = new RangesAtEndpoint.Mutable(RANGES_AT_ENDPOINT.get(0).endpoint(), canonical1.size()); + test.addAll(canonical1, Conflict.NONE); + try + { // incorrect range + test.addAll(canonical1, Conflict.NONE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates + try + { // invalid endpoint; always error + test.add(fullReplica(EP2, BROADCAST_RANGE), Conflict.ALL); + Assert.fail(); + } catch (IllegalArgumentException e) { } + try + { // conflict on isFull/isTransient + test.add(fullReplica(EP1, R3), Conflict.DUPLICATE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.add(fullReplica(EP1, R3), Conflict.ALL); + + new RangesAtEndpointTestCase(test, canonical1).testAll(); + + RangesAtEndpoint view = test.asImmutableView(); + RangesAtEndpoint snapshot = view.subList(0, view.size()); + + ImmutableList canonical2 = RANGES_AT_ENDPOINT; + test.addAll(canonical2.reverse(), Conflict.DUPLICATE); + new TestCase<>(snapshot, canonical1).testAll(); + new TestCase<>(view, canonical2).testAll(); + new TestCase<>(test, canonical2).testAll(); + } + + private static final ImmutableList ENDPOINTS_FOR_X = ImmutableList.of( + fullReplica(EP1, R1), + fullReplica(EP2, R1), + transientReplica(EP3, R1), + fullReplica(EP4, R1), + transientReplica(EP5, R1) + ); + + @Test + public void testEndpointsForRange() + { + ImmutableList canonical = ENDPOINTS_FOR_X; + new TestCase<>( + EndpointsForRange.copyOf(canonical), canonical + ).testAll(); + } + + @Test + public void testMutableEndpointsForRange() + { + ImmutableList canonical1 = ENDPOINTS_FOR_X.subList(0, ENDPOINTS_FOR_X.size() - 1); + EndpointsForRange.Mutable test = new EndpointsForRange.Mutable(R1, canonical1.size()); + test.addAll(canonical1, Conflict.NONE); + try + { // incorrect range + test.addAll(canonical1, Conflict.NONE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates + try + { // incorrect range + test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL); + Assert.fail(); + } catch (IllegalArgumentException e) { } + try + { // conflict on isFull/isTransient + test.add(transientReplica(EP1, R1), Conflict.DUPLICATE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.add(transientReplica(EP1, R1), Conflict.ALL); + + new TestCase<>(test, canonical1).testAll(); + + EndpointsForRange view = test.asImmutableView(); + EndpointsForRange snapshot = view.subList(0, view.size()); + + ImmutableList canonical2 = ENDPOINTS_FOR_X; + test.addAll(canonical2.reverse(), Conflict.DUPLICATE); + new TestCase<>(snapshot, canonical1).testAll(); + new TestCase<>(view, canonical2).testAll(); + new TestCase<>(test, canonical2).testAll(); + } + + @Test + public void testEndpointsForToken() + { + ImmutableList canonical = ENDPOINTS_FOR_X; + new TestCase<>( + EndpointsForToken.copyOf(tk(1), canonical), canonical + ).testAll(); + } + + @Test + public void testMutableEndpointsForToken() + { + ImmutableList canonical1 = ENDPOINTS_FOR_X.subList(0, ENDPOINTS_FOR_X.size() - 1); + EndpointsForToken.Mutable test = new EndpointsForToken.Mutable(tk(1), canonical1.size()); + test.addAll(canonical1, Conflict.NONE); + try + { // incorrect range + test.addAll(canonical1, Conflict.NONE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.addAll(canonical1, Conflict.DUPLICATE); // we ignore exact duplicates + try + { // incorrect range + test.add(fullReplica(BROADCAST_EP, R2), Conflict.ALL); + Assert.fail(); + } catch (IllegalArgumentException e) { } + try + { // conflict on isFull/isTransient + test.add(transientReplica(EP1, R1), Conflict.DUPLICATE); + Assert.fail(); + } catch (IllegalArgumentException e) { } + test.add(transientReplica(EP1, R1), Conflict.ALL); + + new TestCase<>(test, canonical1).testAll(); + + EndpointsForToken view = test.asImmutableView(); + EndpointsForToken snapshot = view.subList(0, view.size()); + + ImmutableList canonical2 = ENDPOINTS_FOR_X; + test.addAll(canonical2.reverse(), Conflict.DUPLICATE); + new TestCase<>(snapshot, canonical1).testAll(); + new TestCase<>(view, canonical2).testAll(); + new TestCase<>(test, canonical2).testAll(); + } +} diff --git a/test/unit/org/apache/cassandra/locator/ReplicaUtils.java b/test/unit/org/apache/cassandra/locator/ReplicaUtils.java new file mode 100644 index 0000000000..66f538f915 --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/ReplicaUtils.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; + +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; + +public class ReplicaUtils +{ + public static final Range FULL_RANGE = new Range<>(Murmur3Partitioner.MINIMUM, Murmur3Partitioner.MINIMUM); + public static final AbstractBounds FULL_BOUNDS = new Range<>(Murmur3Partitioner.MINIMUM.minKeyBound(), Murmur3Partitioner.MINIMUM.maxKeyBound()); + + public static Replica full(InetAddressAndPort endpoint) + { + return fullReplica(endpoint, FULL_RANGE); + } + + public static Replica trans(InetAddressAndPort endpoint) + { + return transientReplica(endpoint, FULL_RANGE); + } +} diff --git a/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java new file mode 100644 index 0000000000..a0427db7c5 --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/ReplicationFactorTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.gms.Gossiper; + +public class ReplicationFactorTest +{ + + @BeforeClass + public static void setupClass() + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + Gossiper.instance.start(1); + } + + private static void assertRfParseFailure(String s) + { + try + { + ReplicationFactor.fromString(s); + Assert.fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + // expected + } + } + + private static void assertRfParse(String s, int expectedReplicas, int expectedTrans) + { + ReplicationFactor rf = ReplicationFactor.fromString(s); + Assert.assertEquals(expectedReplicas, rf.allReplicas); + Assert.assertEquals(expectedTrans, rf.transientReplicas()); + Assert.assertEquals(expectedReplicas - expectedTrans, rf.fullReplicas); + } + + @Test + public void parseTest() + { + assertRfParse("3", 3, 0); + assertRfParse("3/1", 3, 1); + + assertRfParse("5", 5, 0); + assertRfParse("5/2", 5, 2); + + assertRfParseFailure("-1"); + assertRfParseFailure("3/3"); + assertRfParseFailure("3/4"); + } +} diff --git a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java index a8caa721f4..e6a9365cbe 100644 --- a/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java +++ b/test/unit/org/apache/cassandra/locator/ReplicationStrategyEndpointCacheTest.java @@ -17,9 +17,7 @@ */ package org.apache.cassandra.locator; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; @@ -75,7 +73,7 @@ public class ReplicationStrategyEndpointCacheTest public void runEndpointsWereCachedTest(Class stratClass, Map configOptions) throws Exception { setup(stratClass, configOptions); - assert strategy.getNaturalEndpoints(searchToken).equals(strategy.getNaturalEndpoints(searchToken)); + assert strategy.getNaturalReplicasForToken(searchToken).equals(strategy.getNaturalReplicasForToken(searchToken)); } @Test @@ -89,34 +87,34 @@ public class ReplicationStrategyEndpointCacheTest public void runCacheRespectsTokenChangesTest(Class stratClass, Map configOptions) throws Exception { setup(stratClass, configOptions); - ArrayList initial; - ArrayList endpoints; + EndpointsForToken initial; + EndpointsForToken replicas; - endpoints = strategy.getNaturalEndpoints(searchToken); - assert endpoints.size() == 5 : StringUtils.join(endpoints, ","); + replicas = strategy.getNaturalReplicasForToken(searchToken); + assert replicas.size() == 5 : StringUtils.join(replicas, ","); // test token addition, in DC2 before existing token - initial = strategy.getNaturalEndpoints(searchToken); + initial = strategy.getNaturalReplicasForToken(searchToken); tmd.updateNormalToken(new BigIntegerToken(String.valueOf(35)), InetAddressAndPort.getByName("127.0.0.5")); - endpoints = strategy.getNaturalEndpoints(searchToken); - assert endpoints.size() == 5 : StringUtils.join(endpoints, ","); - assert !endpoints.equals(initial); + replicas = strategy.getNaturalReplicasForToken(searchToken); + assert replicas.size() == 5 : StringUtils.join(replicas, ","); + assert !replicas.equals(initial); // test token removal, newly created token - initial = strategy.getNaturalEndpoints(searchToken); + initial = strategy.getNaturalReplicasForToken(searchToken); tmd.removeEndpoint(InetAddressAndPort.getByName("127.0.0.5")); - endpoints = strategy.getNaturalEndpoints(searchToken); - assert endpoints.size() == 5 : StringUtils.join(endpoints, ","); - assert !endpoints.contains(InetAddressAndPort.getByName("127.0.0.5")); - assert !endpoints.equals(initial); + replicas = strategy.getNaturalReplicasForToken(searchToken); + assert replicas.size() == 5 : StringUtils.join(replicas, ","); + assert !replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.5")); + assert !replicas.equals(initial); // test token change - initial = strategy.getNaturalEndpoints(searchToken); + initial = strategy.getNaturalReplicasForToken(searchToken); //move .8 after search token but before other DC3 tmd.updateNormalToken(new BigIntegerToken(String.valueOf(25)), InetAddressAndPort.getByName("127.0.0.8")); - endpoints = strategy.getNaturalEndpoints(searchToken); - assert endpoints.size() == 5 : StringUtils.join(endpoints, ","); - assert !endpoints.equals(initial); + replicas = strategy.getNaturalReplicasForToken(searchToken); + assert replicas.size() == 5 : StringUtils.join(replicas, ","); + assert !replicas.equals(initial); } protected static class FakeSimpleStrategy extends SimpleStrategy @@ -128,11 +126,11 @@ public class ReplicationStrategyEndpointCacheTest super(keyspaceName, tokenMetadata, snitch, configOptions); } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - assert !called : "calculateNaturalEndpoints was already called, result should have been cached"; + assert !called : "calculateNaturalReplicas was already called, result should have been cached"; called = true; - return super.calculateNaturalEndpoints(token, metadata); + return super.calculateNaturalReplicas(token, metadata); } } @@ -145,11 +143,11 @@ public class ReplicationStrategyEndpointCacheTest super(keyspaceName, tokenMetadata, snitch, configOptions); } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - assert !called : "calculateNaturalEndpoints was already called, result should have been cached"; + assert !called : "calculateNaturalReplicas was already called, result should have been cached"; called = true; - return super.calculateNaturalEndpoints(token, metadata); + return super.calculateNaturalReplicas(token, metadata); } } @@ -162,11 +160,11 @@ public class ReplicationStrategyEndpointCacheTest super(keyspaceName, tokenMetadata, snitch, configOptions); } - public List calculateNaturalEndpoints(Token token, TokenMetadata metadata) + public EndpointsForRange calculateNaturalReplicas(Token token, TokenMetadata metadata) { - assert !called : "calculateNaturalEndpoints was already called, result should have been cached"; + assert !called : "calculateNaturalReplicas was already called, result should have been cached"; called = true; - return super.calculateNaturalEndpoints(token, metadata); + return super.calculateNaturalReplicas(token, metadata); } } diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index fe77b0e2a7..1e0c152030 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -19,14 +19,22 @@ package org.apache.cassandra.locator; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.IPartitioner; @@ -53,6 +61,7 @@ public class SimpleStrategyTest { SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1)); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); } @Test @@ -107,12 +116,12 @@ public class SimpleStrategyTest for (int i = 0; i < keyTokens.length; i++) { - List endpoints = strategy.getNaturalEndpoints(keyTokens[i]); - assertEquals(strategy.getReplicationFactor(), endpoints.size()); + EndpointsForToken replicas = strategy.getNaturalReplicasForToken(keyTokens[i]); + assertEquals(strategy.getReplicationFactor().allReplicas, replicas.size()); List correctEndpoints = new ArrayList<>(); - for (int j = 0; j < endpoints.size(); j++) + for (int j = 0; j < replicas.size(); j++) correctEndpoints.add(hosts.get((i + j + 1) % hosts.size())); - assertEquals(new HashSet<>(correctEndpoints), new HashSet<>(endpoints)); + assertEquals(new HashSet<>(correctEndpoints), replicas.endpoints()); } } } @@ -154,30 +163,80 @@ public class SimpleStrategyTest PendingRangeCalculatorService.calculatePendingRanges(strategy, keyspaceName); - int replicationFactor = strategy.getReplicationFactor(); + int replicationFactor = strategy.getReplicationFactor().allReplicas; for (int i = 0; i < keyTokens.length; i++) { - Collection endpoints = tmd.getWriteEndpoints(keyTokens[i], keyspaceName, strategy.getNaturalEndpoints(keyTokens[i])); - assertTrue(endpoints.size() >= replicationFactor); + EndpointsForToken replicas = tmd.getWriteEndpoints(keyTokens[i], keyspaceName, strategy.getNaturalReplicasForToken(keyTokens[i])); + assertTrue(replicas.size() >= replicationFactor); for (int j = 0; j < replicationFactor; j++) { //Check that the old nodes are definitely included - assertTrue(endpoints.contains(hosts.get((i + j + 1) % hosts.size()))); + assertTrue(replicas.endpoints().contains(hosts.get((i + j + 1) % hosts.size()))); } // bootstrapEndpoint should be in the endpoints for i in MAX-RF to MAX, but not in any earlier ep. if (i < RING_SIZE - replicationFactor) - assertFalse(endpoints.contains(bootstrapEndpoint)); + assertFalse(replicas.endpoints().contains(bootstrapEndpoint)); else - assertTrue(endpoints.contains(bootstrapEndpoint)); + assertTrue(replicas.endpoints().contains(bootstrapEndpoint)); } } StorageServiceAccessor.setTokenMetadata(oldTmd); } + private static Token tk(long t) + { + return new Murmur3Partitioner.LongToken(t); + } + + private static Range range(long l, long r) + { + return new Range<>(tk(l), tk(r)); + } + + @Test + public void transientReplica() throws Exception + { + IEndpointSnitch snitch = new SimpleSnitch(); + DatabaseDescriptor.setEndpointSnitch(snitch); + + List endpoints = Lists.newArrayList(InetAddressAndPort.getByName("127.0.0.1"), + InetAddressAndPort.getByName("127.0.0.2"), + InetAddressAndPort.getByName("127.0.0.3"), + InetAddressAndPort.getByName("127.0.0.4")); + + Multimap tokens = HashMultimap.create(); + tokens.put(endpoints.get(0), tk(100)); + tokens.put(endpoints.get(1), tk(200)); + tokens.put(endpoints.get(2), tk(300)); + tokens.put(endpoints.get(3), tk(400)); + TokenMetadata metadata = new TokenMetadata(); + metadata.updateNormalTokens(tokens); + + Map configOptions = new HashMap(); + configOptions.put("replication_factor", "3/1"); + + SimpleStrategy strategy = new SimpleStrategy("ks", metadata, snitch, configOptions); + + Range range1 = range(400, 100); + Assert.assertEquals(EndpointsForToken.of(range1.right, + Replica.fullReplica(endpoints.get(0), range1), + Replica.fullReplica(endpoints.get(1), range1), + Replica.transientReplica(endpoints.get(2), range1)), + strategy.getNaturalReplicasForToken(tk(99))); + + + Range range2 = range(100, 200); + Assert.assertEquals(EndpointsForToken.of(range2.right, + Replica.fullReplica(endpoints.get(1), range2), + Replica.fullReplica(endpoints.get(2), range2), + Replica.transientReplica(endpoints.get(3), range2)), + strategy.getNaturalReplicasForToken(tk(101))); + } + private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd) { KeyspaceMetadata ksmd = Schema.instance.getKeyspaceMetadata(keyspaceName); diff --git a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java index b589d2d595..ae8c01171b 100644 --- a/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java +++ b/test/unit/org/apache/cassandra/locator/TokenMetadataTest.java @@ -118,7 +118,7 @@ public class TokenMetadataTest } @Override - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } @@ -165,7 +165,7 @@ public class TokenMetadataTest } @Override - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } @@ -216,7 +216,7 @@ public class TokenMetadataTest } @Override - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } @@ -263,7 +263,7 @@ public class TokenMetadataTest } @Override - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } diff --git a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java index d97cdb8fbe..e226d32e75 100644 --- a/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java +++ b/test/unit/org/apache/cassandra/net/WriteCallbackInfoTest.java @@ -25,19 +25,20 @@ import org.junit.Test; import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.net.MessagingService.Verb; import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.locator.ReplicaUtils.full; + public class WriteCallbackInfoTest { @BeforeClass @@ -65,7 +66,7 @@ public class WriteCallbackInfoTest ? new Commit(UUID.randomUUID(), new PartitionUpdate.Builder(metadata, ByteBufferUtil.EMPTY_BYTE_BUFFER, RegularAndStaticColumns.NONE, 1).build()) : new Mutation(PartitionUpdate.simpleBuilder(metadata, "").build()); - WriteCallbackInfo wcbi = new WriteCallbackInfo(InetAddressAndPort.getByName("192.168.1.1"), null, new MessageOut(verb, payload, null), null, cl, allowHints); + WriteCallbackInfo wcbi = new WriteCallbackInfo(full(InetAddressAndPort.getByName("192.168.1.1")), null, new MessageOut(verb, payload, null), null, cl, allowHints); Assert.assertEquals(expectHint, wcbi.shouldHint()); if (expectHint) { diff --git a/test/unit/org/apache/cassandra/net/async/OutboundMessagingConnectionTest.java b/test/unit/org/apache/cassandra/net/async/OutboundMessagingConnectionTest.java index 6a8dc8372e..379031c6cf 100644 --- a/test/unit/org/apache/cassandra/net/async/OutboundMessagingConnectionTest.java +++ b/test/unit/org/apache/cassandra/net/async/OutboundMessagingConnectionTest.java @@ -46,6 +46,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.locator.AbstractEndpointSnitch; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.MessagingServiceTest; @@ -172,7 +173,7 @@ public class OutboundMessagingConnectionTest return nodeToDc.get(endpoint); } - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } diff --git a/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java index 2044106882..418d7de25d 100644 --- a/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairRunnableTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.repair; -import java.net.InetAddress; import java.util.Collections; import java.util.List; import java.util.Set; @@ -29,7 +28,6 @@ import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.repair.RepairRunnable.CommonRange; import static org.apache.cassandra.repair.RepairRunnable.filterCommonRanges; @@ -41,7 +39,7 @@ public class RepairRunnableTest extends AbstractRepairTest @Test public void filterCommonIncrementalRangesNotForced() throws Exception { - CommonRange cr = new CommonRange(PARTICIPANTS, ALL_RANGES); + CommonRange cr = new CommonRange(PARTICIPANTS, Collections.emptySet(), ALL_RANGES); List expected = Lists.newArrayList(cr); List actual = filterCommonRanges(expected, Collections.emptySet(), false); @@ -52,13 +50,13 @@ public class RepairRunnableTest extends AbstractRepairTest @Test public void forceFilterCommonIncrementalRanges() throws Exception { - CommonRange cr1 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2), Sets.newHashSet(RANGE1, RANGE2)); - CommonRange cr2 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), Sets.newHashSet(RANGE3)); + CommonRange cr1 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2), Collections.emptySet(), Sets.newHashSet(RANGE1, RANGE2)); + CommonRange cr2 = new CommonRange(Sets.newHashSet(PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), Collections.emptySet(), Sets.newHashSet(RANGE3)); Set liveEndpoints = Sets.newHashSet(PARTICIPANT2, PARTICIPANT3); // PARTICIPANT1 is excluded List initial = Lists.newArrayList(cr1, cr2); - List expected = Lists.newArrayList(new CommonRange(Sets.newHashSet(PARTICIPANT2), Sets.newHashSet(RANGE1, RANGE2)), - new CommonRange(Sets.newHashSet(PARTICIPANT2, PARTICIPANT3), Sets.newHashSet(RANGE3))); + List expected = Lists.newArrayList(new CommonRange(Sets.newHashSet(PARTICIPANT2), Collections.emptySet(), Sets.newHashSet(RANGE1, RANGE2)), + new CommonRange(Sets.newHashSet(PARTICIPANT2, PARTICIPANT3), Collections.emptySet(), Sets.newHashSet(RANGE3))); List actual = filterCommonRanges(initial, liveEndpoints, true); Assert.assertEquals(expected, actual); diff --git a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java index 54f0511f05..e77d657027 100644 --- a/test/unit/org/apache/cassandra/repair/RepairSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairSessionTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.repair; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -62,9 +63,10 @@ public class RepairSessionTest IPartitioner p = Murmur3Partitioner.instance; Range repairRange = new Range<>(p.getToken(ByteBufferUtil.bytes(0)), p.getToken(ByteBufferUtil.bytes(100))); Set endpoints = Sets.newHashSet(remote); - RepairSession session = new RepairSession(parentSessionId, sessionId, Arrays.asList(repairRange), + RepairSession session = new RepairSession(parentSessionId, sessionId, + new CommonRange(endpoints, Collections.emptySet(), Arrays.asList(repairRange)), "Keyspace1", RepairParallelism.SEQUENTIAL, - endpoints, false, false, false, + false, false, false, PreviewKind.NONE, false, "Standard1"); // perform convict diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricLocalSyncTaskTest.java similarity index 73% rename from test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java rename to test/unit/org/apache/cassandra/repair/SymmetricLocalSyncTaskTest.java index 802a673acb..92ae172e2f 100644 --- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/SymmetricLocalSyncTaskTest.java @@ -20,10 +20,11 @@ package org.apache.cassandra.repair; import java.util.Arrays; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.junit.BeforeClass; import org.junit.Test; @@ -40,8 +41,11 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.streaming.StreamCoordinator; +import org.apache.cassandra.streaming.DefaultConnectionFactory; import org.apache.cassandra.streaming.StreamPlan; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; import org.apache.cassandra.utils.MerkleTrees; @@ -53,7 +57,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class LocalSyncTaskTest extends AbstractRepairTest +public class SymmetricLocalSyncTaskTest extends AbstractRepairTest { private static final IPartitioner partitioner = Murmur3Partitioner.instance; public static final String KEYSPACE1 = "DifferencerTest"; @@ -73,7 +77,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest } /** - * When there is no difference between two, LocalSyncTask should return stats with 0 difference. + * When there is no difference between two, SymmetricLocalSyncTask should return stats with 0 difference. */ @Test public void testNoDifference() throws Throwable @@ -92,7 +96,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest // note: we reuse the same endpoint which is bogus in theory but fine here TreeResponse r1 = new TreeResponse(ep1, tree1); TreeResponse r2 = new TreeResponse(ep2, tree2); - LocalSyncTask task = new LocalSyncTask(desc, r1, r2, NO_PENDING_REPAIR, false, PreviewKind.NONE); + SymmetricLocalSyncTask task = new SymmetricLocalSyncTask(desc, r1, r2, false, NO_PENDING_REPAIR, false, PreviewKind.NONE); task.run(); assertEquals(0, task.get().numberOfDifferences); @@ -130,8 +134,18 @@ public class LocalSyncTaskTest extends AbstractRepairTest // note: we reuse the same endpoint which is bogus in theory but fine here TreeResponse r1 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.1"), tree1); TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2); - LocalSyncTask task = new LocalSyncTask(desc, r1, r2, NO_PENDING_REPAIR, false, PreviewKind.NONE); - task.run(); + SymmetricLocalSyncTask task = new SymmetricLocalSyncTask(desc, r1, r2, false, NO_PENDING_REPAIR, false, PreviewKind.NONE); + DefaultConnectionFactory.MAX_CONNECT_ATTEMPTS = 1; + DefaultConnectionFactory.MAX_WAIT_TIME_NANOS = TimeUnit.SECONDS.toNanos(2); + try + { + task.run(); + } + finally + { + DefaultConnectionFactory.MAX_WAIT_TIME_NANOS = TimeUnit.SECONDS.toNanos(30); + DefaultConnectionFactory.MAX_CONNECT_ATTEMPTS = 3; + } // ensure that the changed range was recorded assertEquals("Wrong differing ranges", interesting.size(), task.getCurrentStat().numberOfDifferences); @@ -147,13 +161,21 @@ public class LocalSyncTaskTest extends AbstractRepairTest TreeResponse r1 = new TreeResponse(PARTICIPANT1, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1, r2, NO_PENDING_REPAIR, false, PreviewKind.NONE); + SymmetricLocalSyncTask task = new SymmetricLocalSyncTask(desc, r1, r2, false, NO_PENDING_REPAIR, false, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(PARTICIPANT1, Lists.newArrayList(RANGE1)); assertEquals(NO_PENDING_REPAIR, plan.getPendingRepair()); assertTrue(plan.getFlushBeforeTransfer()); } + private static void assertNumInOut(StreamPlan plan, int expectedIncoming, int expectedOutgoing) + { + StreamCoordinator coordinator = plan.getCoordinator(); + StreamSession session = Iterables.getOnlyElement(coordinator.getAllStreamSessions()); + assertEquals(expectedIncoming, session.getNumRequests()); + assertEquals(expectedOutgoing, session.getNumTransfers()); + } + @Test public void incrementalRepairStreamPlan() throws Exception { @@ -164,11 +186,30 @@ public class LocalSyncTaskTest extends AbstractRepairTest TreeResponse r1 = new TreeResponse(PARTICIPANT1, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); - LocalSyncTask task = new LocalSyncTask(desc, r1, r2, desc.parentSessionId, false, PreviewKind.NONE); + SymmetricLocalSyncTask task = new SymmetricLocalSyncTask(desc, r1, r2, false, desc.parentSessionId, false, PreviewKind.NONE); StreamPlan plan = task.createStreamPlan(PARTICIPANT1, Lists.newArrayList(RANGE1)); assertEquals(desc.parentSessionId, plan.getPendingRepair()); assertFalse(plan.getFlushBeforeTransfer()); + assertNumInOut(plan, 1, 1); + } + + /** + * Don't reciprocate streams if the other endpoint is a transient replica + */ + @Test + public void transientStreamPlan() + { + UUID sessionID = registerSession(cfs, true, true); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(sessionID); + RepairJobDesc desc = new RepairJobDesc(sessionID, UUIDGen.getTimeUUID(), KEYSPACE1, CF_STANDARD, prs.getRanges()); + + TreeResponse r1 = new TreeResponse(PARTICIPANT1, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); + TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); + + SymmetricLocalSyncTask task = new SymmetricLocalSyncTask(desc, r1, r2, true, desc.parentSessionId, false, PreviewKind.NONE); + StreamPlan plan = task.createStreamPlan(PARTICIPANT2, Lists.newArrayList(RANGE1)); + assertNumInOut(plan, 1, 0); } private MerkleTrees createInitialTree(RepairJobDesc desc, IPartitioner partitioner) diff --git a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java new file mode 100644 index 0000000000..06f968f6fc --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.messages.RepairMessage; +import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.UUIDGen; + +public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest +{ + private static final RepairJobDesc DESC = new RepairJobDesc(UUIDGen.getTimeUUID(), UUIDGen.getTimeUUID(), "ks", "tbl", ALL_RANGES); + private static final List> RANGE_LIST = ImmutableList.of(RANGE1); + + private static class InstrumentedSymmetricRemoteSyncTask extends SymmetricRemoteSyncTask + { + public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2) + { + super(DESC, new TreeResponse(e1, null), new TreeResponse(e2, null), PreviewKind.NONE); + } + + RepairMessage sentMessage = null; + InetAddressAndPort sentTo = null; + + @Override + void sendRequest(RepairMessage request, InetAddressAndPort to) + { + Assert.assertNull(sentMessage); + Assert.assertNotNull(request); + Assert.assertNotNull(to); + sentMessage = request; + sentTo = to; + } + } + + @Test + public void normalSync() + { + InstrumentedSymmetricRemoteSyncTask syncTask = new InstrumentedSymmetricRemoteSyncTask(PARTICIPANT1, PARTICIPANT2); + syncTask.startSync(RANGE_LIST); + + Assert.assertNotNull(syncTask.sentMessage); + Assert.assertSame(SyncRequest.class, syncTask.sentMessage.getClass()); + Assert.assertEquals(PARTICIPANT1, syncTask.sentTo); + } +} diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java index 3ea888d5a6..a7e8272910 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java +++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionAccessor.java @@ -44,12 +44,13 @@ public class LocalSessionAccessor ARS.consistent.local.putSessionUnsafe(session); } - public static void finalizeUnsafe(UUID sessionID) + public static long finalizeUnsafe(UUID sessionID) { LocalSession session = ARS.consistent.local.getSession(sessionID); assert session != null; session.setState(ConsistentSession.State.FINALIZED); ARS.consistent.local.save(session); + return session.repairedAt; } public static void failUnsafe(UUID sessionID) diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java index d368510b17..e387c41d32 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java @@ -40,10 +40,9 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.repair.AbstractRepairTest; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.KeyspaceRepairManager; @@ -136,7 +135,11 @@ public class LocalSessionTest extends AbstractRepairTest boolean prepareSessionCalled = false; @Override - ListenableFuture prepareSession(KeyspaceRepairManager repairManager, UUID sessionID, Collection tables, Collection> ranges, ExecutorService executor) + ListenableFuture prepareSession(KeyspaceRepairManager repairManager, + UUID sessionID, + Collection tables, + RangesAtEndpoint ranges, + ExecutorService executor) { prepareSessionCalled = true; if (prepareSessionFuture != null) diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 98bf9ca3ba..7a6b01154b 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -127,7 +127,7 @@ public class MockSchema } SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, header) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, RANDOM_ACCESS_READER_FACTORY.sharedCopy(), RANDOM_ACCESS_READER_FACTORY.sharedCopy(), indexSummary.sharedCopy(), diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 294731ab4b..4f7cde035a 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -20,8 +20,6 @@ package org.apache.cassandra.service; import java.util.*; -import javax.xml.crypto.Data; - import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import org.junit.Assert; @@ -36,13 +34,13 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.View; -import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.streaming.PreviewKind; @@ -107,13 +105,13 @@ public class ActiveRepairServiceTest public void testGetNeighborsPlusOne() throws Throwable { // generate rf+1 nodes, and ensure that all nodes are returned - Set expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + Set expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); expected.remove(FBUtilities.getBroadcastAddressAndPort()); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null)); + neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); } assertEquals(expected, neighbors); } @@ -124,19 +122,19 @@ public class ActiveRepairServiceTest TokenMetadata tmd = StorageService.instance.getTokenMetadata(); // generate rf*2 nodes, and ensure that only neighbors specified by the ARS are returned - addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy(); Set expected = new HashSet<>(); - for (Range replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddressAndPort())) + for (Replica replica : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort())) { - expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange)); + expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replica.range()).endpoints()); } expected.remove(FBUtilities.getBroadcastAddressAndPort()); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null)); + neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, null, null).endpoints()); } assertEquals(expected, neighbors); } @@ -147,18 +145,18 @@ public class ActiveRepairServiceTest TokenMetadata tmd = StorageService.instance.getTokenMetadata(); // generate rf+1 nodes, and ensure that all nodes are returned - Set expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + Set expected = addTokens(1 + Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); expected.remove(FBUtilities.getBroadcastAddressAndPort()); // remove remote endpoints TokenMetadata.Topology topology = tmd.cloneOnlyTokenMap().getTopology(); HashSet localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter())); expected = Sets.intersection(expected, localEndpoints); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null)); + neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); } assertEquals(expected, neighbors); } @@ -169,12 +167,12 @@ public class ActiveRepairServiceTest TokenMetadata tmd = StorageService.instance.getTokenMetadata(); // generate rf*2 nodes, and ensure that only neighbors specified by the ARS are returned - addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy(); Set expected = new HashSet<>(); - for (Range replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddressAndPort())) + for (Replica replica : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort())) { - expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange)); + expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replica.range()).endpoints()); } expected.remove(FBUtilities.getBroadcastAddressAndPort()); // remove remote endpoints @@ -182,11 +180,11 @@ public class ActiveRepairServiceTest HashSet localEndpoints = Sets.newHashSet(topology.getDatacenterEndpoints().get(DatabaseDescriptor.getLocalDataCenter())); expected = Sets.intersection(expected, localEndpoints); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); Set neighbors = new HashSet<>(); for (Range range : ranges) { - neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null)); + neighbors.addAll(ActiveRepairService.getNeighbors(KEYSPACE5, ranges, range, Arrays.asList(DatabaseDescriptor.getLocalDataCenter()), null).endpoints()); } assertEquals(expected, neighbors); } @@ -197,30 +195,30 @@ public class ActiveRepairServiceTest TokenMetadata tmd = StorageService.instance.getTokenMetadata(); // generate rf*2 nodes, and ensure that only neighbors specified by the hosts are returned - addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); AbstractReplicationStrategy ars = Keyspace.open(KEYSPACE5).getReplicationStrategy(); List expected = new ArrayList<>(); - for (Range replicaRange : ars.getAddressRanges().get(FBUtilities.getBroadcastAddressAndPort())) + for (Replica replicas : ars.getAddressReplicas().get(FBUtilities.getBroadcastAddressAndPort())) { - expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicaRange)); + expected.addAll(ars.getRangeAddresses(tmd.cloneOnlyTokenMap()).get(replicas.range()).endpoints()); } expected.remove(FBUtilities.getBroadcastAddressAndPort()); Collection hosts = Arrays.asList(FBUtilities.getBroadcastAddressAndPort().toString(),expected.get(0).toString()); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); assertEquals(expected.get(0), ActiveRepairService.getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), - null, hosts).iterator().next()); + null, hosts).endpoints().iterator().next()); } @Test(expected = IllegalArgumentException.class) public void testGetNeighborsSpecifiedHostsWithNoLocalHost() throws Throwable { - addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor()); + addTokens(2 * Keyspace.open(KEYSPACE5).getReplicationStrategy().getReplicationFactor().allReplicas); //Dont give local endpoint Collection hosts = Arrays.asList("127.0.0.3"); - Collection> ranges = StorageService.instance.getLocalRanges(KEYSPACE5); + Iterable> ranges = StorageService.instance.getLocalReplicas(KEYSPACE5).ranges(); ActiveRepairService.getNeighbors(KEYSPACE5, ranges, ranges.iterator().next(), null, hosts); } diff --git a/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java new file mode 100644 index 0000000000..63973eaf4e --- /dev/null +++ b/test/unit/org/apache/cassandra/service/BootstrapTransientTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.locator.EndpointsByReplica; +import org.apache.cassandra.locator.EndpointsForRange; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.OrderPreservingPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.RangeStreamer; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.AbstractEndpointSnitch; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.utils.Pair; + +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; +import static org.apache.cassandra.service.StorageServiceTest.assertMultimapEqualsIgnoreOrder; + +/** + * This is also fairly effectively testing source retrieval for bootstrap as well since RangeStreamer + * is used to calculate the endpoints to fetch from and check they are alive for both RangeRelocator (move) and + * bootstrap (RangeRelocator). + */ +public class BootstrapTransientTest +{ + static InetAddressAndPort aAddress; + static InetAddressAndPort bAddress; + static InetAddressAndPort cAddress; + static InetAddressAndPort dAddress; + + @BeforeClass + public static void setUpClass() throws Exception + { + aAddress = InetAddressAndPort.getByName("127.0.0.1"); + bAddress = InetAddressAndPort.getByName("127.0.0.2"); + cAddress = InetAddressAndPort.getByName("127.0.0.3"); + dAddress = InetAddressAndPort.getByName("127.0.0.4"); + } + + private final List downNodes = new ArrayList<>(); + Predicate alivePredicate = replica -> !downNodes.contains(replica.endpoint()); + + private final List sourceFilterDownNodes = new ArrayList<>(); + private final Collection> sourceFilters = Collections.singleton(replica -> !sourceFilterDownNodes.contains(replica.endpoint())); + + @After + public void clearDownNode() + { + downNodes.clear(); + sourceFilterDownNodes.clear(); + } + + @BeforeClass + public static void setupDD() + { + DatabaseDescriptor.daemonInitialization(); + } + + Token tenToken = new OrderPreservingPartitioner.StringToken("00010"); + Token twentyToken = new OrderPreservingPartitioner.StringToken("00020"); + Token thirtyToken = new OrderPreservingPartitioner.StringToken("00030"); + Token fourtyToken = new OrderPreservingPartitioner.StringToken("00040"); + + Range aRange = new Range<>(thirtyToken, tenToken); + Range bRange = new Range<>(tenToken, twentyToken); + Range cRange = new Range<>(twentyToken, thirtyToken); + Range dRange = new Range<>(thirtyToken, fourtyToken); + + RangesAtEndpoint toFetch = RangesAtEndpoint.of(new Replica(dAddress, dRange, true), + new Replica(dAddress, cRange, true), + new Replica(dAddress, bRange, false)); + + @Test + public void testRangeStreamerRangesToFetch() throws Exception + { + EndpointsByReplica expectedResult = new EndpointsByReplica(ImmutableMap.of( + fullReplica(dAddress, dRange), EndpointsForRange.builder(aRange).add(fullReplica(bAddress, aRange)).add(transientReplica(cAddress, aRange)).build(), + fullReplica(dAddress, cRange), EndpointsForRange.builder(cRange).add(fullReplica(cAddress, cRange)).add(transientReplica(bAddress, cRange)).build(), + transientReplica(dAddress, bRange), EndpointsForRange.builder(bRange).add(transientReplica(aAddress, bRange)).build())); + + invokeCalculateRangesToFetchWithPreferredEndpoints(toFetch, constructTMDs(), expectedResult); + } + + private Pair constructTMDs() + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(aRange.right, aAddress); + tmd.updateNormalToken(bRange.right, bAddress); + tmd.updateNormalToken(cRange.right, cAddress); + TokenMetadata updated = tmd.cloneOnlyTokenMap(); + updated.updateNormalToken(dRange.right, dAddress); + + return Pair.create(tmd, updated); + } + + private void invokeCalculateRangesToFetchWithPreferredEndpoints(ReplicaCollection toFetch, + Pair tmds, + EndpointsByReplica expectedResult) + { + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + + EndpointsByReplica result = RangeStreamer.calculateRangesToFetchWithPreferredEndpoints((address, replicas) -> replicas, + simpleStrategy(tmds.left), + toFetch, + true, + tmds.left, + tmds.right, + alivePredicate, + "OldNetworkTopologyStrategyTest", + sourceFilters); + result.asMap().forEach((replica, list) -> System.out.printf("Replica %s, sources %s%n", replica, list)); + assertMultimapEqualsIgnoreOrder(expectedResult, result); + + } + + private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd) + { + IEndpointSnitch snitch = new AbstractEndpointSnitch() + { + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) + { + return 0; + } + + public String getRack(InetAddressAndPort endpoint) + { + return "R1"; + } + + public String getDatacenter(InetAddressAndPort endpoint) + { + return "DC1"; + } + }; + + return new SimpleStrategy("MoveTransientTest", + tmd, + snitch, + com.google.common.collect.ImmutableMap.of("replication_factor", "3/1")); + } + +} diff --git a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java index 8ddc4f03df..3c4748ee33 100644 --- a/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java +++ b/test/unit/org/apache/cassandra/service/LeaveAndBootstrapTest.java @@ -130,10 +130,10 @@ public class LeaveAndBootstrapTest strategy = getStrategy(keyspaceName, tmd); for (Token token : keyTokens) { - int replicationFactor = strategy.getReplicationFactor(); + int replicationFactor = strategy.getReplicationFactor().allReplicas; - HashSet actual = new HashSet<>(tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalEndpoints(token, tmd.cloneOnlyTokenMap()))); - HashSet expected = new HashSet<>(); + Set actual = tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalReplicas(token, tmd.cloneOnlyTokenMap()).forToken(token)).endpoints(); + Set expected = new HashSet<>(); for (int i = 0; i < replicationFactor; i++) { @@ -198,8 +198,6 @@ public class LeaveAndBootstrapTest ApplicationState.STATUS, valueFactory.bootstrapping(Collections.singleton(keyTokens.get(7)))); - Collection endpoints = null; - /* don't require test update every time a new keyspace is added to test/conf/cassandra.yaml */ Map keyspaceStrategyMap = new HashMap(); for (int i=1; i<=4; i++) @@ -263,18 +261,18 @@ public class LeaveAndBootstrapTest for (int i = 0; i < keyTokens.size(); i++) { - endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints(); assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size()); assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints)); } // just to be sure that things still work according to the old tests, run them: - if (strategy.getReplicationFactor() != 3) + if (strategy.getReplicationFactor().allReplicas != 3) continue; // tokens 5, 15 and 25 should go three nodes for (int i=0; i<3; ++i) { - endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(i+1))); assertTrue(endpoints.contains(hosts.get(i+2))); @@ -282,7 +280,7 @@ public class LeaveAndBootstrapTest } // token 35 should go to nodes 4, 5, 6, 7 and boot1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3))).endpoints(); assertEquals(5, endpoints.size()); assertTrue(endpoints.contains(hosts.get(4))); assertTrue(endpoints.contains(hosts.get(5))); @@ -291,7 +289,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(boot1)); // token 45 should go to nodes 5, 6, 7, 0, boot1 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4))).endpoints(); assertEquals(6, endpoints.size()); assertTrue(endpoints.contains(hosts.get(5))); assertTrue(endpoints.contains(hosts.get(6))); @@ -301,7 +299,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(boot2)); // token 55 should go to nodes 6, 7, 8, 0, 1, boot1 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5))).endpoints(); assertEquals(7, endpoints.size()); assertTrue(endpoints.contains(hosts.get(6))); assertTrue(endpoints.contains(hosts.get(7))); @@ -312,7 +310,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(boot2)); // token 65 should go to nodes 7, 8, 9, 0, 1 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6))).endpoints(); assertEquals(6, endpoints.size()); assertTrue(endpoints.contains(hosts.get(7))); assertTrue(endpoints.contains(hosts.get(8))); @@ -322,7 +320,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(boot2)); // token 75 should to go nodes 8, 9, 0, 1, 2 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7))).endpoints(); assertEquals(6, endpoints.size()); assertTrue(endpoints.contains(hosts.get(8))); assertTrue(endpoints.contains(hosts.get(9))); @@ -332,7 +330,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(boot2)); // token 85 should go to nodes 9, 0, 1 and 2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8))).endpoints(); assertEquals(4, endpoints.size()); assertTrue(endpoints.contains(hosts.get(9))); assertTrue(endpoints.contains(hosts.get(0))); @@ -340,7 +338,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(hosts.get(2))); // token 95 should go to nodes 0, 1 and 2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(0))); assertTrue(endpoints.contains(hosts.get(1))); @@ -385,18 +383,18 @@ public class LeaveAndBootstrapTest for (int i = 0; i < keyTokens.size(); i++) { - endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints(); assertEquals(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).size(), endpoints.size()); assertTrue(expectedEndpoints.get(keyspaceName).get(keyTokens.get(i)).containsAll(endpoints)); } - if (strategy.getReplicationFactor() != 3) + if (strategy.getReplicationFactor().allReplicas != 3) continue; // leave this stuff in to guarantee the old tests work the way they were supposed to. // tokens 5, 15 and 25 should go three nodes for (int i=0; i<3; ++i) { - endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(i+1))); assertTrue(endpoints.contains(hosts.get(i+2))); @@ -404,21 +402,21 @@ public class LeaveAndBootstrapTest } // token 35 goes to nodes 4, 5 and boot1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3))); + Collection endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(4))); assertTrue(endpoints.contains(hosts.get(5))); assertTrue(endpoints.contains(boot1)); // token 45 goes to nodes 5, boot1 and node7 - endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(5))); assertTrue(endpoints.contains(boot1)); assertTrue(endpoints.contains(hosts.get(7))); // token 55 goes to boot1, 7, boot2, 8 and 0 - endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5))).endpoints(); assertEquals(5, endpoints.size()); assertTrue(endpoints.contains(boot1)); assertTrue(endpoints.contains(hosts.get(7))); @@ -427,7 +425,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(hosts.get(0))); // token 65 goes to nodes 7, boot2, 8, 0 and 1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6))).endpoints(); assertEquals(5, endpoints.size()); assertTrue(endpoints.contains(hosts.get(7))); assertTrue(endpoints.contains(boot2)); @@ -436,7 +434,7 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(hosts.get(1))); // token 75 goes to nodes boot2, 8, 0, 1 and 2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7))).endpoints(); assertEquals(5, endpoints.size()); assertTrue(endpoints.contains(boot2)); assertTrue(endpoints.contains(hosts.get(8))); @@ -445,14 +443,14 @@ public class LeaveAndBootstrapTest assertTrue(endpoints.contains(hosts.get(2))); // token 85 goes to nodes 0, 1 and 2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(0))); assertTrue(endpoints.contains(hosts.get(1))); assertTrue(endpoints.contains(hosts.get(2))); // token 95 goes to nodes 0, 1 and 2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9))); + endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9))).endpoints(); assertEquals(3, endpoints.size()); assertTrue(endpoints.contains(hosts.get(0))); assertTrue(endpoints.contains(hosts.get(1))); diff --git a/test/unit/org/apache/cassandra/service/MoveTest.java b/test/unit/org/apache/cassandra/service/MoveTest.java index 7321fbaf8d..731a25d407 100644 --- a/test/unit/org/apache/cassandra/service/MoveTest.java +++ b/test/unit/org/apache/cassandra/service/MoveTest.java @@ -23,13 +23,21 @@ import java.net.UnknownHostException; import java.util.*; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.RangesByEndpoint; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.schema.MigrationManager; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; @@ -492,24 +500,25 @@ public class MoveTest tmd.updateNormalToken(new BigIntegerToken(String.valueOf(token)), host); } - private Map.Entry, Collection> generatePendingMapEntry(int start, int end, String... endpoints) throws UnknownHostException + private Map.Entry, EndpointsForRange> generatePendingMapEntry(int start, int end, String... endpoints) throws UnknownHostException { - Map, Collection> pendingRanges = new HashMap<>(); - pendingRanges.put(generateRange(start, end), makeAddrs(endpoints)); + Map, EndpointsForRange> pendingRanges = new HashMap<>(); + Range range = generateRange(start, end); + pendingRanges.put(range, makeReplicas(range, endpoints)); return pendingRanges.entrySet().iterator().next(); } - private Map, Collection> generatePendingRanges(Map.Entry, Collection>... entries) + private Map, EndpointsForRange> generatePendingRanges(Map.Entry, EndpointsForRange>... entries) { - Map, Collection> pendingRanges = new HashMap<>(); - for(Map.Entry, Collection> entry : entries) + Map, EndpointsForRange> pendingRanges = new HashMap<>(); + for(Map.Entry, EndpointsForRange> entry : entries) { pendingRanges.put(entry.getKey(), entry.getValue()); } return pendingRanges; } - private void assertPendingRanges(TokenMetadata tmd, Map, Collection> pendingRanges, String keyspaceName) throws ConfigurationException + private void assertPendingRanges(TokenMetadata tmd, Map, EndpointsForRange> pendingRanges, String keyspaceName) throws ConfigurationException { boolean keyspaceFound = false; for (String nonSystemKeyspaceName : Schema.instance.getNonLocalStrategyKeyspaces()) @@ -523,15 +532,15 @@ public class MoveTest assert keyspaceFound; } - private void assertMaps(Map, Collection> expected, PendingRangeMaps actual) + private void assertMaps(Map, EndpointsForRange> expected, PendingRangeMaps actual) { int sizeOfActual = 0; - Iterator, List>> iterator = actual.iterator(); + Iterator, EndpointsForRange.Mutable>> iterator = actual.iterator(); while(iterator.hasNext()) { - Map.Entry, List> actualEntry = iterator.next(); + Map.Entry, EndpointsForRange.Mutable> actualEntry = iterator.next(); assertNotNull(expected.get(actualEntry.getKey())); - assertEquals(new HashSet<>(expected.get(actualEntry.getKey())), new HashSet<>(actualEntry.getValue())); + assertEquals(ImmutableSet.copyOf(expected.get(actualEntry.getKey())), ImmutableSet.copyOf(actualEntry.getValue())); sizeOfActual++; } @@ -589,9 +598,9 @@ public class MoveTest int numMoved = 0; for (Token token : keyTokens) { - int replicationFactor = strategy.getReplicationFactor(); + int replicationFactor = strategy.getReplicationFactor().allReplicas; - HashSet actual = new HashSet<>(tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalEndpoints(token, tmd.cloneOnlyTokenMap()))); + EndpointsForToken actual = tmd.getWriteEndpoints(token, keyspaceName, strategy.calculateNaturalReplicas(token, tmd.cloneOnlyTokenMap()).forToken(token)); HashSet expected = new HashSet<>(); for (int i = 0; i < replicationFactor; i++) @@ -600,10 +609,10 @@ public class MoveTest } if (expected.size() == actual.size()) { - assertEquals("mismatched endpoint sets", expected, actual); + assertEquals("mismatched endpoint sets", expected, actual.endpoints()); } else { expected.add(hosts.get(MOVING_NODE)); - assertEquals("mismatched endpoint sets", expected, actual); + assertEquals("mismatched endpoint sets", expected, actual.endpoints()); numMoved++; } } @@ -648,8 +657,6 @@ public class MoveTest newTokens.put(movingIndex, newToken); } - Collection endpoints; - tmd = tmd.cloneAfterAllSettled(); ss.setTokenMetadataUnsafe(tmd); @@ -693,37 +700,18 @@ public class MoveTest * } */ - Multimap> keyspace1ranges = keyspaceStrategyMap.get(Simple_RF1_KeyspaceName).getAddressRanges(); - Collection> ranges1 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.1")); - assertEquals(1, collectionSize(ranges1)); - assertEquals(generateRange(97, 0), ranges1.iterator().next()); - Collection> ranges2 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.2")); - assertEquals(1, collectionSize(ranges2)); - assertEquals(generateRange(0, 10), ranges2.iterator().next()); - Collection> ranges3 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.3")); - assertEquals(1, collectionSize(ranges3)); - assertEquals(generateRange(10, 20), ranges3.iterator().next()); - Collection> ranges4 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.4")); - assertEquals(1, collectionSize(ranges4)); - assertEquals(generateRange(20, 30), ranges4.iterator().next()); - Collection> ranges5 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.5")); - assertEquals(1, collectionSize(ranges5)); - assertEquals(generateRange(30, 40), ranges5.iterator().next()); - Collection> ranges6 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.6")); - assertEquals(1, collectionSize(ranges6)); - assertEquals(generateRange(40, 50), ranges6.iterator().next()); - Collection> ranges7 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.7")); - assertEquals(1, collectionSize(ranges7)); - assertEquals(generateRange(50, 67), ranges7.iterator().next()); - Collection> ranges8 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.8")); - assertEquals(1, collectionSize(ranges8)); - assertEquals(generateRange(67, 70), ranges8.iterator().next()); - Collection> ranges9 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.9")); - assertEquals(1, collectionSize(ranges9)); - assertEquals(generateRange(70, 87), ranges9.iterator().next()); - Collection> ranges10 = keyspace1ranges.get(InetAddressAndPort.getByName("127.0.0.10")); - assertEquals(1, collectionSize(ranges10)); - assertEquals(generateRange(87, 97), ranges10.iterator().next()); + RangesByEndpoint keyspace1ranges = keyspaceStrategyMap.get(Simple_RF1_KeyspaceName).getAddressReplicas(); + + assertRanges(keyspace1ranges, "127.0.0.1", 97, 0); + assertRanges(keyspace1ranges, "127.0.0.2", 0, 10); + assertRanges(keyspace1ranges, "127.0.0.3", 10, 20); + assertRanges(keyspace1ranges, "127.0.0.4", 20, 30); + assertRanges(keyspace1ranges, "127.0.0.5", 30, 40); + assertRanges(keyspace1ranges, "127.0.0.6", 40, 50); + assertRanges(keyspace1ranges, "127.0.0.7", 50, 67); + assertRanges(keyspace1ranges, "127.0.0.8", 67, 70); + assertRanges(keyspace1ranges, "127.0.0.9", 70, 87); + assertRanges(keyspace1ranges, "127.0.0.10", 87, 97); /** @@ -742,37 +730,17 @@ public class MoveTest * } */ - Multimap> keyspace3ranges = keyspaceStrategyMap.get(KEYSPACE3).getAddressRanges(); - ranges1 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.1")); - assertEquals(collectionSize(ranges1), 5); - assertTrue(ranges1.equals(generateRanges(97, 0, 70, 87, 50, 67, 87, 97, 67, 70))); - ranges2 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.2")); - assertEquals(collectionSize(ranges2), 5); - assertTrue(ranges2.equals(generateRanges(97, 0, 70, 87, 87, 97, 0, 10, 67, 70))); - ranges3 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.3")); - assertEquals(collectionSize(ranges3), 5); - assertTrue(ranges3.equals(generateRanges(97, 0, 70, 87, 87, 97, 0, 10, 10, 20))); - ranges4 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.4")); - assertEquals(collectionSize(ranges4), 5); - assertTrue(ranges4.equals(generateRanges(97, 0, 20, 30, 87, 97, 0, 10, 10, 20))); - ranges5 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.5")); - assertEquals(collectionSize(ranges5), 5); - assertTrue(ranges5.equals(generateRanges(97, 0, 30, 40, 20, 30, 0, 10, 10, 20))); - ranges6 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.6")); - assertEquals(collectionSize(ranges6), 5); - assertTrue(ranges6.equals(generateRanges(40, 50, 30, 40, 20, 30, 0, 10, 10, 20))); - ranges7 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.7")); - assertEquals(collectionSize(ranges7), 5); - assertTrue(ranges7.equals(generateRanges(40, 50, 30, 40, 50, 67, 20, 30, 10, 20))); - ranges8 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.8")); - assertEquals(collectionSize(ranges8), 5); - assertTrue(ranges8.equals(generateRanges(40, 50, 30, 40, 50, 67, 20, 30, 67, 70))); - ranges9 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.9")); - assertEquals(collectionSize(ranges9), 5); - assertTrue(ranges9.equals(generateRanges(40, 50, 70, 87, 30, 40, 50, 67, 67, 70))); - ranges10 = keyspace3ranges.get(InetAddressAndPort.getByName("127.0.0.10")); - assertEquals(collectionSize(ranges10), 5); - assertTrue(ranges10.equals(generateRanges(40, 50, 70, 87, 50, 67, 87, 97, 67, 70))); + RangesByEndpoint keyspace3ranges = keyspaceStrategyMap.get(KEYSPACE3).getAddressReplicas(); + assertRanges(keyspace3ranges, "127.0.0.1", 97, 0, 70, 87, 50, 67, 87, 97, 67, 70); + assertRanges(keyspace3ranges, "127.0.0.2", 97, 0, 70, 87, 87, 97, 0, 10, 67, 70); + assertRanges(keyspace3ranges, "127.0.0.3", 97, 0, 70, 87, 87, 97, 0, 10, 10, 20); + assertRanges(keyspace3ranges, "127.0.0.4", 97, 0, 20, 30, 87, 97, 0, 10, 10, 20); + assertRanges(keyspace3ranges, "127.0.0.5", 97, 0, 30, 40, 20, 30, 0, 10, 10, 20); + assertRanges(keyspace3ranges, "127.0.0.6", 40, 50, 30, 40, 20, 30, 0, 10, 10, 20); + assertRanges(keyspace3ranges, "127.0.0.7", 40, 50, 30, 40, 50, 67, 20, 30, 10, 20); + assertRanges(keyspace3ranges, "127.0.0.8", 40, 50, 30, 40, 50, 67, 20, 30, 67, 70); + assertRanges(keyspace3ranges, "127.0.0.9", 40, 50, 70, 87, 30, 40, 50, 67, 67, 70); + assertRanges(keyspace3ranges, "127.0.0.10", 40, 50, 70, 87, 50, 67, 87, 97, 67, 70); /** @@ -790,37 +758,18 @@ public class MoveTest * /127.0.0.10=[(70,87], (87,97], (67,70]] * } */ - Multimap> keyspace4ranges = keyspaceStrategyMap.get(Simple_RF3_KeyspaceName).getAddressRanges(); - ranges1 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.1")); - assertEquals(collectionSize(ranges1), 3); - assertTrue(ranges1.equals(generateRanges(97, 0, 70, 87, 87, 97))); - ranges2 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.2")); - assertEquals(collectionSize(ranges2), 3); - assertTrue(ranges2.equals(generateRanges(97, 0, 87, 97, 0, 10))); - ranges3 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.3")); - assertEquals(collectionSize(ranges3), 3); - assertTrue(ranges3.equals(generateRanges(97, 0, 0, 10, 10, 20))); - ranges4 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.4")); - assertEquals(collectionSize(ranges4), 3); - assertTrue(ranges4.equals(generateRanges(20, 30, 0, 10, 10, 20))); - ranges5 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.5")); - assertEquals(collectionSize(ranges5), 3); - assertTrue(ranges5.equals(generateRanges(30, 40, 20, 30, 10, 20))); - ranges6 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.6")); - assertEquals(collectionSize(ranges6), 3); - assertTrue(ranges6.equals(generateRanges(40, 50, 30, 40, 20, 30))); - ranges7 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.7")); - assertEquals(collectionSize(ranges7), 3); - assertTrue(ranges7.equals(generateRanges(40, 50, 30, 40, 50, 67))); - ranges8 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.8")); - assertEquals(collectionSize(ranges8), 3); - assertTrue(ranges8.equals(generateRanges(40, 50, 50, 67, 67, 70))); - ranges9 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.9")); - assertEquals(collectionSize(ranges9), 3); - assertTrue(ranges9.equals(generateRanges(70, 87, 50, 67, 67, 70))); - ranges10 = keyspace4ranges.get(InetAddressAndPort.getByName("127.0.0.10")); - assertEquals(collectionSize(ranges10), 3); - assertTrue(ranges10.equals(generateRanges(70, 87, 87, 97, 67, 70))); + RangesByEndpoint keyspace4ranges = keyspaceStrategyMap.get(Simple_RF3_KeyspaceName).getAddressReplicas(); + + assertRanges(keyspace4ranges, "127.0.0.1", 97, 0, 70, 87, 87, 97); + assertRanges(keyspace4ranges, "127.0.0.2", 97, 0, 87, 97, 0, 10); + assertRanges(keyspace4ranges, "127.0.0.3", 97, 0, 0, 10, 10, 20); + assertRanges(keyspace4ranges, "127.0.0.4", 20, 30, 0, 10, 10, 20); + assertRanges(keyspace4ranges, "127.0.0.5", 30, 40, 20, 30, 10, 20); + assertRanges(keyspace4ranges, "127.0.0.6", 40, 50, 30, 40, 20, 30); + assertRanges(keyspace4ranges, "127.0.0.7", 40, 50, 30, 40, 50, 67); + assertRanges(keyspace4ranges, "127.0.0.8", 40, 50, 50, 67, 67, 70); + assertRanges(keyspace4ranges, "127.0.0.9", 70, 87, 50, 67, 67, 70); + assertRanges(keyspace4ranges, "127.0.0.10", 70, 87, 87, 97, 67, 70); // pre-calculate the results. Map> expectedEndpoints = new HashMap<>(); @@ -876,79 +825,80 @@ public class MoveTest for (Token token : keyTokens) { - endpoints = tmd.getWriteEndpoints(token, keyspaceName, strategy.getNaturalEndpoints(token)); + Collection endpoints = tmd.getWriteEndpoints(token, keyspaceName, strategy.getNaturalReplicasForToken(token)).endpoints(); assertEquals(expectedEndpoints.get(keyspaceName).get(token).size(), endpoints.size()); assertTrue(expectedEndpoints.get(keyspaceName).get(token).containsAll(endpoints)); } // just to be sure that things still work according to the old tests, run them: - if (strategy.getReplicationFactor() != 3) + if (strategy.getReplicationFactor().allReplicas != 3) continue; + ReplicaCollection replicas = null; // tokens 5, 15 and 25 should go three nodes for (int i = 0; i < 3; i++) { - endpoints = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(i))); - assertEquals(3, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(i+1))); - assertTrue(endpoints.contains(hosts.get(i+2))); - assertTrue(endpoints.contains(hosts.get(i+3))); + replicas = tmd.getWriteEndpoints(keyTokens.get(i), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(i))); + assertEquals(3, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(i + 1))); + assertTrue(replicas.endpoints().contains(hosts.get(i + 2))); + assertTrue(replicas.endpoints().contains(hosts.get(i + 3))); } // token 35 should go to nodes 4, 5, 6 and boot1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(3))); - assertEquals(4, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(4))); - assertTrue(endpoints.contains(hosts.get(5))); - assertTrue(endpoints.contains(hosts.get(6))); - assertTrue(endpoints.contains(boot1)); + replicas = tmd.getWriteEndpoints(keyTokens.get(3), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(3))); + assertEquals(4, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(4))); + assertTrue(replicas.endpoints().contains(hosts.get(5))); + assertTrue(replicas.endpoints().contains(hosts.get(6))); + assertTrue(replicas.endpoints().contains(boot1)); // token 45 should go to nodes 5, 6, 7 boot1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(4))); - assertEquals(4, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(5))); - assertTrue(endpoints.contains(hosts.get(6))); - assertTrue(endpoints.contains(hosts.get(7))); - assertTrue(endpoints.contains(boot1)); + replicas = tmd.getWriteEndpoints(keyTokens.get(4), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(4))); + assertEquals(4, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(5))); + assertTrue(replicas.endpoints().contains(hosts.get(6))); + assertTrue(replicas.endpoints().contains(hosts.get(7))); + assertTrue(replicas.endpoints().contains(boot1)); // token 55 should go to nodes 6, 7, 8 boot1 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(5))); - assertEquals(5, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(6))); - assertTrue(endpoints.contains(hosts.get(7))); - assertTrue(endpoints.contains(hosts.get(8))); - assertTrue(endpoints.contains(boot1)); - assertTrue(endpoints.contains(boot2)); + replicas = tmd.getWriteEndpoints(keyTokens.get(5), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(5))); + assertEquals(5, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(6))); + assertTrue(replicas.endpoints().contains(hosts.get(7))); + assertTrue(replicas.endpoints().contains(hosts.get(8))); + assertTrue(replicas.endpoints().contains(boot1)); + assertTrue(replicas.endpoints().contains(boot2)); // token 65 should go to nodes 6, 7, 8 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(6))); - assertEquals(4, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(6))); - assertTrue(endpoints.contains(hosts.get(7))); - assertTrue(endpoints.contains(hosts.get(8))); - assertTrue(endpoints.contains(boot2)); + replicas = tmd.getWriteEndpoints(keyTokens.get(6), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(6))); + assertEquals(4, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(6))); + assertTrue(replicas.endpoints().contains(hosts.get(7))); + assertTrue(replicas.endpoints().contains(hosts.get(8))); + assertTrue(replicas.endpoints().contains(boot2)); // token 75 should to go nodes 8, 9, 0 and boot2 - endpoints = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(7))); - assertEquals(4, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(8))); - assertTrue(endpoints.contains(hosts.get(9))); - assertTrue(endpoints.contains(hosts.get(0))); - assertTrue(endpoints.contains(boot2)); + replicas = tmd.getWriteEndpoints(keyTokens.get(7), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(7))); + assertEquals(4, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(8))); + assertTrue(replicas.endpoints().contains(hosts.get(9))); + assertTrue(replicas.endpoints().contains(hosts.get(0))); + assertTrue(replicas.endpoints().contains(boot2)); // token 85 should go to nodes 8, 9 and 0 - endpoints = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(8))); - assertEquals(3, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(8))); - assertTrue(endpoints.contains(hosts.get(9))); - assertTrue(endpoints.contains(hosts.get(0))); + replicas = tmd.getWriteEndpoints(keyTokens.get(8), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(8))); + assertEquals(3, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(8))); + assertTrue(replicas.endpoints().contains(hosts.get(9))); + assertTrue(replicas.endpoints().contains(hosts.get(0))); // token 95 should go to nodes 9, 0 and 1 - endpoints = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalEndpoints(keyTokens.get(9))); - assertEquals(3, endpoints.size()); - assertTrue(endpoints.contains(hosts.get(9))); - assertTrue(endpoints.contains(hosts.get(0))); - assertTrue(endpoints.contains(hosts.get(1))); + replicas = tmd.getWriteEndpoints(keyTokens.get(9), keyspaceName, strategy.getNaturalReplicasForToken(keyTokens.get(9))); + assertEquals(3, replicas.size()); + assertTrue(replicas.endpoints().contains(hosts.get(9))); + assertTrue(replicas.endpoints().contains(hosts.get(0))); + assertTrue(replicas.endpoints().contains(hosts.get(1))); } // all moving nodes are back to the normal state @@ -1009,6 +959,14 @@ public class MoveTest return addrs; } + private static EndpointsForRange makeReplicas(Range range, String... hosts) throws UnknownHostException + { + EndpointsForRange.Builder replicas = EndpointsForRange.builder(range, hosts.length); + for (String host : hosts) + replicas.add(Replica.fullReplica(InetAddressAndPort.getByName(host), range)); + return replicas.build(); + } + private AbstractReplicationStrategy getStrategy(String keyspaceName, TokenMetadata tmd) { KeyspaceMetadata ksmd = Schema.instance.getKeyspaceMetadata(keyspaceName); @@ -1025,7 +983,7 @@ public class MoveTest return new BigIntegerToken(String.valueOf(10 * position + 7)); } - private int collectionSize(Collection collection) + private static int collectionSize(Collection collection) { if (collection.isEmpty()) return 0; @@ -1057,8 +1015,52 @@ public class MoveTest return ranges; } - private Range generateRange(int left, int right) + private static Token tk(int v) { - return new Range(new BigIntegerToken(String.valueOf(left)), new BigIntegerToken(String.valueOf(right))); + return new BigIntegerToken(String.valueOf(v)); + } + + private static Range generateRange(int left, int right) + { + return new Range(tk(left), tk(right)); + } + + private static Replica replica(InetAddressAndPort endpoint, int left, int right, boolean full) + { + return new Replica(endpoint, tk(left), tk(right), full); + } + + private static InetAddressAndPort inet(String name) + { + try + { + return InetAddressAndPort.getByName(name); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } + + private static Replica replica(InetAddressAndPort endpoint, int left, int right) + { + return replica(endpoint, left, right, true); + } + + private static void assertRanges(RangesByEndpoint epReplicas, String endpoint, int... rangePairs) + { + if (rangePairs.length % 2 == 1) + throw new RuntimeException("assertRanges argument count should be even"); + + InetAddressAndPort ep = inet(endpoint); + List expected = new ArrayList<>(rangePairs.length/2); + for (int i=0; i downNodes = new ArrayList(); + Predicate alivePredicate = replica -> !downNodes.contains(replica.endpoint()); + + private final List sourceFilterDownNodes = new ArrayList<>(); + private final Collection> sourceFilters = Collections.singleton(replica -> !sourceFilterDownNodes.contains(replica.endpoint())); + + @After + public void clearDownNode() + { + downNodes.clear(); + sourceFilterDownNodes.clear(); + } + + @BeforeClass + public static void setupDD() + { + DatabaseDescriptor.daemonInitialization(); + } + + Token oneToken = new RandomPartitioner.BigIntegerToken("1"); + Token twoToken = new RandomPartitioner.BigIntegerToken("2"); + Token threeToken = new RandomPartitioner.BigIntegerToken("3"); + Token fourToken = new RandomPartitioner.BigIntegerToken("4"); + Token sixToken = new RandomPartitioner.BigIntegerToken("6"); + Token sevenToken = new RandomPartitioner.BigIntegerToken("7"); + Token nineToken = new RandomPartitioner.BigIntegerToken("9"); + Token elevenToken = new RandomPartitioner.BigIntegerToken("11"); + Token fourteenToken = new RandomPartitioner.BigIntegerToken("14"); + + Range aRange = new Range(oneToken, threeToken); + Range bRange = new Range(threeToken, sixToken); + Range cRange = new Range(sixToken, nineToken); + Range dRange = new Range(nineToken, elevenToken); + Range eRange = new Range(elevenToken, oneToken); + + + RangesAtEndpoint current = RangesAtEndpoint.of(new Replica(aAddress, aRange, true), + new Replica(aAddress, eRange, true), + new Replica(aAddress, dRange, false)); + + + /** + * Ring with start A 1-3 B 3-6 C 6-9 D 9-1 + * A's token moves from 3 to 4. + *

    + * Result is A gains some range + * + * @throws Exception + */ + @Test + public void testCalculateStreamAndFetchRangesMoveForward() throws Exception + { + calculateStreamAndFetchRangesMoveForward(); + } + + private Pair calculateStreamAndFetchRangesMoveForward() throws Exception + { + Range aPrimeRange = new Range<>(oneToken, fourToken); + + RangesAtEndpoint updated = RangesAtEndpoint.of( + new Replica(aAddress, aPrimeRange, true), + new Replica(aAddress, eRange, true), + new Replica(aAddress, dRange, false) + ); + + Pair result = StorageService.calculateStreamAndFetchRanges(current, updated); + assertContentsIgnoreOrder(result.left); + assertContentsIgnoreOrder(result.right, fullReplica(aAddress, threeToken, fourToken)); + return result; + } + + /** + * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1 + * A's token moves from 3 to 14 + *

    + * Result is A loses range and it must be streamed + * + * @throws Exception + */ + @Test + public void testCalculateStreamAndFetchRangesMoveBackwardBetween() throws Exception + { + calculateStreamAndFetchRangesMoveBackwardBetween(); + } + + public Pair calculateStreamAndFetchRangesMoveBackwardBetween() throws Exception + { + Range aPrimeRange = new Range<>(elevenToken, fourteenToken); + + RangesAtEndpoint updated = RangesAtEndpoint.of( + new Replica(aAddress, aPrimeRange, true), + new Replica(aAddress, dRange, true), + new Replica(aAddress, cRange, false) + ); + + + Pair result = StorageService.calculateStreamAndFetchRanges(current, updated); + assertContentsIgnoreOrder(result.left, fullReplica(aAddress, oneToken, threeToken), fullReplica(aAddress, fourteenToken, oneToken)); + assertContentsIgnoreOrder(result.right, transientReplica(aAddress, sixToken, nineToken), fullReplica(aAddress, nineToken, elevenToken)); + return result; + } + + /** + * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1 + * A's token moves from 3 to 2 + * + * Result is A loses range and it must be streamed + * @throws Exception + */ + @Test + public void testCalculateStreamAndFetchRangesMoveBackward() throws Exception + { + calculateStreamAndFetchRangesMoveBackward(); + } + + private Pair calculateStreamAndFetchRangesMoveBackward() throws Exception + { + Range aPrimeRange = new Range<>(oneToken, twoToken); + + RangesAtEndpoint updated = RangesAtEndpoint.of( + new Replica(aAddress, aPrimeRange, true), + new Replica(aAddress, eRange, true), + new Replica(aAddress, dRange, false) + ); + + Pair result = StorageService.calculateStreamAndFetchRanges(current, updated); + + //Moving backwards has no impact on any replica. We already fully replicate counter clockwise + //The transient replica does transiently replicate slightly more, but that is addressed by cleanup + assertContentsIgnoreOrder(result.left, fullReplica(aAddress, twoToken, threeToken)); + assertContentsIgnoreOrder(result.right); + + return result; + } + + /** + * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1 + * A's moves from 3 to 7 + * + * @throws Exception + */ + private Pair calculateStreamAndFetchRangesMoveForwardBetween() throws Exception + { + Range aPrimeRange = new Range<>(sixToken, sevenToken); + Range bPrimeRange = new Range<>(oneToken, sixToken); + + + RangesAtEndpoint updated = RangesAtEndpoint.of( + new Replica(aAddress, aPrimeRange, true), + new Replica(aAddress, bPrimeRange, true), + new Replica(aAddress, eRange, false) + ); + + Pair result = StorageService.calculateStreamAndFetchRanges(current, updated); + + assertContentsIgnoreOrder(result.left, fullReplica(aAddress, elevenToken, oneToken), transientReplica(aAddress, nineToken, elevenToken)); + assertContentsIgnoreOrder(result.right, fullReplica(aAddress, threeToken, sixToken), fullReplica(aAddress, sixToken, sevenToken)); + return result; + } + + /** + * Ring with start A 1-3 B 3-6 C 6-9 D 9-11 E 11-1 + * A's token moves from 3 to 7 + * + * @throws Exception + */ + @Test + public void testCalculateStreamAndFetchRangesMoveForwardBetween() throws Exception + { + calculateStreamAndFetchRangesMoveForwardBetween(); + } + + /** + * Construct the ring state for calculateStreamAndFetchRangesMoveBackwardBetween + * Where are A moves from 3 to 14 + * @return + */ + private Pair constructTMDsMoveBackwardBetween() + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(aRange.right, aAddress); + tmd.updateNormalToken(bRange.right, bAddress); + tmd.updateNormalToken(cRange.right, cAddress); + tmd.updateNormalToken(dRange.right, dAddress); + tmd.updateNormalToken(eRange.right, eAddress); + tmd.addMovingEndpoint(fourteenToken, aAddress); + TokenMetadata updated = tmd.cloneAfterAllSettled(); + + return Pair.create(tmd, updated); + } + + + /** + * Construct the ring state for calculateStreamAndFetchRangesMoveForwardBetween + * Where are A moves from 3 to 7 + * @return + */ + private Pair constructTMDsMoveForwardBetween() + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(aRange.right, aAddress); + tmd.updateNormalToken(bRange.right, bAddress); + tmd.updateNormalToken(cRange.right, cAddress); + tmd.updateNormalToken(dRange.right, dAddress); + tmd.updateNormalToken(eRange.right, eAddress); + tmd.addMovingEndpoint(sevenToken, aAddress); + TokenMetadata updated = tmd.cloneAfterAllSettled(); + + return Pair.create(tmd, updated); + } + + private Pair constructTMDsMoveBackward() + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(aRange.right, aAddress); + tmd.updateNormalToken(bRange.right, bAddress); + tmd.updateNormalToken(cRange.right, cAddress); + tmd.updateNormalToken(dRange.right, dAddress); + tmd.updateNormalToken(eRange.right, eAddress); + tmd.addMovingEndpoint(twoToken, aAddress); + TokenMetadata updated = tmd.cloneAfterAllSettled(); + + return Pair.create(tmd, updated); + } + + private Pair constructTMDsMoveForward() + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(aRange.right, aAddress); + tmd.updateNormalToken(bRange.right, bAddress); + tmd.updateNormalToken(cRange.right, cAddress); + tmd.updateNormalToken(dRange.right, dAddress); + tmd.updateNormalToken(eRange.right, eAddress); + tmd.addMovingEndpoint(fourToken, aAddress); + TokenMetadata updated = tmd.cloneAfterAllSettled(); + + return Pair.create(tmd, updated); + } + + + @Test + public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints() throws Exception + { + EndpointsByReplica.Mutable expectedResult = new EndpointsByReplica.Mutable(); + + InetAddressAndPort cOrB = (downNodes.contains(cAddress) || sourceFilterDownNodes.contains(cAddress)) ? bAddress : cAddress; + + //Need to pull the full replica and the transient replica that is losing the range + expectedResult.put(fullReplica(aAddress, sixToken, sevenToken), fullReplica(dAddress, sixToken, nineToken)); + expectedResult.put(fullReplica(aAddress, sixToken, sevenToken), transientReplica(eAddress, sixToken, nineToken)); + + //Same need both here as well + expectedResult.put(fullReplica(aAddress, threeToken, sixToken), fullReplica(cOrB, threeToken, sixToken)); + expectedResult.put(fullReplica(aAddress, threeToken, sixToken), transientReplica(dAddress, threeToken, sixToken)); + + invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForwardBetween().right, + constructTMDsMoveForwardBetween(), + expectedResult.asImmutableView()); + } + + @Test + public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception + { + for (InetAddressAndPort downNode : new InetAddressAndPort[] {dAddress, eAddress}) + { + downNodes.clear(); + downNodes.add(downNode); + boolean threw = false; + try + { + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + catch (IllegalStateException ise) + { + ise.printStackTrace(); + assertTrue(downNode.toString(), + ise.getMessage().startsWith("A node required to move the data consistently is down:") + && ise.getMessage().contains(downNode.toString())); + threw = true; + } + assertTrue("Didn't throw for " + downNode, threw); + } + + //Shouldn't throw because another full replica is available + downNodes.clear(); + downNodes.add(cAddress); + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + @Test + public void testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception + { + for (InetAddressAndPort downNode : new InetAddressAndPort[] {dAddress, eAddress}) + { + sourceFilterDownNodes.clear(); + sourceFilterDownNodes.add(downNode); + boolean threw = false; + try + { + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + catch (IllegalStateException ise) + { + ise.printStackTrace(); + assertTrue(downNode.toString(), + ise.getMessage().startsWith("Necessary replicas for strict consistency were removed by source filters:") + && ise.getMessage().contains(downNode.toString())); + threw = true; + } + assertTrue("Didn't throw for " + downNode, threw); + } + + //Shouldn't throw because another full replica is available + sourceFilterDownNodes.clear(); + sourceFilterDownNodes.add(cAddress); + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + @Test + public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints() throws Exception + { + EndpointsByReplica.Mutable expectedResult = new EndpointsByReplica.Mutable(); + + //Need to pull the full replica and the transient replica that is losing the range + expectedResult.put(fullReplica(aAddress, nineToken, elevenToken), fullReplica(eAddress, nineToken, elevenToken)); + expectedResult.put(transientReplica(aAddress, sixToken, nineToken), transientReplica(eAddress, sixToken, nineToken)); + + invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackwardBetween().right, + constructTMDsMoveBackwardBetween(), + expectedResult.asImmutableView()); + + } + + @Test(expected = IllegalStateException.class) + public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception + { + //Any replica can be the full replica so this will always fail on the transient range + downNodes.add(eAddress); + testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + @Test(expected = IllegalStateException.class) + public void testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception + { + //Any replica can be the full replica so this will always fail on the transient range + sourceFilterDownNodes.add(eAddress); + testMoveBackwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + + //There is no down node version of this test because nothing needs to be fetched + @Test + public void testMoveBackwardCalculateRangesToFetchWithPreferredEndpoints() throws Exception + { + //Moving backwards should fetch nothing and fetch ranges is emptys so this doesn't test a ton + EndpointsByReplica.Mutable expectedResult = new EndpointsByReplica.Mutable(); + + invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackward().right, + constructTMDsMoveBackward(), + expectedResult.asImmutableView()); + + } + + @Test + public void testMoveForwardCalculateRangesToFetchWithPreferredEndpoints() throws Exception + { + EndpointsByReplica.Mutable expectedResult = new EndpointsByReplica.Mutable(); + + InetAddressAndPort cOrBAddress = (downNodes.contains(cAddress) || sourceFilterDownNodes.contains(cAddress)) ? bAddress : cAddress; + + //Need to pull the full replica and the transient replica that is losing the range + expectedResult.put(fullReplica(aAddress, threeToken, fourToken), fullReplica(cOrBAddress, threeToken, sixToken)); + expectedResult.put(fullReplica(aAddress, threeToken, fourToken), transientReplica(dAddress, threeToken, sixToken)); + + invokeCalculateRangesToFetchWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForward().right, + constructTMDsMoveForward(), + expectedResult.asImmutableView()); + + } + + @Test + public void testMoveForwardCalculateRangesToFetchWithPreferredEndpointsDownNodes() throws Exception + { + downNodes.add(dAddress); + boolean threw = false; + try + { + testMoveForwardCalculateRangesToFetchWithPreferredEndpoints(); + } + catch (IllegalStateException ise) + { + ise.printStackTrace(); + assertTrue(dAddress.toString(), + ise.getMessage().startsWith("A node required to move the data consistently is down:") + && ise.getMessage().contains(dAddress.toString())); + threw = true; + } + assertTrue("Didn't throw for " + dAddress, threw); + + //Shouldn't throw because another full replica is available + downNodes.clear(); + downNodes.add(cAddress); + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + @Test + public void testMoveForwardCalculateRangesToFetchWithPreferredEndpointsDownNodesSourceFilter() throws Exception + { + sourceFilterDownNodes.add(dAddress); + boolean threw = false; + try + { + testMoveForwardCalculateRangesToFetchWithPreferredEndpoints(); + } + catch (IllegalStateException ise) + { + ise.printStackTrace(); + assertTrue(dAddress.toString(), + ise.getMessage().startsWith("Necessary replicas for strict consistency were removed by source filters:") + && ise.getMessage().contains(dAddress.toString())); + threw = true; + } + assertTrue("Didn't throw for " + dAddress, threw); + + //Shouldn't throw because another full replica is available + sourceFilterDownNodes.clear(); + sourceFilterDownNodes.add(cAddress); + testMoveForwardBetweenCalculateRangesToFetchWithPreferredEndpoints(); + } + + private void invokeCalculateRangesToFetchWithPreferredEndpoints(RangesAtEndpoint toFetch, + Pair tmds, + EndpointsByReplica expectedResult) + { + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + + EndpointsByReplica result = RangeStreamer.calculateRangesToFetchWithPreferredEndpoints((address, replicas) -> replicas.sorted((a, b) -> b.endpoint().compareTo(a.endpoint())), + simpleStrategy(tmds.left), + toFetch, + true, + tmds.left, + tmds.right, + alivePredicate, + "OldNetworkTopologyStrategyTest", + sourceFilters); + logger.info("Ranges to fetch with preferred endpoints"); + logger.info(result.toString()); + assertMultimapEqualsIgnoreOrder(expectedResult, result); + + } + + private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd) + { + IEndpointSnitch snitch = new AbstractEndpointSnitch() + { + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) + { + return 0; + } + + public String getRack(InetAddressAndPort endpoint) + { + return "R1"; + } + + public String getDatacenter(InetAddressAndPort endpoint) + { + return "DC1"; + } + }; + + return new SimpleStrategy("MoveTransientTest", + tmd, + snitch, + com.google.common.collect.ImmutableMap.of("replication_factor", "3/1")); + } + + @Test + public void testMoveForwardBetweenCalculateRangesToStreamWithPreferredEndpoints() throws Exception + { + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + RangesByEndpoint.Mutable expectedResult = new RangesByEndpoint.Mutable(); + + //Need to pull the full replica and the transient replica that is losing the range + expectedResult.put(bAddress, transientReplica(bAddress, nineToken, elevenToken)); + expectedResult.put(bAddress, fullReplica(bAddress, elevenToken, oneToken)); + + invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForwardBetween().left, + constructTMDsMoveForwardBetween(), + expectedResult.asImmutableView()); + } + + @Test + public void testMoveBackwardBetweenCalculateRangesToStreamWithPreferredEndpoints() throws Exception + { + RangesByEndpoint.Mutable expectedResult = new RangesByEndpoint.Mutable(); + + expectedResult.put(bAddress, fullReplica(bAddress, fourteenToken, oneToken)); + + expectedResult.put(dAddress, transientReplica(dAddress, oneToken, threeToken)); + + expectedResult.put(cAddress, fullReplica(cAddress, oneToken, threeToken)); + expectedResult.put(cAddress, transientReplica(cAddress, fourteenToken, oneToken)); + + invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackwardBetween().left, + constructTMDsMoveBackwardBetween(), + expectedResult.asImmutableView()); + } + + @Test + public void testMoveBackwardCalculateRangesToStreamWithPreferredEndpoints() throws Exception + { + RangesByEndpoint.Mutable expectedResult = new RangesByEndpoint.Mutable(); + expectedResult.put(cAddress, fullReplica(cAddress, twoToken, threeToken)); + expectedResult.put(dAddress, transientReplica(dAddress, twoToken, threeToken)); + + invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveBackward().left, + constructTMDsMoveBackward(), + expectedResult.asImmutableView()); + } + + @Test + public void testMoveForwardCalculateRangesToStreamWithPreferredEndpoints() throws Exception + { + //Nothing to stream moving forward because we are acquiring more range not losing range + RangesByEndpoint.Mutable expectedResult = new RangesByEndpoint.Mutable(); + + invokeCalculateRangesToStreamWithPreferredEndpoints(calculateStreamAndFetchRangesMoveForward().left, + constructTMDsMoveForward(), + expectedResult.asImmutableView()); + } + + private void invokeCalculateRangesToStreamWithPreferredEndpoints(RangesAtEndpoint toStream, + Pair tmds, + RangesByEndpoint expectedResult) + { + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + StorageService.RangeRelocator relocator = new StorageService.RangeRelocator(); + RangesByEndpoint result = relocator.calculateRangesToStreamWithEndpoints(toStream, + simpleStrategy(tmds.left), + tmds.left, + tmds.right); + logger.info("Ranges to stream by endpoint"); + logger.info(result.toString()); + assertMultimapEqualsIgnoreOrder(expectedResult, result); + } + + private static void assertContentsIgnoreOrder(RangesAtEndpoint ranges, Replica ... replicas) + { + assertEquals(ranges.size(), replicas.length); + for (Replica replica : replicas) + if (!ranges.contains(replica)) + assertEquals(RangesAtEndpoint.of(replicas), ranges); + } + +} diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java new file mode 100644 index 0000000000..9d5c324c92 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import org.apache.cassandra.locator.EndpointsByReplica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.AbstractEndpointSnitch; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaMultimap; +import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.locator.TokenMetadata; + +import static org.junit.Assert.assertEquals; + +public class StorageServiceTest +{ + static InetAddressAndPort aAddress; + static InetAddressAndPort bAddress; + static InetAddressAndPort cAddress; + static InetAddressAndPort dAddress; + static InetAddressAndPort eAddress; + + @BeforeClass + public static void setUpClass() throws Exception + { + aAddress = InetAddressAndPort.getByName("127.0.0.1"); + bAddress = InetAddressAndPort.getByName("127.0.0.2"); + cAddress = InetAddressAndPort.getByName("127.0.0.3"); + dAddress = InetAddressAndPort.getByName("127.0.0.4"); + eAddress = InetAddressAndPort.getByName("127.0.0.5"); + } + + private static final Token threeToken = new RandomPartitioner.BigIntegerToken("3"); + private static final Token sixToken = new RandomPartitioner.BigIntegerToken("6"); + private static final Token nineToken = new RandomPartitioner.BigIntegerToken("9"); + private static final Token elevenToken = new RandomPartitioner.BigIntegerToken("11"); + private static final Token oneToken = new RandomPartitioner.BigIntegerToken("1"); + + Range aRange = new Range<>(oneToken, threeToken); + Range bRange = new Range<>(threeToken, sixToken); + Range cRange = new Range<>(sixToken, nineToken); + Range dRange = new Range<>(nineToken, elevenToken); + Range eRange = new Range<>(elevenToken, oneToken); + + @Before + public void setUp() + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + IEndpointSnitch snitch = new AbstractEndpointSnitch() + { + public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2) + { + return 0; + } + + public String getRack(InetAddressAndPort endpoint) + { + return "R1"; + } + + public String getDatacenter(InetAddressAndPort endpoint) + { + return "DC1"; + } + }; + + DatabaseDescriptor.setEndpointSnitch(snitch); + } + + private AbstractReplicationStrategy simpleStrategy(TokenMetadata tmd) + { + return new SimpleStrategy("MoveTransientTest", + tmd, + DatabaseDescriptor.getEndpointSnitch(), + com.google.common.collect.ImmutableMap.of("replication_factor", "3/1")); + } + + public static > void assertMultimapEqualsIgnoreOrder(ReplicaMultimap a, ReplicaMultimap b) + { + if (!a.keySet().equals(b.keySet())) + assertEquals(a, b); + for (K key : a.keySet()) + { + C ac = a.get(key); + C bc = b.get(key); + if (ac.size() != bc.size()) + assertEquals(a, b); + for (Replica r : ac) + { + if (!bc.contains(r)) + assertEquals(a, b); + } + } + } + + @Test + public void testGetChangedReplicasForLeaving() throws Exception + { + TokenMetadata tmd = new TokenMetadata(); + tmd.updateNormalToken(threeToken, aAddress); + tmd.updateNormalToken(sixToken, bAddress); + tmd.updateNormalToken(nineToken, cAddress); + tmd.updateNormalToken(elevenToken, dAddress); + tmd.updateNormalToken(oneToken, eAddress); + + tmd.addLeavingEndpoint(aAddress); + + AbstractReplicationStrategy strat = simpleStrategy(tmd); + + EndpointsByReplica result = StorageService.getChangedReplicasForLeaving("StorageServiceTest", aAddress, tmd, strat); + System.out.println(result); + EndpointsByReplica.Mutable expectedResult = new EndpointsByReplica.Mutable(); + expectedResult.put(new Replica(aAddress, aRange, true), new Replica(cAddress, new Range<>(oneToken, sixToken), true)); + expectedResult.put(new Replica(aAddress, aRange, true), new Replica(dAddress, new Range<>(oneToken, sixToken), false)); + expectedResult.put(new Replica(aAddress, eRange, true), new Replica(bAddress, eRange, true)); + expectedResult.put(new Replica(aAddress, eRange, true), new Replica(cAddress, eRange, false)); + expectedResult.put(new Replica(aAddress, dRange, false), new Replica(bAddress, dRange, false)); + assertMultimapEqualsIgnoreOrder(result, expectedResult.asImmutableView()); + } +} diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java index f8567e8210..cf1e06ae75 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java @@ -20,12 +20,14 @@ package org.apache.cassandra.service; import java.net.InetAddress; -import java.util.Collection; -import java.util.List; +import java.net.UnknownHostException; import java.util.UUID; import java.util.concurrent.TimeUnit; -import com.google.common.collect.ImmutableList; +import com.google.common.base.Predicates; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaLayout; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -38,9 +40,13 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.WriteType; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -49,12 +55,26 @@ public class WriteResponseHandlerTest { static Keyspace ks; static ColumnFamilyStore cfs; - static List targets; + static EndpointsForToken targets; + static EndpointsForToken pending; + + private static Replica full(String name) + { + try + { + return ReplicaUtils.full(InetAddressAndPort.getByName(name)); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } @BeforeClass public static void setUpClass() throws Throwable { SchemaLoader.loadSchema(); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); // Register peers with expected DC for NetworkTopologyStrategy. TokenMetadata metadata = StorageService.instance.getTokenMetadata(); metadata.clearUnsafe(); @@ -77,17 +97,12 @@ public class WriteResponseHandlerTest return "datacenter2"; } - public List getSortedListByProximity(InetAddressAndPort address, Collection unsortedAddress) + public > C sortedByProximity(InetAddressAndPort address, C replicas) { - return null; + return replicas; } - public void sortByProximity(InetAddressAndPort address, List addresses) - { - - } - - public int compareEndpoints(InetAddressAndPort target, InetAddressAndPort a1, InetAddressAndPort a2) + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) { return 0; } @@ -97,7 +112,7 @@ public class WriteResponseHandlerTest } - public boolean isWorthMergingForRangeQuery(List merged, List l1, List l2) + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) { return false; } @@ -106,8 +121,10 @@ public class WriteResponseHandlerTest SchemaLoader.createKeyspace("Foo", KeyspaceParams.nts("datacenter1", 3, "datacenter2", 3), SchemaLoader.standardCFMD("Foo", "Bar")); ks = Keyspace.open("Foo"); cfs = ks.getColumnFamilyStore("Bar"); - targets = ImmutableList.of(InetAddressAndPort.getByName("127.1.0.255"), InetAddressAndPort.getByName("127.1.0.254"), InetAddressAndPort.getByName("127.1.0.253"), - InetAddressAndPort.getByName("127.2.0.255"), InetAddressAndPort.getByName("127.2.0.254"), InetAddressAndPort.getByName("127.2.0.253")); + targets = EndpointsForToken.of(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)), + full("127.1.0.255"), full("127.1.0.254"), full("127.1.0.253"), + full("127.2.0.255"), full("127.2.0.254"), full("127.2.0.253")); + pending = EndpointsForToken.empty(DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0))); } @Before @@ -197,7 +214,6 @@ public class WriteResponseHandlerTest @Test public void failedIdealCLIncrementsStat() throws Throwable { - AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM); //Succeed in local DC @@ -220,16 +236,12 @@ public class WriteResponseHandlerTest private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal, long queryStartTime) { - return ks.getReplicationStrategy().getWriteResponseHandler(targets, ImmutableList.of(), cl, new Runnable() { - public void run() - { - - } - }, WriteType.SIMPLE, queryStartTime, ideal); + return ks.getReplicationStrategy().getWriteResponseHandler(ReplicaLayout.forWriteWithDownNodes(ks, cl, targets.token(), targets, pending), + null, WriteType.SIMPLE, queryStartTime, ideal); } private static MessageIn createDummyMessage(int target) { - return MessageIn.create(targets.get(target), null, null, null, 0, 0L); + return MessageIn.create(targets.get(target).endpoint(), null, null, null, 0, 0L); } } diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java new file mode 100644 index 0000000000..c19e65e756 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTransientTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; + +import com.google.common.base.Predicates; +import com.google.common.collect.Sets; + +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.ReplicaUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; +import static org.apache.cassandra.locator.ReplicaUtils.FULL_RANGE; +import static org.apache.cassandra.locator.ReplicaUtils.full; +import static org.apache.cassandra.locator.ReplicaUtils.trans; + +public class WriteResponseHandlerTransientTest +{ + static Keyspace ks; + static ColumnFamilyStore cfs; + + static final InetAddressAndPort EP1; + static final InetAddressAndPort EP2; + static final InetAddressAndPort EP3; + static final InetAddressAndPort EP4; + static final InetAddressAndPort EP5; + static final InetAddressAndPort EP6; + + static final String DC1 = "datacenter1"; + static final String DC2 = "datacenter2"; + static Token dummy; + static + { + try + { + EP1 = InetAddressAndPort.getByName("127.1.0.1"); + EP2 = InetAddressAndPort.getByName("127.1.0.2"); + EP3 = InetAddressAndPort.getByName("127.1.0.3"); + EP4 = InetAddressAndPort.getByName("127.2.0.4"); + EP5 = InetAddressAndPort.getByName("127.2.0.5"); + EP6 = InetAddressAndPort.getByName("127.2.0.6"); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } + + @BeforeClass + public static void setupClass() throws Throwable + { + SchemaLoader.loadSchema(); + DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + + // Register peers with expected DC for NetworkTopologyStrategy. + TokenMetadata metadata = StorageService.instance.getTokenMetadata(); + metadata.clearUnsafe(); + metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.1.0.1")); + metadata.updateHostId(UUID.randomUUID(), InetAddressAndPort.getByName("127.2.0.1")); + + DatabaseDescriptor.setEndpointSnitch(new IEndpointSnitch() + { + public String getRack(InetAddressAndPort endpoint) + { + return null; + } + + public String getDatacenter(InetAddressAndPort endpoint) + { + byte[] address = endpoint.address.getAddress(); + if (address[1] == 1) + return DC1; + else + return DC2; + } + + public > C sortedByProximity(InetAddressAndPort address, C unsortedAddress) + { + return unsortedAddress; + } + + public int compareEndpoints(InetAddressAndPort target, Replica a1, Replica a2) + { + return 0; + } + + public void gossiperStarting() + { + + } + + public boolean isWorthMergingForRangeQuery(ReplicaCollection merged, ReplicaCollection l1, ReplicaCollection l2) + { + return false; + } + }); + + DatabaseDescriptor.setBroadcastAddress(InetAddress.getByName("127.1.0.1")); + SchemaLoader.createKeyspace("ks", KeyspaceParams.nts(DC1, "3/1", DC2, "3/1"), SchemaLoader.standardCFMD("ks", "tbl")); + ks = Keyspace.open("ks"); + cfs = ks.getColumnFamilyStore("tbl"); + dummy = DatabaseDescriptor.getPartitioner().getToken(ByteBufferUtil.bytes(0)); + } + + @Ignore("Throws unavailable for quorum as written") + @Test + public void checkPendingReplicasAreNotFiltered() + { + EndpointsForToken natural = EndpointsForToken.of(dummy.getToken(), full(EP1), full(EP2), trans(EP3)); + EndpointsForToken pending = EndpointsForToken.of(dummy.getToken(), full(EP4), full(EP5), trans(EP6)); + ReplicaLayout.ForToken replicaLayout = ReplicaLayout.forWrite(ks, ConsistencyLevel.QUORUM, dummy.getToken(), 2, natural, pending, Predicates.alwaysTrue()); + + Assert.assertEquals(EndpointsForRange.of(full(EP4), full(EP5), trans(EP6)), replicaLayout.pending()); + } + + private static ReplicaLayout.ForToken expected(EndpointsForToken all, EndpointsForToken selected) + { + return new ReplicaLayout.ForToken(ks, ConsistencyLevel.QUORUM, dummy.getToken(), all, EndpointsForToken.empty(dummy.getToken()), selected); + } + + private static ReplicaLayout.ForToken getSpeculationContext(EndpointsForToken replicas, int blockFor, Predicate livePredicate) + { + return ReplicaLayout.forWrite(ks, ConsistencyLevel.QUORUM, dummy.getToken(), blockFor, replicas, EndpointsForToken.empty(dummy.getToken()), livePredicate); + } + + private static void assertSpeculationReplicas(ReplicaLayout.ForToken expected, EndpointsForToken replicas, int blockFor, Predicate livePredicate) + { + ReplicaLayout.ForToken actual = getSpeculationContext(replicas, blockFor, livePredicate); + Assert.assertEquals(expected.natural(), actual.natural()); + Assert.assertEquals(expected.selected(), actual.selected()); + } + + private static Predicate dead(InetAddressAndPort... endpoints) + { + Set deadSet = Sets.newHashSet(endpoints); + return ep -> !deadSet.contains(ep); + } + + private static EndpointsForToken replicas(Replica... rr) + { + return EndpointsForToken.of(dummy.getToken(), rr); + } + + @Ignore("Throws unavailable for quorum as written") + @Test + public void checkSpeculationContext() + { + EndpointsForToken all = replicas(full(EP1), full(EP2), trans(EP3)); + // in happy path, transient replica should be classified as a backup + assertSpeculationReplicas(expected(all, + replicas(full(EP1), full(EP2))), + replicas(full(EP1), full(EP2), trans(EP3)), + 2, dead()); + + // if one of the full replicas is dead, they should all be in the initial contacts + assertSpeculationReplicas(expected(all, + replicas(full(EP1), trans(EP3))), + replicas(full(EP1), full(EP2), trans(EP3)), + 2, dead(EP2)); + + // block only for 1 full replica, use transient as backups + assertSpeculationReplicas(expected(all, + replicas(full(EP1))), + replicas(full(EP1), full(EP2), trans(EP3)), + 1, dead(EP2)); + } + + @Test (expected = UnavailableException.class) + public void noFullReplicas() + { + getSpeculationContext(replicas(full(EP1), trans(EP2), trans(EP3)), 2, dead(EP1)); + } + + @Test (expected = UnavailableException.class) + public void notEnoughTransientReplicas() + { + getSpeculationContext(replicas(full(EP1), trans(EP2), trans(EP3)), 2, dead(EP2, EP3)); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java new file mode 100644 index 0000000000..c6f2232769 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/reads/AbstractReadResponseTest.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads; + +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringBoundary; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadResponse; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.ByteType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessageIn; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +/** + * Base class for testing various components which deal with read responses + */ +@Ignore +public abstract class AbstractReadResponseTest +{ + public static final String KEYSPACE1 = "DataResolverTest"; + public static final String CF_STANDARD = "Standard1"; + public static final String CF_COLLECTION = "Collection1"; + + public static Keyspace ks; + public static ColumnFamilyStore cfs; + public static ColumnFamilyStore cfs2; + public static TableMetadata cfm; + public static TableMetadata cfm2; + public static ColumnMetadata m; + + public static DecoratedKey dk; + static int nowInSec; + + static final InetAddressAndPort EP1; + static final InetAddressAndPort EP2; + static final InetAddressAndPort EP3; + + static + { + try + { + EP1 = InetAddressAndPort.getByName("127.0.0.1"); + EP2 = InetAddressAndPort.getByName("127.0.0.2"); + EP3 = InetAddressAndPort.getByName("127.0.0.3"); + } + catch (UnknownHostException e) + { + throw new RuntimeException(e); + } + } + + @BeforeClass + public static void setupClass() throws Throwable + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + + TableMetadata.Builder builder1 = + TableMetadata.builder(KEYSPACE1, CF_STANDARD) + .addPartitionKeyColumn("key", BytesType.instance) + .addClusteringColumn("col1", AsciiType.instance) + .addRegularColumn("c1", AsciiType.instance) + .addRegularColumn("c2", AsciiType.instance) + .addRegularColumn("one", AsciiType.instance) + .addRegularColumn("two", AsciiType.instance); + + TableMetadata.Builder builder2 = + TableMetadata.builder(KEYSPACE1, CF_COLLECTION) + .addPartitionKeyColumn("k", ByteType.instance) + .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true)); + + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), builder1, builder2); + + ks = Keyspace.open(KEYSPACE1); + cfs = ks.getColumnFamilyStore(CF_STANDARD); + cfm = cfs.metadata(); + cfs2 = ks.getColumnFamilyStore(CF_COLLECTION); + cfm2 = cfs2.metadata(); + m = cfm2.getColumn(new ColumnIdentifier("m", false)); + } + + @Before + public void setUp() throws Exception + { + dk = Util.dk("key1"); + nowInSec = FBUtilities.nowInSeconds(); + } + + static void assertPartitionsEqual(RowIterator l, RowIterator r) + { + try (RowIterator left = l; RowIterator right = r) + { + Assert.assertTrue(Util.sameContent(left, right)); + } + } + + static void assertPartitionsEqual(UnfilteredRowIterator left, UnfilteredRowIterator right) + { + Assert.assertTrue(Util.sameContent(left, right)); + } + + static void assertPartitionsEqual(UnfilteredPartitionIterator left, UnfilteredPartitionIterator right) + { + while (left.hasNext()) + { + Assert.assertTrue(right.hasNext()); + assertPartitionsEqual(left.next(), right.next()); + } + Assert.assertFalse(right.hasNext()); + } + + static void assertPartitionsEqual(PartitionIterator l, PartitionIterator r) + { + try (PartitionIterator left = l; PartitionIterator right = r) + { + while (left.hasNext()) + { + Assert.assertTrue(right.hasNext()); + assertPartitionsEqual(left.next(), right.next()); + } + Assert.assertFalse(right.hasNext()); + } + } + + static void consume(PartitionIterator i) + { + try (PartitionIterator iterator = i) + { + while (iterator.hasNext()) + { + try (RowIterator rows = iterator.next()) + { + while (rows.hasNext()) + rows.next(); + } + } + } + } + + static PartitionIterator filter(UnfilteredPartitionIterator iter) + { + return UnfilteredPartitionIterators.filter(iter, nowInSec); + } + + static DecoratedKey dk(String k) + { + return cfs.decorateKey(ByteBufferUtil.bytes(k)); + } + + static DecoratedKey dk(int k) + { + return dk(Integer.toString(k)); + } + + static MessageIn response(ReadCommand command, InetAddressAndPort from, UnfilteredPartitionIterator data, boolean digest) + { + ReadResponse response = digest ? ReadResponse.createDigestResponse(data, command) : ReadResponse.createDataResponse(data, command); + return MessageIn.create(from, response, Collections.emptyMap(), MessagingService.Verb.READ, MessagingService.current_version); + } + + static MessageIn response(ReadCommand command, InetAddressAndPort from, UnfilteredPartitionIterator data) + { + return response(command, from, data, false); + } + + public RangeTombstone tombstone(Object start, Object end, long markedForDeleteAt, int localDeletionTime) + { + return tombstone(start, true, end, true, markedForDeleteAt, localDeletionTime); + } + + public RangeTombstone tombstone(Object start, boolean inclusiveStart, Object end, boolean inclusiveEnd, long markedForDeleteAt, int localDeletionTime) + { + ClusteringBound startBound = rtBound(start, true, inclusiveStart); + ClusteringBound endBound = rtBound(end, false, inclusiveEnd); + return new RangeTombstone(Slice.make(startBound, endBound), new DeletionTime(markedForDeleteAt, localDeletionTime)); + } + + public ClusteringBound rtBound(Object value, boolean isStart, boolean inclusive) + { + ClusteringBound.Kind kind = isStart + ? (inclusive ? ClusteringPrefix.Kind.INCL_START_BOUND : ClusteringPrefix.Kind.EXCL_START_BOUND) + : (inclusive ? ClusteringPrefix.Kind.INCL_END_BOUND : ClusteringPrefix.Kind.EXCL_END_BOUND); + + return ClusteringBound.create(kind, cfm.comparator.make(value).getRawValues()); + } + + public ClusteringBoundary rtBoundary(Object value, boolean inclusiveOnEnd) + { + ClusteringBound.Kind kind = inclusiveOnEnd + ? ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY + : ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY; + return ClusteringBoundary.create(kind, cfm.comparator.make(value).getRawValues()); + } + + public RangeTombstoneBoundMarker marker(Object value, boolean isStart, boolean inclusive, long markedForDeleteAt, int localDeletionTime) + { + return new RangeTombstoneBoundMarker(rtBound(value, isStart, inclusive), new DeletionTime(markedForDeleteAt, localDeletionTime)); + } + + public RangeTombstoneBoundaryMarker boundary(Object value, boolean inclusiveOnEnd, long markedForDeleteAt1, int localDeletionTime1, long markedForDeleteAt2, int localDeletionTime2) + { + return new RangeTombstoneBoundaryMarker(rtBoundary(value, inclusiveOnEnd), + new DeletionTime(markedForDeleteAt1, localDeletionTime1), + new DeletionTime(markedForDeleteAt2, localDeletionTime2)); + } + + public UnfilteredPartitionIterator fullPartitionDelete(TableMetadata table, DecoratedKey dk, long timestamp, int nowInSec) + { + return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(table, dk, timestamp, nowInSec).unfilteredIterator()); + } + + public UnfilteredPartitionIterator iter(PartitionUpdate update) + { + return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator()); + } + + public UnfilteredPartitionIterator iter(DecoratedKey key, Unfiltered... unfiltereds) + { + SortedSet s = new TreeSet<>(cfm.comparator); + Collections.addAll(s, unfiltereds); + final Iterator iterator = s.iterator(); + + UnfilteredRowIterator rowIter = new AbstractUnfilteredRowIterator(cfm, + key, + DeletionTime.LIVE, + cfm.regularAndStaticColumns(), + Rows.EMPTY_STATIC_ROW, + false, + EncodingStats.NO_STATS) + { + protected Unfiltered computeNext() + { + return iterator.hasNext() ? iterator.next() : endOfData(); + } + }; + return new SingletonUnfilteredPartitionIterator(rowIter); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java index ac8ed0b9d5..abec25d8a4 100644 --- a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java +++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java @@ -19,114 +19,101 @@ package org.apache.cassandra.service.reads; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Collections; +import java.util.Iterator; import com.google.common.collect.Iterators; import com.google.common.collect.Sets; -import org.junit.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; -import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.MutableDeletionInfo; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.CellPath; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.ByteType; -import org.apache.cassandra.db.marshal.IntegerType; -import org.apache.cassandra.db.marshal.MapType; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.net.*; -import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.service.reads.repair.TestableReadRepair; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.Util.assertClustering; import static org.apache.cassandra.Util.assertColumn; import static org.apache.cassandra.Util.assertColumns; +import static org.apache.cassandra.db.ClusteringBound.Kind; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.apache.cassandra.db.ClusteringBound.Kind; -public class DataResolverTest +public class DataResolverTest extends AbstractReadResponseTest { public static final String KEYSPACE1 = "DataResolverTest"; public static final String CF_STANDARD = "Standard1"; - public static final String CF_COLLECTION = "Collection1"; - // counter to generate the last byte of the respondent's address in a ReadResponse message - private int addressSuffix = 10; - - private DecoratedKey dk; - private Keyspace ks; - private ColumnFamilyStore cfs; - private ColumnFamilyStore cfs2; - private TableMetadata cfm; - private TableMetadata cfm2; - private ColumnMetadata m; - private int nowInSec; private ReadCommand command; private TestableReadRepair readRepair; - @BeforeClass - public static void defineSchema() throws ConfigurationException - { - DatabaseDescriptor.daemonInitialization(); - - TableMetadata.Builder builder1 = - TableMetadata.builder(KEYSPACE1, CF_STANDARD) - .addPartitionKeyColumn("key", BytesType.instance) - .addClusteringColumn("col1", AsciiType.instance) - .addRegularColumn("c1", AsciiType.instance) - .addRegularColumn("c2", AsciiType.instance) - .addRegularColumn("one", AsciiType.instance) - .addRegularColumn("two", AsciiType.instance); - - TableMetadata.Builder builder2 = - TableMetadata.builder(KEYSPACE1, CF_COLLECTION) - .addPartitionKeyColumn("k", ByteType.instance) - .addRegularColumn("m", MapType.getInstance(IntegerType.instance, IntegerType.instance, true)); - - SchemaLoader.prepareServer(); - SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), builder1, builder2); - } - @Before public void setup() { - dk = Util.dk("key1"); - ks = Keyspace.open(KEYSPACE1); - cfs = ks.getColumnFamilyStore(CF_STANDARD); - cfm = cfs.metadata(); - cfs2 = ks.getColumnFamilyStore(CF_COLLECTION); - cfm2 = cfs2.metadata(); - m = cfm2.getColumn(new ColumnIdentifier("m", false)); - - nowInSec = FBUtilities.nowInSeconds(); command = Util.cmd(cfs, dk).withNowInSeconds(nowInSec).build(); readRepair = new TestableReadRepair(command, ConsistencyLevel.QUORUM); } - @Test - public void testResolveNewerSingleRow() throws UnknownHostException + private static EndpointsForRange makeReplicas(int num) { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") - .add("c1", "v1") - .buildUpdate()))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") - .add("c1", "v2") - .buildUpdate()))); + EndpointsForRange.Builder replicas = EndpointsForRange.builder(ReplicaUtils.FULL_RANGE, num); + for (int i = 0; i < num; i++) + { + try + { + replicas.add(ReplicaUtils.full(InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, (byte) (i + 1) }))); + } + catch (UnknownHostException e) + { + throw new AssertionError(e); + } + } + return replicas.build(); + } + + @Test + public void testResolveNewerSingleRow() + { + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()), false)); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c1", "v2") + .buildUpdate()), false)); try(PartitionIterator data = resolver.resolve()) { @@ -149,16 +136,17 @@ public class DataResolverTest @Test public void testResolveDisjointSingleRow() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") - .add("c1", "v1") - .buildUpdate()))); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") - .add("c2", "v2") - .buildUpdate()))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c2", "v2") + .buildUpdate()))); try(PartitionIterator data = resolver.resolve()) { @@ -185,15 +173,16 @@ public class DataResolverTest @Test public void testResolveDisjointMultipleRows() throws UnknownHostException { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") - .add("c1", "v1") - .buildUpdate()))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("2") - .add("c2", "v2") - .buildUpdate()))); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("c1", "v1") + .buildUpdate()))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("2") + .add("c2", "v2") + .buildUpdate()))); try (PartitionIterator data = resolver.resolve()) { @@ -231,37 +220,38 @@ public class DataResolverTest @Test public void testResolveDisjointMultipleRowsWithRangeTombstones() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4, System.nanoTime(), readRepair); + EndpointsForRange replicas = makeReplicas(4); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); RangeTombstone tombstone1 = tombstone("1", "11", 1, nowInSec); RangeTombstone tombstone2 = tombstone("3", "31", 1, nowInSec); PartitionUpdate update = new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1) - .addRangeTombstone(tombstone2) - .buildUpdate(); + .addRangeTombstone(tombstone2) + .buildUpdate(); - InetAddressAndPort peer1 = peer(); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); UnfilteredPartitionIterator iter1 = iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).addRangeTombstone(tombstone1) - .addRangeTombstone(tombstone2) - .buildUpdate()); - resolver.preprocess(readResponseMessage(peer1, iter1)); + .addRangeTombstone(tombstone2) + .buildUpdate()); + resolver.preprocess(response(command, peer1, iter1)); // not covered by any range tombstone - InetAddressAndPort peer2 = peer(); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); UnfilteredPartitionIterator iter2 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("0") - .add("c1", "v0") - .buildUpdate()); - resolver.preprocess(readResponseMessage(peer2, iter2)); + .add("c1", "v0") + .buildUpdate()); + resolver.preprocess(response(command, peer2, iter2)); // covered by a range tombstone - InetAddressAndPort peer3 = peer(); + InetAddressAndPort peer3 = replicas.get(2).endpoint(); UnfilteredPartitionIterator iter3 = iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("10") - .add("c2", "v1") - .buildUpdate()); - resolver.preprocess(readResponseMessage(peer3, iter3)); + .add("c2", "v1") + .buildUpdate()); + resolver.preprocess(response(command, peer3, iter3)); // range covered by rt, but newer - InetAddressAndPort peer4 = peer(); + InetAddressAndPort peer4 = replicas.get(3).endpoint(); UnfilteredPartitionIterator iter4 = iter(new RowUpdateBuilder(cfm, nowInSec, 2L, dk).clustering("3") - .add("one", "A") - .buildUpdate()); - resolver.preprocess(readResponseMessage(peer4, iter4)); + .add("one", "A") + .buildUpdate()); + resolver.preprocess(response(command, peer4, iter4)); try (PartitionIterator data = resolver.resolve()) { try (RowIterator rows = data.next()) @@ -311,13 +301,14 @@ public class DataResolverTest @Test public void testResolveWithOneEmpty() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") - .add("c2", "v2") - .buildUpdate()))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, EmptyIterators.unfilteredPartition(cfm))); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("c2", "v2") + .buildUpdate()))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, EmptyIterators.unfilteredPartition(cfm))); try(PartitionIterator data = resolver.resolve()) { @@ -340,10 +331,11 @@ public class DataResolverTest @Test public void testResolveWithBothEmpty() { + EndpointsForRange replicas = makeReplicas(2); TestableReadRepair readRepair = new TestableReadRepair(command, ConsistencyLevel.QUORUM); - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - resolver.preprocess(readResponseMessage(peer(), EmptyIterators.unfilteredPartition(cfm))); - resolver.preprocess(readResponseMessage(peer(), EmptyIterators.unfilteredPartition(cfm))); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + resolver.preprocess(response(command, replicas.get(0).endpoint(), EmptyIterators.unfilteredPartition(cfm))); + resolver.preprocess(response(command, replicas.get(1).endpoint(), EmptyIterators.unfilteredPartition(cfm))); try(PartitionIterator data = resolver.resolve()) { @@ -356,14 +348,15 @@ public class DataResolverTest @Test public void testResolveDeleted() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); // one response with columns timestamped before a delete in another response - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") - .add("one", "A") - .buildUpdate()))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, fullPartitionDelete(cfm, dk, 1, nowInSec))); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, iter(new RowUpdateBuilder(cfm, nowInSec, 0L, dk).clustering("1") + .add("one", "A") + .buildUpdate()))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, fullPartitionDelete(cfm, dk, 1, nowInSec))); try (PartitionIterator data = resolver.resolve()) { @@ -381,23 +374,24 @@ public class DataResolverTest @Test public void testResolveMultipleDeleted() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 4, System.nanoTime(), readRepair); + EndpointsForRange replicas = makeReplicas(4); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); // deletes and columns with interleaved timestamp, with out of order return sequence - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, fullPartitionDelete(cfm, dk, 0, nowInSec))); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(command, peer1, fullPartitionDelete(cfm, dk, 0, nowInSec))); // these columns created after the previous deletion - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") - .add("one", "A") - .add("two", "A") - .buildUpdate()))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(command, peer2, iter(new RowUpdateBuilder(cfm, nowInSec, 1L, dk).clustering("1") + .add("one", "A") + .add("two", "A") + .buildUpdate()))); //this column created after the next delete - InetAddressAndPort peer3 = peer(); - resolver.preprocess(readResponseMessage(peer3, iter(new RowUpdateBuilder(cfm, nowInSec, 3L, dk).clustering("1") - .add("two", "B") - .buildUpdate()))); - InetAddressAndPort peer4 = peer(); - resolver.preprocess(readResponseMessage(peer4, fullPartitionDelete(cfm, dk, 2, nowInSec))); + InetAddressAndPort peer3 = replicas.get(2).endpoint(); + resolver.preprocess(response(command, peer3, iter(new RowUpdateBuilder(cfm, nowInSec, 3L, dk).clustering("1") + .add("two", "B") + .buildUpdate()))); + InetAddressAndPort peer4 = replicas.get(3).endpoint(); + resolver.preprocess(response(command, peer4, fullPartitionDelete(cfm, dk, 2, nowInSec))); try(PartitionIterator data = resolver.resolve()) { @@ -465,9 +459,10 @@ public class DataResolverTest */ private void resolveRangeTombstonesOnBoundary(long timestamp1, long timestamp2) { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - InetAddressAndPort peer2 = peer(); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); // 1st "stream" RangeTombstone one_two = tombstone("1", true , "2", false, timestamp1, nowInSec); @@ -485,8 +480,8 @@ public class DataResolverTest .addRangeTombstone(four_five) .buildUpdate()); - resolver.preprocess(readResponseMessage(peer1, iter1)); - resolver.preprocess(readResponseMessage(peer2, iter2)); + resolver.preprocess(response(command, peer1, iter1)); + resolver.preprocess(response(command, peer2, iter2)); // No results, we've only reconciled tombstones. try (PartitionIterator data = resolver.resolve()) @@ -538,9 +533,10 @@ public class DataResolverTest */ private void testRepairRangeTombstoneBoundary(int timestamp1, int timestamp2, int timestamp3) throws UnknownHostException { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - InetAddressAndPort peer2 = peer(); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); // 1st "stream" RangeTombstone one_nine = tombstone("0", true , "9", true, timestamp1, nowInSec); @@ -554,8 +550,8 @@ public class DataResolverTest RangeTombstoneBoundMarker close_nine = marker("9", false, true, timestamp3, nowInSec); UnfilteredPartitionIterator iter2 = iter(dk, open_one, boundary_five, close_nine); - resolver.preprocess(readResponseMessage(peer1, iter1)); - resolver.preprocess(readResponseMessage(peer2, iter2)); + resolver.preprocess(response(command, peer1, iter1)); + resolver.preprocess(response(command, peer2, iter2)); boolean shouldHaveRepair = timestamp1 != timestamp2 || timestamp1 != timestamp3; @@ -589,9 +585,10 @@ public class DataResolverTest @Test public void testRepairRangeTombstoneWithPartitionDeletion() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - InetAddressAndPort peer2 = peer(); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); // 1st "stream": just a partition deletion UnfilteredPartitionIterator iter1 = iter(PartitionUpdate.fullPartitionDelete(cfm, dk, 10, nowInSec)); @@ -602,8 +599,8 @@ public class DataResolverTest .addRangeTombstone(rt) .buildUpdate()); - resolver.preprocess(readResponseMessage(peer1, iter1)); - resolver.preprocess(readResponseMessage(peer2, iter2)); + resolver.preprocess(response(command, peer1, iter1)); + resolver.preprocess(response(command, peer2, iter2)); // No results, we've only reconciled tombstones. try (PartitionIterator data = resolver.resolve()) @@ -627,15 +624,16 @@ public class DataResolverTest @Test public void testRepairRangeTombstoneWithPartitionDeletion2() { - DataResolver resolver = new DataResolver(ks, command, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); - InetAddressAndPort peer1 = peer(); - InetAddressAndPort peer2 = peer(); + EndpointsForRange replicas = makeReplicas(2); + DataResolver resolver = new DataResolver(command, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); // 1st "stream": a partition deletion and a range tombstone RangeTombstone rt1 = tombstone("0", true , "9", true, 11, nowInSec); PartitionUpdate upd1 = new RowUpdateBuilder(cfm, nowInSec, 1L, dk) - .addRangeTombstone(rt1) - .buildUpdate(); + .addRangeTombstone(rt1) + .buildUpdate(); ((MutableDeletionInfo)upd1.deletionInfo()).add(new DeletionTime(10, nowInSec)); UnfilteredPartitionIterator iter1 = iter(upd1); @@ -647,8 +645,8 @@ public class DataResolverTest .addRangeTombstone(rt3) .buildUpdate()); - resolver.preprocess(readResponseMessage(peer1, iter1)); - resolver.preprocess(readResponseMessage(peer2, iter2)); + resolver.preprocess(response(command, peer1, iter1)); + resolver.preprocess(response(command, peer2, iter2)); // No results, we've only reconciled tombstones. try (PartitionIterator data = resolver.resolve()) @@ -678,8 +676,8 @@ public class DataResolverTest Slice slice = rt.deletedSlice(); ClusteringBound newStart = ClusteringBound.create(Kind.EXCL_START_BOUND, slice.start().getRawValues()); return condition - ? new RangeTombstone(Slice.make(newStart, slice.end()), rt.deletionTime()) - : rt; + ? new RangeTombstone(Slice.make(newStart, slice.end()), rt.deletionTime()) + : rt; } // Forces the end to be exclusive if the condition holds @@ -691,8 +689,8 @@ public class DataResolverTest Slice slice = rt.deletedSlice(); ClusteringBound newEnd = ClusteringBound.create(Kind.EXCL_END_BOUND, slice.end().getRawValues()); return condition - ? new RangeTombstone(Slice.make(slice.start(), newEnd), rt.deletionTime()) - : rt; + ? new RangeTombstone(Slice.make(slice.start(), newEnd), rt.deletionTime()) + : rt; } private static ByteBuffer bb(int b) @@ -708,9 +706,10 @@ public class DataResolverTest @Test public void testResolveComplexDelete() { + EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd, ConsistencyLevel.QUORUM); - DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); + DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); long[] ts = {100, 200}; @@ -719,8 +718,8 @@ public class DataResolverTest builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec)); builder.addCell(mapCell(0, 0, ts[0])); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); builder.newRow(Clustering.EMPTY); DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec); @@ -728,8 +727,8 @@ public class DataResolverTest Cell expectedCell = mapCell(1, 1, ts[1]); builder.addCell(expectedCell); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); try(PartitionIterator data = resolver.resolve()) { @@ -742,7 +741,6 @@ public class DataResolverTest } } - Mutation mutation = readRepair.getForEndpoint(peer1); Iterator rowIter = mutation.getPartitionUpdate(cfm2).iterator(); assertTrue(rowIter.hasNext()); @@ -760,9 +758,10 @@ public class DataResolverTest @Test public void testResolveDeletedCollection() { + EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd, ConsistencyLevel.QUORUM); - DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); + DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); long[] ts = {100, 200}; @@ -771,15 +770,15 @@ public class DataResolverTest builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec)); builder.addCell(mapCell(0, 0, ts[0])); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); builder.newRow(Clustering.EMPTY); DeletionTime expectedCmplxDelete = new DeletionTime(ts[1] - 1, nowInSec); builder.addComplexDeletion(m, expectedCmplxDelete); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); try(PartitionIterator data = resolver.resolve()) { @@ -803,9 +802,10 @@ public class DataResolverTest @Test public void testResolveNewCollection() { + EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd, ConsistencyLevel.QUORUM); - DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); + DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); long[] ts = {100, 200}; @@ -818,11 +818,11 @@ public class DataResolverTest builder.addCell(expectedCell); // empty map column - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.emptyUpdate(cfm2, dk)))); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.emptyUpdate(cfm2, dk)))); try(PartitionIterator data = resolver.resolve()) { @@ -852,9 +852,10 @@ public class DataResolverTest @Test public void testResolveNewCollectionOverwritingDeleted() { + EndpointsForRange replicas = makeReplicas(2); ReadCommand cmd = Util.cmd(cfs2, dk).withNowInSeconds(nowInSec).build(); TestableReadRepair readRepair = new TestableReadRepair(cmd, ConsistencyLevel.QUORUM); - DataResolver resolver = new DataResolver(ks, cmd, ConsistencyLevel.ALL, 2, System.nanoTime(), readRepair); + DataResolver resolver = new DataResolver(cmd, plan(replicas, ConsistencyLevel.ALL), readRepair, System.nanoTime()); long[] ts = {100, 200}; @@ -863,8 +864,8 @@ public class DataResolverTest builder.newRow(Clustering.EMPTY); builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec)); - InetAddressAndPort peer1 = peer(); - resolver.preprocess(readResponseMessage(peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer1 = replicas.get(0).endpoint(); + resolver.preprocess(response(cmd, peer1, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); // newer, overwritten map column builder.newRow(Clustering.EMPTY); @@ -873,8 +874,8 @@ public class DataResolverTest Cell expectedCell = mapCell(1, 1, ts[1]); builder.addCell(expectedCell); - InetAddressAndPort peer2 = peer(); - resolver.preprocess(readResponseMessage(peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())), cmd)); + InetAddressAndPort peer2 = replicas.get(1).endpoint(); + resolver.preprocess(response(cmd, peer2, iter(PartitionUpdate.singleRowUpdate(cfm2, dk, builder.build())))); try(PartitionIterator data = resolver.resolve()) { @@ -897,18 +898,6 @@ public class DataResolverTest Assert.assertNull(readRepair.sent.get(peer2)); } - private InetAddressAndPort peer() - { - try - { - return InetAddressAndPort.getByAddress(new byte[]{ 127, 0, 0, (byte) addressSuffix++ }); - } - catch (UnknownHostException e) - { - throw new RuntimeException(e); - } - } - private void assertRepairContainsDeletions(Mutation mutation, DeletionTime deletionTime, RangeTombstone...rangeTombstones) @@ -961,91 +950,8 @@ public class DataResolverTest assertEquals(update.metadata().name, cfm.name); } - - public MessageIn readResponseMessage(InetAddressAndPort from, UnfilteredPartitionIterator partitionIterator) + private ReplicaLayout.ForRange plan(EndpointsForRange replicas, ConsistencyLevel consistencyLevel) { - return readResponseMessage(from, partitionIterator, command); - + return new ReplicaLayout.ForRange(ks, consistencyLevel, ReplicaUtils.FULL_BOUNDS, replicas, replicas); } - public MessageIn readResponseMessage(InetAddressAndPort from, UnfilteredPartitionIterator partitionIterator, ReadCommand cmd) - { - return MessageIn.create(from, - ReadResponse.createRemoteDataResponse(partitionIterator, cmd), - Collections.EMPTY_MAP, - MessagingService.Verb.REQUEST_RESPONSE, - MessagingService.current_version); - } - - private RangeTombstone tombstone(Object start, Object end, long markedForDeleteAt, int localDeletionTime) - { - return tombstone(start, true, end, true, markedForDeleteAt, localDeletionTime); - } - - private RangeTombstone tombstone(Object start, boolean inclusiveStart, Object end, boolean inclusiveEnd, long markedForDeleteAt, int localDeletionTime) - { - ClusteringBound startBound = rtBound(start, true, inclusiveStart); - ClusteringBound endBound = rtBound(end, false, inclusiveEnd); - return new RangeTombstone(Slice.make(startBound, endBound), new DeletionTime(markedForDeleteAt, localDeletionTime)); - } - - private ClusteringBound rtBound(Object value, boolean isStart, boolean inclusive) - { - ClusteringBound.Kind kind = isStart - ? (inclusive ? Kind.INCL_START_BOUND : Kind.EXCL_START_BOUND) - : (inclusive ? Kind.INCL_END_BOUND : Kind.EXCL_END_BOUND); - - return ClusteringBound.create(kind, cfm.comparator.make(value).getRawValues()); - } - - private ClusteringBoundary rtBoundary(Object value, boolean inclusiveOnEnd) - { - ClusteringBound.Kind kind = inclusiveOnEnd - ? Kind.INCL_END_EXCL_START_BOUNDARY - : Kind.EXCL_END_INCL_START_BOUNDARY; - return ClusteringBoundary.create(kind, cfm.comparator.make(value).getRawValues()); - } - - private RangeTombstoneBoundMarker marker(Object value, boolean isStart, boolean inclusive, long markedForDeleteAt, int localDeletionTime) - { - return new RangeTombstoneBoundMarker(rtBound(value, isStart, inclusive), new DeletionTime(markedForDeleteAt, localDeletionTime)); - } - - private RangeTombstoneBoundaryMarker boundary(Object value, boolean inclusiveOnEnd, long markedForDeleteAt1, int localDeletionTime1, long markedForDeleteAt2, int localDeletionTime2) - { - return new RangeTombstoneBoundaryMarker(rtBoundary(value, inclusiveOnEnd), - new DeletionTime(markedForDeleteAt1, localDeletionTime1), - new DeletionTime(markedForDeleteAt2, localDeletionTime2)); - } - - private UnfilteredPartitionIterator fullPartitionDelete(TableMetadata table, DecoratedKey dk, long timestamp, int nowInSec) - { - return new SingletonUnfilteredPartitionIterator(PartitionUpdate.fullPartitionDelete(table, dk, timestamp, nowInSec).unfilteredIterator()); - } - - private UnfilteredPartitionIterator iter(PartitionUpdate update) - { - return new SingletonUnfilteredPartitionIterator(update.unfilteredIterator()); - } - - private UnfilteredPartitionIterator iter(DecoratedKey key, Unfiltered... unfiltereds) - { - SortedSet s = new TreeSet<>(cfm.comparator); - Collections.addAll(s, unfiltereds); - final Iterator iterator = s.iterator(); - - UnfilteredRowIterator rowIter = new AbstractUnfilteredRowIterator(cfm, - key, - DeletionTime.LIVE, - cfm.regularAndStaticColumns(), - Rows.EMPTY_STATIC_ROW, - false, - EncodingStats.NO_STATS) - { - protected Unfiltered computeNext() - { - return iterator.hasNext() ? iterator.next() : endOfData(); - } - }; - return new SingletonUnfilteredPartitionIterator(rowIter); - } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTransientTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTransientTest.java new file mode 100644 index 0000000000..811940033b --- /dev/null +++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTransientTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads; + +import java.util.concurrent.TimeUnit; + +import com.google.common.primitives.Ints; + +import org.apache.cassandra.Util; +import org.apache.cassandra.db.DecoratedKey; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.SimpleBuilders; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.reads.repair.TestableReadRepair; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.db.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.Replica.transientReplica; +import static org.apache.cassandra.locator.ReplicaUtils.full; +import static org.apache.cassandra.locator.ReplicaUtils.trans; + +/** + * Tests DataResolvers handing of transient replicas + */ +public class DataResolverTransientTest extends AbstractReadResponseTest +{ + private static DecoratedKey key; + + @Before + public void setUp() + { + key = Util.dk("key1"); + } + + private static PartitionUpdate.Builder update(TableMetadata metadata, String key, Row... rows) + { + PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, dk(key), metadata.regularAndStaticColumns(), rows.length, false); + for (Row row: rows) + { + builder.add(row); + } + return builder; + } + + private static PartitionUpdate.Builder update(Row... rows) + { + return update(cfm, "key1", rows); + } + + private static Row.SimpleBuilder rowBuilder(int clustering) + { + return new SimpleBuilders.RowBuilder(cfm, Integer.toString(clustering)); + } + + private static Row row(long timestamp, int clustering, int value) + { + return rowBuilder(clustering).timestamp(timestamp).add("c1", Integer.toString(value)).build(); + } + + private static DeletionTime deletion(long timeMillis) + { + TimeUnit MILLIS = TimeUnit.MILLISECONDS; + return new DeletionTime(MILLIS.toMicros(timeMillis), Ints.checkedCast(MILLIS.toSeconds(timeMillis))); + } + + /** + * Tests that the given update doesn't cause data resolver to attempt to repair a transient replica + */ + private void assertNoTransientRepairs(PartitionUpdate update) + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(update.metadata(), nowInSec, key); + EndpointsForToken targetReplicas = EndpointsForToken.of(key.getToken(), full(EP1), full(EP2), trans(EP3)); + TestableReadRepair repair = new TestableReadRepair(command, QUORUM); + DataResolver resolver = new DataResolver(command, plan(targetReplicas, ConsistencyLevel.QUORUM), repair, 0); + + Assert.assertFalse(resolver.isDataPresent()); + resolver.preprocess(response(command, EP1, iter(update), false)); + resolver.preprocess(response(command, EP2, iter(update), false)); + resolver.preprocess(response(command, EP3, EmptyIterators.unfilteredPartition(update.metadata()), false)); + + Assert.assertFalse(repair.dataWasConsumed()); + assertPartitionsEqual(filter(iter(update)), resolver.resolve()); + Assert.assertTrue(repair.dataWasConsumed()); + Assert.assertTrue(repair.sent.toString(), repair.sent.isEmpty()); + } + + @Test + public void emptyRowRepair() + { + assertNoTransientRepairs(update(row(1000, 4, 4), row(1000, 5, 5)).build()); + } + + @Test + public void emptyPartitionDeletionRepairs() + { + PartitionUpdate.Builder builder = update(); + builder.addPartitionDeletion(deletion(1999)); + assertNoTransientRepairs(builder.build()); + } + + /** + * Partition level deletion responses shouldn't sent data to a transient replica + */ + @Test + public void emptyRowDeletionRepairs() + { + PartitionUpdate.Builder builder = update(); + builder.add(rowBuilder(1).timestamp(1999).delete().build()); + assertNoTransientRepairs(builder.build()); + } + + @Test + public void emptyComplexDeletionRepair() + { + + long[] ts = {1000, 2000}; + + Row.Builder builder = BTreeRow.unsortedBuilder(); + builder.newRow(Clustering.EMPTY); + builder.addComplexDeletion(m, new DeletionTime(ts[0] - 1, nowInSec)); + assertNoTransientRepairs(update(cfm2, "key", builder.build()).build()); + + } + + @Test + public void emptyRangeTombstoneRepairs() + { + Slice slice = Slice.make(Clustering.make(ByteBufferUtil.bytes("a")), Clustering.make(ByteBufferUtil.bytes("b"))); + PartitionUpdate.Builder builder = update(); + builder.add(new RangeTombstone(slice, deletion(2000))); + assertNoTransientRepairs(builder.build()); + } + + /** + * If the full replicas need to repair each other, repairs shouldn't be sent to transient replicas + */ + @Test + public void fullRepairsIgnoreTransientReplicas() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk(5)); + EndpointsForToken targetReplicas = EndpointsForToken.of(key.getToken(), full(EP1), full(EP2), trans(EP3)); + TestableReadRepair repair = new TestableReadRepair(command, QUORUM); + DataResolver resolver = new DataResolver(command, plan(targetReplicas, QUORUM), repair, 0); + + Assert.assertFalse(resolver.isDataPresent()); + resolver.preprocess(response(command, EP1, iter(update(row(1000, 5, 5)).build()), false)); + resolver.preprocess(response(command, EP2, iter(update(row(2000, 4, 4)).build()), false)); + resolver.preprocess(response(command, EP3, EmptyIterators.unfilteredPartition(cfm), false)); + + Assert.assertFalse(repair.dataWasConsumed()); + + consume(resolver.resolve()); + + Assert.assertTrue(repair.dataWasConsumed()); + + Assert.assertTrue(repair.sent.containsKey(EP1)); + Assert.assertTrue(repair.sent.containsKey(EP2)); + Assert.assertFalse(repair.sent.containsKey(EP3)); + } + + /** + * If the transient replica has new data, the full replicas shoould be repaired, the transient one should not + */ + @Test + public void transientMismatchesRepairFullReplicas() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk(5)); + EndpointsForToken targetReplicas = EndpointsForToken.of(key.getToken(), full(EP1), full(EP2), trans(EP3)); + TestableReadRepair repair = new TestableReadRepair(command, QUORUM); + DataResolver resolver = new DataResolver(command, plan(targetReplicas, QUORUM), repair, 0); + + Assert.assertFalse(resolver.isDataPresent()); + PartitionUpdate transData = update(row(1000, 5, 5)).build(); + resolver.preprocess(response(command, EP1, EmptyIterators.unfilteredPartition(cfm), false)); + resolver.preprocess(response(command, EP2, EmptyIterators.unfilteredPartition(cfm), false)); + resolver.preprocess(response(command, EP3, iter(transData), false)); + + Assert.assertFalse(repair.dataWasConsumed()); + + assertPartitionsEqual(filter(iter(transData)), resolver.resolve()); + + Assert.assertTrue(repair.dataWasConsumed()); + + assertPartitionsEqual(filter(iter(transData)), filter(iter(repair.sent.get(EP1).getPartitionUpdate(cfm)))); + assertPartitionsEqual(filter(iter(transData)), filter(iter(repair.sent.get(EP2).getPartitionUpdate(cfm)))); + Assert.assertFalse(repair.sent.containsKey(EP3)); + + } + + private ReplicaLayout.ForToken plan(EndpointsForToken replicas, ConsistencyLevel consistencyLevel) + { + return new ReplicaLayout.ForToken(ks, consistencyLevel, replicas.token(), replicas, null, replicas); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java new file mode 100644 index 0000000000..5306a74180 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.SimpleBuilders; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.reads.repair.NoopReadRepair; +import org.apache.cassandra.service.reads.repair.TestableReadRepair; + +import static org.apache.cassandra.locator.ReplicaUtils.full; +import static org.apache.cassandra.locator.ReplicaUtils.trans; + +public class DigestResolverTest extends AbstractReadResponseTest +{ + private static PartitionUpdate.Builder update(TableMetadata metadata, String key, Row... rows) + { + PartitionUpdate.Builder builder = new PartitionUpdate.Builder(metadata, dk(key), metadata.regularAndStaticColumns(), rows.length, false); + for (Row row: rows) + { + builder.add(row); + } + return builder; + } + + private static PartitionUpdate.Builder update(Row... rows) + { + return update(cfm, "key1", rows); + } + + private static Row row(long timestamp, int clustering, int value) + { + SimpleBuilders.RowBuilder builder = new SimpleBuilders.RowBuilder(cfm, Integer.toString(clustering)); + builder.timestamp(timestamp).add("c1", Integer.toString(value)); + return builder.build(); + } + + @Test + public void noRepairNeeded() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); + EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2)); + TestableReadRepair readRepair = new TestableReadRepair(command, ConsistencyLevel.QUORUM); + DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), readRepair, 0); + + PartitionUpdate response = update(row(1000, 4, 4), row(1000, 5, 5)).build(); + + Assert.assertFalse(resolver.isDataPresent()); + resolver.preprocess(response(command, EP2, iter(response), true)); + resolver.preprocess(response(command, EP1, iter(response), false)); + Assert.assertTrue(resolver.isDataPresent()); + Assert.assertTrue(resolver.responsesMatch()); + + assertPartitionsEqual(filter(iter(response)), resolver.getData()); + } + + @Test + public void digestMismatch() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); + EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2)); + DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), NoopReadRepair.instance,0); + + PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build(); + PartitionUpdate response2 = update(row(2000, 4, 5)).build(); + + Assert.assertFalse(resolver.isDataPresent()); + resolver.preprocess(response(command, EP2, iter(response1), true)); + resolver.preprocess(response(command, EP1, iter(response2), false)); + Assert.assertTrue(resolver.isDataPresent()); + Assert.assertFalse(resolver.responsesMatch()); + Assert.assertFalse(resolver.hasTransientResponse()); + } + + /** + * A full response and a transient response, with the transient response being a subset of the full one + */ + @Test + public void agreeingTransient() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); + EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2)); + TestableReadRepair readRepair = new TestableReadRepair(command, ConsistencyLevel.QUORUM); + DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), readRepair, 0); + + PartitionUpdate response1 = update(row(1000, 4, 4), row(1000, 5, 5)).build(); + PartitionUpdate response2 = update(row(1000, 5, 5)).build(); + + Assert.assertFalse(resolver.isDataPresent()); + resolver.preprocess(response(command, EP1, iter(response1), false)); + resolver.preprocess(response(command, EP2, iter(response2), false)); + Assert.assertTrue(resolver.isDataPresent()); + Assert.assertTrue(resolver.responsesMatch()); + Assert.assertTrue(resolver.hasTransientResponse()); + Assert.assertTrue(readRepair.sent.isEmpty()); + } + + /** + * Transient responses shouldn't be classified as the single dataResponse + */ + @Test + public void transientResponse() + { + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); + EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), trans(EP2)); + DigestResolver resolver = new DigestResolver(command, plan(ConsistencyLevel.QUORUM, targetReplicas), NoopReadRepair.instance, 0); + + PartitionUpdate response2 = update(row(1000, 5, 5)).build(); + Assert.assertFalse(resolver.isDataPresent()); + Assert.assertFalse(resolver.hasTransientResponse()); + resolver.preprocess(response(command, EP2, iter(response2), false)); + Assert.assertFalse(resolver.isDataPresent()); + Assert.assertTrue(resolver.hasTransientResponse()); + } + + private ReplicaLayout.ForToken plan(ConsistencyLevel consistencyLevel, EndpointsForToken replicas) + { + return new ReplicaLayout.ForToken(ks, consistencyLevel, replicas.token(), replicas, null, replicas); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java index de7b2e4e5b..3b102f2834 100644 --- a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java @@ -18,10 +18,12 @@ package org.apache.cassandra.service.reads; -import java.util.List; import java.util.concurrent.TimeUnit; -import com.google.common.collect.ImmutableList; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.EndpointsForRange; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -35,12 +37,15 @@ import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.service.reads.AbstractReadExecutor; +import static org.apache.cassandra.locator.ReplicaUtils.full; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -48,7 +53,8 @@ public class ReadExecutorTest { static Keyspace ks; static ColumnFamilyStore cfs; - static List targets; + static EndpointsForToken targets; + static Token dummy; @BeforeClass public static void setUpClass() throws Throwable @@ -57,8 +63,13 @@ public class ReadExecutorTest SchemaLoader.createKeyspace("Foo", KeyspaceParams.simple(3), SchemaLoader.standardCFMD("Foo", "Bar")); ks = Keyspace.open("Foo"); cfs = ks.getColumnFamilyStore("Bar"); - targets = ImmutableList.of(InetAddressAndPort.getByName("127.0.0.255"), InetAddressAndPort.getByName("127.0.0.254"), InetAddressAndPort.getByName("127.0.0.253")); - cfs.sampleLatencyNanos = 0; + dummy = Murmur3Partitioner.instance.getMinimumToken(); + targets = EndpointsForToken.of(dummy, + full(InetAddressAndPort.getByName("127.0.0.255")), + full(InetAddressAndPort.getByName("127.0.0.254")), + full(InetAddressAndPort.getByName("127.0.0.253")) + ); + cfs.sampleReadLatencyNanos = 0; } @Before @@ -78,7 +89,7 @@ public class ReadExecutorTest { assertEquals(0, cfs.metric.speculativeInsufficientReplicas.getCount()); assertEquals(0, ks.metric.speculativeInsufficientReplicas.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(ks, cfs, new MockSinglePartitionReadCommand(), ConsistencyLevel.LOCAL_QUORUM, targets, System.nanoTime(), true); + AbstractReadExecutor executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, ConsistencyLevel.LOCAL_QUORUM), System.nanoTime(), true); executor.maybeTryAdditionalReplicas(); try { @@ -93,7 +104,7 @@ public class ReadExecutorTest assertEquals(1, ks.metric.speculativeInsufficientReplicas.getCount()); //Shouldn't increment - executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(ks, cfs, new MockSinglePartitionReadCommand(), ConsistencyLevel.LOCAL_QUORUM, targets, System.nanoTime(), false); + executor = new AbstractReadExecutor.NeverSpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(targets, ConsistencyLevel.LOCAL_QUORUM), System.nanoTime(), false); executor.maybeTryAdditionalReplicas(); try { @@ -119,7 +130,7 @@ public class ReadExecutorTest assertEquals(0, cfs.metric.speculativeFailedRetries.getCount()); assertEquals(0, ks.metric.speculativeRetries.getCount()); assertEquals(0, ks.metric.speculativeFailedRetries.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ks, cfs, new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365)), ConsistencyLevel.LOCAL_QUORUM, targets, System.nanoTime()); + AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365)), plan(ConsistencyLevel.LOCAL_QUORUM, targets, targets.subList(0, 2)), System.nanoTime()); executor.maybeTryAdditionalReplicas(); new Thread() { @@ -127,8 +138,8 @@ public class ReadExecutorTest public void run() { //Failures end the read promptly but don't require mock data to be suppleid - executor.handler.onFailure(targets.get(0), RequestFailureReason.READ_TOO_MANY_TOMBSTONES); - executor.handler.onFailure(targets.get(1), RequestFailureReason.READ_TOO_MANY_TOMBSTONES); + executor.handler.onFailure(targets.get(0).endpoint(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES); + executor.handler.onFailure(targets.get(1).endpoint(), RequestFailureReason.READ_TOO_MANY_TOMBSTONES); executor.handler.condition.signalAll(); } }.start(); @@ -160,7 +171,7 @@ public class ReadExecutorTest assertEquals(0, cfs.metric.speculativeFailedRetries.getCount()); assertEquals(0, ks.metric.speculativeRetries.getCount()); assertEquals(0, ks.metric.speculativeFailedRetries.getCount()); - AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ks, cfs, new MockSinglePartitionReadCommand(), ConsistencyLevel.LOCAL_QUORUM, targets, System.nanoTime()); + AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(cfs, new MockSinglePartitionReadCommand(), plan(ConsistencyLevel.LOCAL_QUORUM, targets, targets.subList(0, 2)), System.nanoTime()); executor.maybeTryAdditionalReplicas(); try { @@ -188,7 +199,7 @@ public class ReadExecutorTest MockSinglePartitionReadCommand(long timeout) { - super(false, 0, cfs.metadata(), 0, null, null, null, Util.dk("ry@n_luvs_teh_y@nk33z"), null, null); + super(false, 0, false, cfs.metadata(), 0, null, null, null, Util.dk("ry@n_luvs_teh_y@nk33z"), null, null); this.timeout = timeout; } @@ -213,4 +224,13 @@ public class ReadExecutorTest } + private ReplicaLayout.ForToken plan(EndpointsForToken targets, ConsistencyLevel consistencyLevel) + { + return plan(consistencyLevel, targets, targets); + } + + private ReplicaLayout.ForToken plan(ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken selected) + { + return new ReplicaLayout.ForToken(ks, consistencyLevel, natural.token(), natural, null, selected); + } } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java index 9717c4ea05..7e6ee292c5 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java @@ -8,7 +8,6 @@ import java.util.function.Consumer; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import org.junit.Assert; @@ -39,7 +38,11 @@ import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.net.MessageIn; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -49,6 +52,9 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.locator.Replica.fullReplica; +import static org.apache.cassandra.locator.ReplicaUtils.FULL_RANGE; + @Ignore public abstract class AbstractReadRepairTest { @@ -60,6 +66,12 @@ public abstract class AbstractReadRepairTest static InetAddressAndPort target3; static List targets; + static Replica replica1; + static Replica replica2; + static Replica replica3; + static EndpointsForRange replicas; + static ReplicaLayout replicaLayout; + static long now = TimeUnit.NANOSECONDS.toMicros(System.nanoTime()); static DecoratedKey key; static Cell cell1; @@ -191,7 +203,8 @@ public abstract class AbstractReadRepairTest ks = Keyspace.open(ksName); cfs = ks.getColumnFamilyStore("tbl"); - cfs.sampleLatencyNanos = 0; + cfs.sampleReadLatencyNanos = 0; + cfs.transientWriteLatencyNanos = 0; target1 = InetAddressAndPort.getByName("127.0.0.255"); target2 = InetAddressAndPort.getByName("127.0.0.254"); @@ -199,6 +212,13 @@ public abstract class AbstractReadRepairTest targets = ImmutableList.of(target1, target2, target3); + replica1 = fullReplica(target1, FULL_RANGE); + replica2 = fullReplica(target2, FULL_RANGE); + replica3 = fullReplica(target3, FULL_RANGE); + replicas = EndpointsForRange.of(replica1, replica2, replica3); + + replicaLayout = replicaLayout(ConsistencyLevel.QUORUM, replicas); + // default test values key = dk(5); cell1 = cell("v", "val1", now); @@ -220,14 +240,26 @@ public abstract class AbstractReadRepairTest public void setUp() { assert configured : "configureClass must be called in a @BeforeClass method"; - cfs.sampleLatencyNanos = 0; + + cfs.sampleReadLatencyNanos = 0; + cfs.transientWriteLatencyNanos = 0; } - public abstract InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency); - - public InstrumentedReadRepair createInstrumentedReadRepair() + static ReplicaLayout.ForRange replicaLayout(EndpointsForRange replicas, EndpointsForRange targets) { - return createInstrumentedReadRepair(command, System.nanoTime(), ConsistencyLevel.QUORUM); + return new ReplicaLayout.ForRange(ks, ConsistencyLevel.QUORUM, ReplicaUtils.FULL_BOUNDS, replicas, targets); + } + + static ReplicaLayout.ForRange replicaLayout(ConsistencyLevel consistencyLevel, EndpointsForRange replicas) + { + return new ReplicaLayout.ForRange(ks, consistencyLevel, ReplicaUtils.FULL_BOUNDS, replicas, replicas); + } + + public abstract InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaLayout replicaLayout, long queryStartNanoTime); + + public InstrumentedReadRepair createInstrumentedReadRepair(ReplicaLayout replicaLayout) + { + return createInstrumentedReadRepair(command, replicaLayout, System.nanoTime()); } @@ -238,12 +270,11 @@ public abstract class AbstractReadRepairTest @Test public void readSpeculationCycle() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(); + InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaLayout(replicas, EndpointsForRange.of(replica1, replica2))); ResultConsumer consumer = new ResultConsumer(); - Assert.assertEquals(epSet(), repair.getReadRecipients()); - repair.startRepair(null, targets, Lists.newArrayList(target1, target2), consumer); + repair.startRepair(null, consumer); Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients()); repair.maybeSendAdditionalReads(); @@ -258,11 +289,11 @@ public abstract class AbstractReadRepairTest @Test public void noSpeculationRequired() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(); + InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaLayout(replicas, EndpointsForRange.of(replica1, replica2))); ResultConsumer consumer = new ResultConsumer(); Assert.assertEquals(epSet(), repair.getReadRecipients()); - repair.startRepair(null, targets, Lists.newArrayList(target1, target2), consumer); + repair.startRepair(null, consumer); Assert.assertEquals(epSet(target1, target2), repair.getReadRecipients()); repair.getReadCallback().response(msg(target1, cell1)); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java index b06e88a4d2..a574d0296c 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java @@ -18,10 +18,8 @@ package org.apache.cassandra.service.reads.repair; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -31,24 +29,26 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.service.reads.ReadCallback; public class BlockingReadRepairTest extends AbstractReadRepairTest { - - private static class InstrumentedReadRepairHandler extends BlockingPartitionRepair + private static class InstrumentedReadRepairHandler, L extends ReplicaLayout> extends BlockingPartitionRepair { - public InstrumentedReadRepairHandler(Keyspace keyspace, DecoratedKey key, ConsistencyLevel consistency, Map repairs, int maxBlockFor, InetAddressAndPort[] participants) + public InstrumentedReadRepairHandler(Map repairs, int maxBlockFor, L replicaLayout) { - super(keyspace, key, consistency, repairs, maxBlockFor, participants); + super(Util.dk("not a real usable value"), repairs, maxBlockFor, replicaLayout); } Map mutationsSent = new HashMap<>(); @@ -58,13 +58,6 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest mutationsSent.put(endpoint, message.payload); } - List candidates = targets; - - protected List getCandidateEndpoints() - { - return candidates; - } - @Override protected boolean isLocal(InetAddressAndPort endpoint) { @@ -78,23 +71,22 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest configureClass(ReadRepairStrategy.BLOCKING); } - private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor, Collection participants) + private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor, ReplicaLayout replicaLayout) { - InetAddressAndPort[] participantArray = new InetAddressAndPort[participants.size()]; - participants.toArray(participantArray); - return new InstrumentedReadRepairHandler(ks, key, ConsistencyLevel.LOCAL_QUORUM, repairs, maxBlockFor, participantArray); + return new InstrumentedReadRepairHandler(repairs, maxBlockFor, replicaLayout); } - private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor) + private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor) { - return createRepairHandler(repairs, maxBlockFor, repairs.keySet()); + EndpointsForRange replicas = EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet())); + return createRepairHandler(repairs, maxBlockFor, replicaLayout(replicas, replicas)); } - private static class InstrumentedBlockingReadRepair extends BlockingReadRepair implements InstrumentedReadRepair + private static class InstrumentedBlockingReadRepair, L extends ReplicaLayout> extends BlockingReadRepair implements InstrumentedReadRepair { - public InstrumentedBlockingReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public InstrumentedBlockingReadRepair(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - super(command, queryStartNanoTime, consistency); + super(command, replicaLayout, queryStartNanoTime); } Set readCommandRecipients = new HashSet<>(); @@ -108,12 +100,6 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest readCallback = callback; } - @Override - Iterable getCandidatesForToken(Token token) - { - return targets; - } - @Override public Set getReadRecipients() { @@ -128,9 +114,9 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest } @Override - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaLayout replicaLayout, long queryStartNanoTime) { - return new InstrumentedBlockingReadRepair(command, queryStartNanoTime, consistency); + return new InstrumentedBlockingReadRepair(command, replicaLayout, queryStartNanoTime); } @Test @@ -152,12 +138,12 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest Mutation repair2 = mutation(cell1); // check that the correct repairs are calculated - Map repairs = new HashMap<>(); - repairs.put(target1, repair1); - repairs.put(target2, repair2); + Map repairs = new HashMap<>(); + repairs.put(replica1, repair1); + repairs.put(replica2, repair2); - - InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2); + ReplicaLayout.ForRange replicaLayout = replicaLayout(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, replicaLayout); Assert.assertTrue(handler.mutationsSent.isEmpty()); @@ -188,9 +174,9 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest @Test public void noAdditionalMutationRequired() throws Exception { - Map repairs = new HashMap<>(); - repairs.put(target1, mutation(cell2)); - repairs.put(target2, mutation(cell1)); + Map repairs = new HashMap<>(); + repairs.put(replica1, mutation(cell2)); + repairs.put(replica2, mutation(cell1)); InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2); handler.sendInitialRepairs(); @@ -209,15 +195,14 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest @Test public void noAdditionalMutationPossible() throws Exception { - Map repairs = new HashMap<>(); - repairs.put(target1, mutation(cell2)); - repairs.put(target2, mutation(cell1)); + Map repairs = new HashMap<>(); + repairs.put(replica1, mutation(cell2)); + repairs.put(replica2, mutation(cell1)); InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2); handler.sendInitialRepairs(); // we've already sent mutations to all candidates, so we shouldn't send any more - handler.candidates = Lists.newArrayList(target1, target2); handler.mutationsSent.clear(); handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS); Assert.assertTrue(handler.mutationsSent.isEmpty()); @@ -232,12 +217,11 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest { Mutation repair1 = mutation(cell2); - Map repairs = new HashMap<>(); - repairs.put(target1, repair1); - Collection participants = Lists.newArrayList(target1, target2); + Map repairs = new HashMap<>(); + repairs.put(replica1, repair1); // check that the correct initial mutations are sent out - InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, participants); + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, replicaLayout(replicas, EndpointsForRange.of(replica1, replica2))); handler.sendInitialRepairs(); Assert.assertEquals(1, handler.mutationsSent.size()); Assert.assertTrue(handler.mutationsSent.containsKey(target1)); @@ -252,10 +236,10 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest @Test public void onlyBlockOnQuorum() { - Map repairs = new HashMap<>(); - repairs.put(target1, mutation(cell1)); - repairs.put(target2, mutation(cell2)); - repairs.put(target3, mutation(cell3)); + Map repairs = new HashMap<>(); + repairs.put(replica1, mutation(cell1)); + repairs.put(replica2, mutation(cell2)); + repairs.put(replica3, mutation(cell3)); Assert.assertEquals(3, repairs.size()); InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2); @@ -277,30 +261,29 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest @Test public void remoteDCTest() throws Exception { - Map repairs = new HashMap<>(); - repairs.put(target1, mutation(cell1)); + Map repairs = new HashMap<>(); + repairs.put(replica1, mutation(cell1)); - - InetAddressAndPort remote1 = InetAddressAndPort.getByName("10.0.0.1"); - InetAddressAndPort remote2 = InetAddressAndPort.getByName("10.0.0.2"); + Replica remote1 = ReplicaUtils.full(InetAddressAndPort.getByName("10.0.0.1")); + Replica remote2 = ReplicaUtils.full(InetAddressAndPort.getByName("10.0.0.2")); repairs.put(remote1, mutation(cell1)); - Collection participants = Lists.newArrayList(target1, target2, remote1, remote2); - - InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, participants); + EndpointsForRange participants = EndpointsForRange.of(replica1, replica2, remote1, remote2); + ReplicaLayout.ForRange replicaLayout = new ReplicaLayout.ForRange(ks, ConsistencyLevel.LOCAL_QUORUM, ReplicaUtils.FULL_BOUNDS, participants, participants); + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, replicaLayout); handler.sendInitialRepairs(); Assert.assertEquals(2, handler.mutationsSent.size()); - Assert.assertTrue(handler.mutationsSent.containsKey(target1)); - Assert.assertTrue(handler.mutationsSent.containsKey(remote1)); + Assert.assertTrue(handler.mutationsSent.containsKey(replica1.endpoint())); + Assert.assertTrue(handler.mutationsSent.containsKey(remote1.endpoint())); Assert.assertEquals(1, handler.waitingOn()); Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); - handler.ack(remote1); + handler.ack(remote1.endpoint()); Assert.assertEquals(1, handler.waitingOn()); Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); - handler.ack(target1); + handler.ack(replica1.endpoint()); Assert.assertEquals(0, handler.waitingOn()); Assert.assertTrue(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java index 1f07c2b4d4..c345ee6fc4 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java @@ -26,20 +26,23 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import com.google.common.collect.Lists; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.OverrideConfigurationLoader; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.dht.Token; import org.apache.cassandra.diag.DiagnosticEventService; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.net.MessageOut; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.repair.ReadRepairEvent.ReadRepairEventType; @@ -72,12 +75,13 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest Mutation repair2 = mutation(cell1); // check that the correct repairs are calculated - Map repairs = new HashMap<>(); - repairs.put(target1, repair1); - repairs.put(target2, repair2); + Map repairs = new HashMap<>(); + repairs.put(replica1, repair1); + repairs.put(replica2, repair2); - DiagnosticPartitionReadRepairHandler handler = createRepairHandler(repairs, 2); + ReplicaLayout.ForRange replicaLayout = replicaLayout(replicas, EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet()))); + DiagnosticPartitionReadRepairHandler handler = createRepairHandler(repairs, 2, replicaLayout); Assert.assertTrue(handler.updatesByEp.isEmpty()); @@ -102,17 +106,20 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest Assert.assertTrue(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); } - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaLayout replicaLayout, long queryStartNanoTime) { - return new DiagnosticBlockingRepairHandler(command, queryStartNanoTime, consistency); + return new DiagnosticBlockingRepairHandler(command, replicaLayout, queryStartNanoTime); } - private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor) + private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor, ReplicaLayout replicaLayout) { - Set participants = repairs.keySet(); - InetAddressAndPort[] participantArray = new InetAddressAndPort[participants.size()]; - participants.toArray(participantArray); - return new DiagnosticPartitionReadRepairHandler(ks, key, ConsistencyLevel.LOCAL_QUORUM, repairs, maxBlockFor, participantArray); + return new DiagnosticPartitionReadRepairHandler(key, repairs, maxBlockFor, replicaLayout); + } + + private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor) + { + EndpointsForRange replicas = EndpointsForRange.copyOf(Lists.newArrayList(repairs.keySet())); + return createRepairHandler(repairs, maxBlockFor, replicaLayout(replicas, replicas)); } private static class DiagnosticBlockingRepairHandler extends BlockingReadRepair implements InstrumentedReadRepair @@ -120,9 +127,9 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest private Set recipients = Collections.emptySet(); private ReadCallback readCallback = null; - DiagnosticBlockingRepairHandler(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + DiagnosticBlockingRepairHandler(ReadCommand command, ReplicaLayout replicaLayout, long queryStartNanoTime) { - super(command, queryStartNanoTime, consistency); + super(command, replicaLayout, queryStartNanoTime); DiagnosticEventService.instance().subscribe(ReadRepairEvent.class, this::onRepairEvent); } @@ -130,7 +137,7 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest { if (e.getType() == ReadRepairEventType.START_REPAIR) recipients = new HashSet<>(e.destinations); else if (e.getType() == ReadRepairEventType.SPECULATED_READ) recipients.addAll(e.destinations); - Assert.assertEquals(targets, e.allEndpoints); + Assert.assertEquals(new HashSet<>(targets), new HashSet<>(e.allEndpoints)); Assert.assertNotNull(e.toMap()); } @@ -156,13 +163,13 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest } } - private static class DiagnosticPartitionReadRepairHandler extends BlockingPartitionRepair + private static class DiagnosticPartitionReadRepairHandler, L extends ReplicaLayout> extends BlockingPartitionRepair { private final Map updatesByEp = new HashMap<>(); - DiagnosticPartitionReadRepairHandler(Keyspace keyspace, DecoratedKey key, ConsistencyLevel consistency, Map repairs, int maxBlockFor, InetAddressAndPort[] participants) + DiagnosticPartitionReadRepairHandler(DecoratedKey key, Map repairs, int maxBlockFor, L replicaLayout) { - super(keyspace, key, consistency, repairs, maxBlockFor, participants); + super(key, repairs, maxBlockFor, replicaLayout); DiagnosticEventService.instance().subscribe(PartitionRepairEvent.class, this::onRepairEvent); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java index 2fb8ffcd9a..f3d2866aad 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/InstrumentedReadRepair.java @@ -20,10 +20,12 @@ package org.apache.cassandra.service.reads.repair; import java.util.Set; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.service.reads.ReadCallback; -public interface InstrumentedReadRepair extends ReadRepair +public interface InstrumentedReadRepair, L extends ReplicaLayout> extends ReadRepair { Set getReadRecipients(); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java index efce59a0be..9bb7eed1fe 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java @@ -26,20 +26,20 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; -import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.service.reads.ReadCallback; public class ReadOnlyReadRepairTest extends AbstractReadRepairTest { - private static class InstrumentedReadOnlyReadRepair extends ReadOnlyReadRepair implements InstrumentedReadRepair + private static class InstrumentedReadOnlyReadRepair, L extends ReplicaLayout> extends ReadOnlyReadRepair implements InstrumentedReadRepair { - public InstrumentedReadOnlyReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public InstrumentedReadOnlyReadRepair(ReadCommand command, L replicaLayout, long queryStartNanoTime) { - super(command, queryStartNanoTime, consistency); + super(command, replicaLayout, queryStartNanoTime); } Set readCommandRecipients = new HashSet<>(); @@ -53,12 +53,6 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest readCallback = callback; } - @Override - Iterable getCandidatesForToken(Token token) - { - return targets; - } - @Override public Set getReadRecipients() { @@ -79,22 +73,24 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest } @Override - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, long queryStartNanoTime, ConsistencyLevel consistency) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaLayout replicaLayout, long queryStartNanoTime) { - return new InstrumentedReadOnlyReadRepair(command, queryStartNanoTime, consistency); + return new InstrumentedReadOnlyReadRepair(command, replicaLayout, queryStartNanoTime); } @Test public void getMergeListener() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(); - Assert.assertSame(UnfilteredPartitionIterators.MergeListener.NOOP, repair.getMergeListener(new InetAddressAndPort[]{})); + ReplicaLayout replicaLayout = replicaLayout(replicas, replicas); + InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaLayout); + Assert.assertSame(UnfilteredPartitionIterators.MergeListener.NOOP, repair.getMergeListener(replicaLayout)); } @Test(expected = UnsupportedOperationException.class) public void repairPartitionFailure() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(); - repair.repairPartition(dk(1), Collections.emptyMap(), new InetAddressAndPort[]{}); + ReplicaLayout replicaLayout = replicaLayout(replicas, replicas); + InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaLayout); + repair.repairPartition(null, Collections.emptyMap(), replicaLayout); } } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java new file mode 100644 index 0000000000..e4ba25de2e --- /dev/null +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadRepairTest.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.repair; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.Iterables; + +import org.apache.cassandra.Util; +import org.apache.cassandra.locator.Endpoints; +import org.apache.cassandra.locator.EndpointsForRange; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; +import org.apache.cassandra.net.MessageOut; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.MigrationManager; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.locator.ReplicaUtils.full; + +public class ReadRepairTest +{ + static Keyspace ks; + static ColumnFamilyStore cfs; + static TableMetadata cfm; + static Replica target1; + static Replica target2; + static Replica target3; + static EndpointsForRange targets; + + private static class InstrumentedReadRepairHandler, L extends ReplicaLayout> extends BlockingPartitionRepair + { + public InstrumentedReadRepairHandler(Map repairs, int maxBlockFor, L replicaLayout) + { + super(Util.dk("not a valid key"), repairs, maxBlockFor, replicaLayout); + } + + Map mutationsSent = new HashMap<>(); + + protected void sendRR(MessageOut message, InetAddressAndPort endpoint) + { + mutationsSent.put(endpoint, message.payload); + } + + @Override + protected boolean isLocal(InetAddressAndPort endpoint) + { + return targets.endpoints().contains(endpoint); + } + } + + static long now = TimeUnit.NANOSECONDS.toMicros(System.nanoTime()); + static DecoratedKey key; + static Cell cell1; + static Cell cell2; + static Cell cell3; + static Mutation resolved; + + private static void assertRowsEqual(Row expected, Row actual) + { + try + { + Assert.assertEquals(expected == null, actual == null); + if (expected == null) + return; + Assert.assertEquals(expected.clustering(), actual.clustering()); + Assert.assertEquals(expected.deletion(), actual.deletion()); + Assert.assertArrayEquals(Iterables.toArray(expected.cells(), Cell.class), Iterables.toArray(expected.cells(), Cell.class)); + } catch (Throwable t) + { + throw new AssertionError(String.format("Row comparison failed, expected %s got %s", expected, actual), t); + } + } + + @BeforeClass + public static void setUpClass() throws Throwable + { + SchemaLoader.loadSchema(); + String ksName = "ks"; + + cfm = CreateTableStatement.parse("CREATE TABLE tbl (k int primary key, v text)", ksName).build(); + KeyspaceMetadata ksm = KeyspaceMetadata.create(ksName, KeyspaceParams.simple(3), Tables.of(cfm)); + MigrationManager.announceNewKeyspace(ksm, false); + + ks = Keyspace.open(ksName); + cfs = ks.getColumnFamilyStore("tbl"); + + cfs.sampleReadLatencyNanos = 0; + + target1 = full(InetAddressAndPort.getByName("127.0.0.255")); + target2 = full(InetAddressAndPort.getByName("127.0.0.254")); + target3 = full(InetAddressAndPort.getByName("127.0.0.253")); + + targets = EndpointsForRange.of(target1, target2, target3); + + // default test values + key = dk(5); + cell1 = cell("v", "val1", now); + cell2 = cell("v", "val2", now); + cell3 = cell("v", "val3", now); + resolved = mutation(cell1, cell2); + } + + private static DecoratedKey dk(int v) + { + return DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(v)); + } + + private static Cell cell(String name, String value, long timestamp) + { + return BufferCell.live(cfm.getColumn(ColumnIdentifier.getInterned(name, false)), timestamp, ByteBufferUtil.bytes(value)); + } + + private static Mutation mutation(Cell... cells) + { + Row.Builder builder = BTreeRow.unsortedBuilder(); + builder.newRow(Clustering.EMPTY); + for (Cell cell: cells) + { + builder.addCell(cell); + } + return new Mutation(PartitionUpdate.singleRowUpdate(cfm, key, builder.build())); + } + + private static InstrumentedReadRepairHandler createRepairHandler(Map repairs, int maxBlockFor, EndpointsForRange all, EndpointsForRange targets) + { + ReplicaLayout.ForRange replicaLayout = new ReplicaLayout.ForRange(ks, ConsistencyLevel.LOCAL_QUORUM, ReplicaUtils.FULL_BOUNDS, all, targets); + return new InstrumentedReadRepairHandler(repairs, maxBlockFor, replicaLayout); + } + + @Test + public void consistencyLevelTest() throws Exception + { + Assert.assertTrue(ConsistencyLevel.QUORUM.satisfies(ConsistencyLevel.QUORUM, ks)); + Assert.assertTrue(ConsistencyLevel.THREE.satisfies(ConsistencyLevel.QUORUM, ks)); + Assert.assertTrue(ConsistencyLevel.TWO.satisfies(ConsistencyLevel.QUORUM, ks)); + Assert.assertFalse(ConsistencyLevel.ONE.satisfies(ConsistencyLevel.QUORUM, ks)); + Assert.assertFalse(ConsistencyLevel.ANY.satisfies(ConsistencyLevel.QUORUM, ks)); + } + + private static void assertMutationEqual(Mutation expected, Mutation actual) + { + Assert.assertEquals(expected.getKeyspaceName(), actual.getKeyspaceName()); + Assert.assertEquals(expected.key(), actual.key()); + Assert.assertEquals(expected.key(), actual.key()); + PartitionUpdate expectedUpdate = Iterables.getOnlyElement(expected.getPartitionUpdates()); + PartitionUpdate actualUpdate = Iterables.getOnlyElement(actual.getPartitionUpdates()); + assertRowsEqual(Iterables.getOnlyElement(expectedUpdate), Iterables.getOnlyElement(actualUpdate)); + } + + @Test + public void additionalMutationRequired() throws Exception + { + Mutation repair1 = mutation(cell2); + Mutation repair2 = mutation(cell1); + + // check that the correct repairs are calculated + Map repairs = new HashMap<>(); + repairs.put(target1, repair1); + repairs.put(target2, repair2); + + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, + targets, EndpointsForRange.of(target1, target2)); + + Assert.assertTrue(handler.mutationsSent.isEmpty()); + + // check that the correct mutations are sent + handler.sendInitialRepairs(); + Assert.assertEquals(2, handler.mutationsSent.size()); + assertMutationEqual(repair1, handler.mutationsSent.get(target1.endpoint())); + assertMutationEqual(repair2, handler.mutationsSent.get(target2.endpoint())); + + // check that a combined mutation is speculatively sent to the 3rd target + handler.mutationsSent.clear(); + handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS); + Assert.assertEquals(1, handler.mutationsSent.size()); + assertMutationEqual(resolved, handler.mutationsSent.get(target3.endpoint())); + + // check repairs stop blocking after receiving 2 acks + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + handler.ack(target1.endpoint()); + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + handler.ack(target3.endpoint()); + Assert.assertTrue(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + } + + /** + * If we've received enough acks, we shouldn't send any additional mutations + */ + @Test + public void noAdditionalMutationRequired() throws Exception + { + Map repairs = new HashMap<>(); + repairs.put(target1, mutation(cell2)); + repairs.put(target2, mutation(cell1)); + + EndpointsForRange replicas = EndpointsForRange.of(target1, target2); + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, replicas, targets); + handler.sendInitialRepairs(); + handler.ack(target1.endpoint()); + handler.ack(target2.endpoint()); + + // both replicas have acked, we shouldn't send anything else out + handler.mutationsSent.clear(); + handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS); + Assert.assertTrue(handler.mutationsSent.isEmpty()); + } + + /** + * If there are no additional nodes we can send mutations to, we... shouldn't + */ + @Test + public void noAdditionalMutationPossible() throws Exception + { + Map repairs = new HashMap<>(); + repairs.put(target1, mutation(cell2)); + repairs.put(target2, mutation(cell1)); + + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, EndpointsForRange.of(target1, target2), + EndpointsForRange.of(target1, target2)); + handler.sendInitialRepairs(); + + // we've already sent mutations to all candidates, so we shouldn't send any more + handler.mutationsSent.clear(); + handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS); + Assert.assertTrue(handler.mutationsSent.isEmpty()); + } + + /** + * If we didn't send a repair to a replica because there wasn't a diff with the + * resolved column family, we shouldn't send it a speculative mutation + */ + @Test + public void mutationsArentSentToInSyncNodes() throws Exception + { + Mutation repair1 = mutation(cell2); + + Map repairs = new HashMap<>(); + repairs.put(target1, repair1); + + // check that the correct initial mutations are sent out + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, targets, EndpointsForRange.of(target1, target2)); + handler.sendInitialRepairs(); + Assert.assertEquals(1, handler.mutationsSent.size()); + Assert.assertTrue(handler.mutationsSent.containsKey(target1.endpoint())); + + // check that speculative mutations aren't sent to target2 + handler.mutationsSent.clear(); + handler.maybeSendAdditionalWrites(0, TimeUnit.NANOSECONDS); + + Assert.assertEquals(1, handler.mutationsSent.size()); + Assert.assertTrue(handler.mutationsSent.containsKey(target3.endpoint())); + } + + @Test + public void onlyBlockOnQuorum() + { + Map repairs = new HashMap<>(); + repairs.put(target1, mutation(cell1)); + repairs.put(target2, mutation(cell2)); + repairs.put(target3, mutation(cell3)); + Assert.assertEquals(3, repairs.size()); + + EndpointsForRange replicas = EndpointsForRange.of(target1, target2, target3); + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, replicas, replicas); + handler.sendInitialRepairs(); + + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + handler.ack(target1.endpoint()); + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + + // here we should stop blocking, even though we've sent 3 repairs + handler.ack(target2.endpoint()); + Assert.assertTrue(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + + } + + /** + * For dc local consistency levels, noop mutations and responses from remote dcs should not affect effective blockFor + */ + @Test + public void remoteDCTest() throws Exception + { + Map repairs = new HashMap<>(); + repairs.put(target1, mutation(cell1)); + + Replica remote1 = full(InetAddressAndPort.getByName("10.0.0.1")); + Replica remote2 = full(InetAddressAndPort.getByName("10.0.0.2")); + repairs.put(remote1, mutation(cell1)); + + EndpointsForRange participants = EndpointsForRange.of(target1, target2, remote1, remote2); + EndpointsForRange targets = EndpointsForRange.of(target1, target2); + + InstrumentedReadRepairHandler handler = createRepairHandler(repairs, 2, participants, targets); + handler.sendInitialRepairs(); + Assert.assertEquals(2, handler.mutationsSent.size()); + Assert.assertTrue(handler.mutationsSent.containsKey(target1.endpoint())); + Assert.assertTrue(handler.mutationsSent.containsKey(remote1.endpoint())); + + Assert.assertEquals(1, handler.waitingOn()); + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + + handler.ack(remote1.endpoint()); + Assert.assertEquals(1, handler.waitingOn()); + Assert.assertFalse(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + + handler.ack(target1.endpoint()); + Assert.assertEquals(0, handler.waitingOn()); + Assert.assertTrue(handler.awaitRepairs(0, TimeUnit.NANOSECONDS)); + } +} diff --git a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java index f97980b5df..2a2dec2fed 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/TestableReadRepair.java @@ -29,17 +29,25 @@ import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.service.reads.DigestResolver; -public class TestableReadRepair implements ReadRepair +public class TestableReadRepair, L extends ReplicaLayout> implements ReadRepair { public final Map sent = new HashMap<>(); private final ReadCommand command; private final ConsistencyLevel consistency; + private boolean partitionListenerClosed = false; + private boolean rowListenerClosed = true; + public TestableReadRepair(ReadCommand command, ConsistencyLevel consistency) { this.command = command; @@ -47,13 +55,35 @@ public class TestableReadRepair implements ReadRepair } @Override - public UnfilteredPartitionIterators.MergeListener getMergeListener(InetAddressAndPort[] endpoints) + public UnfilteredPartitionIterators.MergeListener getMergeListener(L endpoints) { - return new PartitionIteratorMergeListener(endpoints, command, consistency, this); + return new PartitionIteratorMergeListener(endpoints, command, consistency, this) { + @Override + public void close() + { + super.close(); + partitionListenerClosed = true; + } + + @Override + public UnfilteredRowIterators.MergeListener getRowMergeListener(DecoratedKey partitionKey, List versions) + { + assert rowListenerClosed; + rowListenerClosed = false; + return new RowIteratorMergeListener(partitionKey, columns(versions), isReversed(versions), endpoints, command, consistency, TestableReadRepair.this) { + @Override + public void close() + { + super.close(); + rowListenerClosed = true; + } + }; + } + }; } @Override - public void startRepair(DigestResolver digestResolver, List allEndpoints, List contactedEndpoints, Consumer resultConsumer) + public void startRepair(DigestResolver digestResolver, Consumer resultConsumer) { } @@ -83,13 +113,19 @@ public class TestableReadRepair implements ReadRepair } @Override - public void repairPartition(DecoratedKey key, Map mutations, InetAddressAndPort[] destinations) + public void repairPartition(DecoratedKey partitionKey, Map mutations, L replicaLayout) { - sent.putAll(mutations); + for (Map.Entry entry: mutations.entrySet()) + sent.put(entry.getKey().endpoint(), entry.getValue()); } public Mutation getForEndpoint(InetAddressAndPort endpoint) { return sent.get(endpoint); } -} + + public boolean dataWasConsumed() + { + return partitionListenerClosed && rowListenerClosed; + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java index bc501be6b3..909e221ae2 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; @@ -144,7 +145,7 @@ public class StreamingTransferTest ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("key2")), p.getMinimumToken())); StreamResultFuture futureResult = new StreamPlan(StreamOperation.OTHER) - .requestRanges(LOCAL, KEYSPACE2, ranges) + .requestRanges(LOCAL, KEYSPACE2, RangesAtEndpoint.toDummyList(ranges), RangesAtEndpoint.toDummyList(Collections.emptyList())) .execute(); UUID planId = futureResult.planId; @@ -238,13 +239,13 @@ public class StreamingTransferTest List> ranges = new ArrayList<>(); // wrapped range ranges.add(new Range(p.getToken(ByteBufferUtil.bytes("key1")), p.getToken(ByteBufferUtil.bytes("key0")))); - StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getTableName()); + StreamPlan streamPlan = new StreamPlan(StreamOperation.OTHER).transferRanges(LOCAL, cfs.keyspace.getName(), RangesAtEndpoint.toDummyList(ranges), cfs.getTableName()); streamPlan.execute().get(); //cannot add ranges after stream session is finished try { - streamPlan.transferRanges(LOCAL, cfs.keyspace.getName(), ranges, cfs.getTableName()); + streamPlan.transferRanges(LOCAL, cfs.keyspace.getName(), RangesAtEndpoint.toDummyList(ranges), cfs.getTableName()); fail("Should have thrown exception"); } catch (RuntimeException e) diff --git a/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java b/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java index 28423740ed..f8f6b12ecd 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/AccumulatorTest.java @@ -95,7 +95,7 @@ public class AccumulatorTest assertEquals("0", accu.get(3)); - Iterator iter = accu.iterator(); + Iterator iter = accu.snapshot().iterator(); assertEquals("3", iter.next()); assertEquals("2", iter.next());