From 51e63f0ff3dc5f47dcc1214e81eae93fe4a75bce Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Wed, 14 May 2025 12:35:25 -0400 Subject: [PATCH] Add mutation tracking summary to SSTables Patch by Abe Ratnofsky; reviewed by Ariel Weisberg, Blake Eggleston for CASSANDRA-20336 --- .../schema/AlterTableStatement.java | 3 + .../db/CassandraTableWriteHandler.java | 5 +- .../cassandra/db/ColumnFamilyStore.java | 20 +- .../db/CoordinatorLogBoundaries.java | 88 ++++++ .../db/CoordinatorLogBoundariesBuilder.java | 92 +++++++ .../db/CoordinatorLogBoundariesMap.java | 80 ++++++ .../org/apache/cassandra/db/Keyspace.java | 4 +- .../db/MutableCoordinatorLogBoundaries.java | 46 ++++ .../apache/cassandra/db/SSTableImporter.java | 8 +- .../cassandra/db/TableWriteHandler.java | 3 +- .../AbstractCompactionStrategy.java | 2 + .../db/compaction/AbstractStrategyHolder.java | 2 + .../db/compaction/CompactionManager.java | 18 +- .../compaction/CompactionStrategyHolder.java | 3 + .../compaction/CompactionStrategyManager.java | 3 + .../db/compaction/CompactionTask.java | 9 + .../db/compaction/PendingRepairHolder.java | 3 + .../compaction/UnifiedCompactionStrategy.java | 2 + .../cassandra/db/compaction/Upgrader.java | 1 + .../unified/ShardedMultiWriter.java | 5 + .../writers/CompactionAwareWriter.java | 4 + .../cassandra/db/memtable/Flushing.java | 1 + .../cassandra/db/memtable/Memtable.java | 11 +- .../db/memtable/ShardedSkipListMemtable.java | 34 ++- .../db/memtable/SkipListMemtable.java | 18 +- .../cassandra/db/memtable/TrieMemtable.java | 26 +- .../CassandraCompressedStreamReader.java | 2 +- .../db/streaming/CassandraOutgoingFile.java | 7 + .../db/streaming/CassandraStreamReader.java | 9 +- .../index/internal/CassandraIndex.java | 5 +- .../sstable/AbstractSSTableSimpleWriter.java | 13 +- .../io/sstable/CQLSSTableWriter.java | 7 + .../io/sstable/RangeAwareSSTableWriter.java | 9 +- .../sstable/SSTableSimpleUnsortedWriter.java | 8 + .../io/sstable/SSTableSimpleWriter.java | 8 + .../io/sstable/SSTableTxnWriter.java | 16 +- .../io/sstable/SimpleSSTableMultiWriter.java | 3 + .../io/sstable/format/SSTableReader.java | 6 + .../io/sstable/format/SSTableWriter.java | 17 ++ .../sstable/format/SortedTableScrubber.java | 4 +- .../cassandra/io/sstable/format/Version.java | 2 + .../io/sstable/format/big/BigFormat.java | 13 +- .../io/sstable/format/bti/BtiFormat.java | 13 +- .../sstable/metadata/MetadataCollector.java | 36 +-- .../sstable/metadata/MetadataSerializer.java | 14 +- .../io/sstable/metadata/StatsMetadata.java | 19 ++ .../replication/CoordinatorLogId.java | 5 +- .../cassandra/schema/TableMetadata.java | 3 + .../cassandra/streaming/OutgoingStream.java | 3 + .../messages/OutgoingStreamMessage.java | 3 +- .../messages/StreamMessageHeader.java | 11 +- .../db-51-bti-CompressionInfo.db | Bin 0 -> 207 bytes .../legacy_db_clust/db-51-bti-Data.db | Bin 0 -> 8496 bytes .../legacy_db_clust/db-51-bti-Digest.crc32 | 1 + .../legacy_db_clust/db-51-bti-Filter.db | Bin 0 -> 24 bytes .../legacy_db_clust/db-51-bti-Partitions.db | Bin 0 -> 62 bytes .../legacy_db_clust/db-51-bti-Rows.db | Bin 0 -> 508 bytes .../legacy_db_clust/db-51-bti-Statistics.db | Bin 0 -> 7313 bytes .../legacy_db_clust/db-51-bti-TOC.txt | 8 + .../db-51-bti-CompressionInfo.db | Bin 0 -> 207 bytes .../db-51-bti-Data.db | Bin 0 -> 8451 bytes .../db-51-bti-Digest.crc32 | 1 + .../db-51-bti-Filter.db | Bin 0 -> 24 bytes .../db-51-bti-Partitions.db | Bin 0 -> 62 bytes .../db-51-bti-Rows.db | Bin 0 -> 508 bytes .../db-51-bti-Statistics.db | Bin 0 -> 7333 bytes .../db-51-bti-TOC.txt | 8 + .../db-51-bti-CompressionInfo.db | Bin 0 -> 199 bytes .../legacy_db_clust_counter/db-51-bti-Data.db | Bin 0 -> 7332 bytes .../db-51-bti-Digest.crc32 | 1 + .../db-51-bti-Filter.db | Bin 0 -> 24 bytes .../db-51-bti-Partitions.db | Bin 0 -> 62 bytes .../legacy_db_clust_counter/db-51-bti-Rows.db | Bin 0 -> 508 bytes .../db-51-bti-Statistics.db | Bin 0 -> 7322 bytes .../legacy_db_clust_counter/db-51-bti-TOC.txt | 8 + .../db-51-bti-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_db_simple/db-51-bti-Data.db | Bin 0 -> 78 bytes .../legacy_db_simple/db-51-bti-Digest.crc32 | 1 + .../legacy_db_simple/db-51-bti-Filter.db | Bin 0 -> 24 bytes .../legacy_db_simple/db-51-bti-Partitions.db | Bin 0 -> 58 bytes .../legacy_db_simple/db-51-bti-Rows.db | 0 .../legacy_db_simple/db-51-bti-Statistics.db | Bin 0 -> 4823 bytes .../legacy_db_simple/db-51-bti-TOC.txt | 8 + .../db-51-bti-CompressionInfo.db | Bin 0 -> 47 bytes .../db-51-bti-Data.db | Bin 0 -> 132 bytes .../db-51-bti-Digest.crc32 | 1 + .../db-51-bti-Filter.db | Bin 0 -> 24 bytes .../db-51-bti-Partitions.db | Bin 0 -> 60 bytes .../db-51-bti-Rows.db | 0 .../db-51-bti-Statistics.db | Bin 0 -> 4832 bytes .../db-51-bti-TOC.txt | 8 + .../db-51-bti-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_db_tuple/db-51-bti-Data.db | Bin 0 -> 300 bytes .../legacy_db_tuple/db-51-bti-Digest.crc32 | 1 + .../legacy_db_tuple/db-51-bti-Filter.db | Bin 0 -> 24 bytes .../legacy_db_tuple/db-51-bti-Partitions.db | Bin 0 -> 62 bytes .../legacy_db_tuple/db-51-bti-Rows.db | 0 .../legacy_db_tuple/db-51-bti-Statistics.db | Bin 0 -> 5802 bytes .../legacy_db_tuple/db-51-bti-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 207 bytes .../legacy_ob_clust/ob-51-big-Data.db | Bin 0 -> 8568 bytes .../legacy_ob_clust/ob-51-big-Digest.crc32 | 1 + .../legacy_ob_clust/ob-51-big-Filter.db | Bin 0 -> 24 bytes .../legacy_ob_clust/ob-51-big-Index.db | Bin 0 -> 157498 bytes .../legacy_ob_clust/ob-51-big-Statistics.db | Bin 0 -> 7333 bytes .../legacy_ob_clust/ob-51-big-Summary.db | Bin 0 -> 47 bytes .../legacy_ob_clust/ob-51-big-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 207 bytes .../ob-51-big-Data.db | Bin 0 -> 8623 bytes .../ob-51-big-Digest.crc32 | 1 + .../ob-51-big-Filter.db | Bin 0 -> 24 bytes .../ob-51-big-Index.db | Bin 0 -> 157498 bytes .../ob-51-big-Statistics.db | Bin 0 -> 7333 bytes .../ob-51-big-Summary.db | Bin 0 -> 47 bytes .../ob-51-big-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 199 bytes .../legacy_ob_clust_counter/ob-51-big-Data.db | Bin 0 -> 7909 bytes .../ob-51-big-Digest.crc32 | 1 + .../ob-51-big-Filter.db | Bin 0 -> 24 bytes .../ob-51-big-Index.db | Bin 0 -> 157498 bytes .../ob-51-big-Statistics.db | Bin 0 -> 7342 bytes .../ob-51-big-Summary.db | Bin 0 -> 47 bytes .../legacy_ob_clust_counter/ob-51-big-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_ob_simple/ob-51-big-Data.db | Bin 0 -> 79 bytes .../legacy_ob_simple/ob-51-big-Digest.crc32 | 1 + .../legacy_ob_simple/ob-51-big-Filter.db | Bin 0 -> 24 bytes .../legacy_ob_simple/ob-51-big-Index.db | Bin 0 -> 25 bytes .../legacy_ob_simple/ob-51-big-Statistics.db | Bin 0 -> 4843 bytes .../legacy_ob_simple/ob-51-big-Summary.db | Bin 0 -> 47 bytes .../legacy_ob_simple/ob-51-big-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 47 bytes .../ob-51-big-Data.db | Bin 0 -> 139 bytes .../ob-51-big-Digest.crc32 | 1 + .../ob-51-big-Filter.db | Bin 0 -> 24 bytes .../ob-51-big-Index.db | Bin 0 -> 27 bytes .../ob-51-big-Statistics.db | Bin 0 -> 4852 bytes .../ob-51-big-Summary.db | Bin 0 -> 47 bytes .../ob-51-big-TOC.txt | 8 + .../ob-51-big-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_ob_tuple/ob-51-big-Data.db | Bin 0 -> 301 bytes .../legacy_ob_tuple/ob-51-big-Digest.crc32 | 1 + .../legacy_ob_tuple/ob-51-big-Filter.db | Bin 0 -> 24 bytes .../legacy_ob_tuple/ob-51-big-Index.db | Bin 0 -> 29 bytes .../legacy_ob_tuple/ob-51-big-Statistics.db | Bin 0 -> 5806 bytes .../legacy_ob_tuple/ob-51-big-Summary.db | Bin 0 -> 47 bytes .../legacy_ob_tuple/ob-51-big-TOC.txt | 8 + .../microbench/ZeroCopyStreamingBench.java | 5 +- .../cassandra/db/ColumnFamilyStoreTest.java | 2 +- ...CoordinatorLogBoundariesLifecycleTest.java | 252 ++++++++++++++++++ .../db/CoordinatorLogBoundariesTest.java | 137 ++++++++++ .../cassandra/db/SerializationHeaderTest.java | 1 + .../db/compaction/AntiCompactionTest.java | 3 +- .../db/lifecycle/LogTransactionTest.java | 5 +- .../db/lifecycle/RealTransactionsTest.java | 2 + .../cassandra/db/lifecycle/TrackerTest.java | 13 +- ...assandraEntireSSTableStreamWriterTest.java | 3 +- ...StreamConcurrentComponentMutationTest.java | 3 +- .../index/internal/CustomCassandraIndex.java | 5 +- .../io/sstable/CQLSSTableWriterTest.java | 6 +- .../io/sstable/LegacySSTableTest.java | 2 +- .../sstable/RangeAwareSSTableWriterTest.java | 2 + .../io/sstable/SSTableFlushObserverTest.java | 5 +- .../io/sstable/SSTableRewriterTest.java | 3 +- .../cassandra/io/sstable/SSTableUtils.java | 4 +- .../io/sstable/SSTableWriterTestBase.java | 2 + .../sstable/SSTableWriterTransactionTest.java | 3 +- .../cassandra/io/sstable/ScrubTest.java | 2 + .../metadata/MetadataSerializerTest.java | 7 +- .../apache/cassandra/schema/MockSchema.java | 42 ++- .../cassandra/streaming/StreamReaderTest.java | 8 +- .../async/StreamingInboundHandlerTest.java | 9 +- 172 files changed, 1215 insertions(+), 146 deletions(-) create mode 100644 src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java create mode 100644 src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java create mode 100644 src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java create mode 100644 src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Data.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Partitions.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Rows.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Statistics.db create mode 100644 test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-TOC.txt create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Data.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Index.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db create mode 100644 test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-TOC.txt create mode 100644 test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java create mode 100644 test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java index aee1496cb5..cebc174e24 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java @@ -711,6 +711,9 @@ public abstract class AlterTableStatement extends AlterSchemaStatement params = validateAndUpdateTransactionalMigration(table.isCounter(), table.params, params); + if (table.replicationType().isTracked() && params.memtable.factory().streamFromMemtable()) + throw ire("Cannot use mutation tracking with persistent memtables"); + return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params))); } } diff --git a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java index fc0979ff4f..089fbb2416 100644 --- a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java +++ b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.tracing.Tracing; public class CassandraTableWriteHandler implements TableWriteHandler @@ -31,10 +32,10 @@ public class CassandraTableWriteHandler implements TableWriteHandler } @Override - public void write(PartitionUpdate update, WriteContext context, boolean updateIndexes) + public void write(MutationId mutationId, PartitionUpdate update, WriteContext context, boolean updateIndexes) { CassandraWriteContext ctx = CassandraWriteContext.fromContext(context); Tracing.trace("Adding to {} memtable", update.metadata().name); - cfs.apply(update, ctx, updateIndexes); + cfs.apply(mutationId, update, ctx, updateIndexes); } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index a60d4a01a7..f3f8ef70ec 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -136,6 +136,7 @@ import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.repair.TableRepairManager; import org.apache.cassandra.repair.consistent.admin.CleanupSummary; import org.apache.cassandra.repair.consistent.admin.PendingStat; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; @@ -674,19 +675,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return memtableFactory.streamFromMemtable(); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, ILifecycleTransaction txn) { - return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, null, 0, header, txn); + return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, 0, header, txn); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet commitLogPositions, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, SerializationHeader header, ILifecycleTransaction txn) { - return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, 0, header, txn); + return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, commitLogPositions, 0, header, txn); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn) { - return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn); + return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn); } public boolean supportsEarlyOpen() @@ -1498,7 +1499,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner * @param context write context for current update * @param updateIndexes whether secondary indexes should be updated */ - public void apply(PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes) + public void apply(MutationId mutationId, PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes) { long start = nanoTime(); @@ -1508,7 +1509,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner { Memtable mt = data.getMemtableFor(opGroup, commitLogPosition); UpdateTransaction indexer = newUpdateTransaction(update, context, updateIndexes, mt); - long timeDelta = mt.put(update, indexer, opGroup); + long timeDelta = mt.put(mutationId, update, indexer, opGroup); DecoratedKey key = update.partitionKey(); invalidateCachedPartition(key); metric.topWritePartitionFrequency.addSample(key.getKey(), 1); @@ -2419,12 +2420,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return null; List> dataSets = new ArrayList<>(ranges.size()); + CoordinatorLogBoundariesBuilder boundaries = new CoordinatorLogBoundariesBuilder(); IntervalSet.Builder commitLogIntervals = new IntervalSet.Builder(); long keys = 0; for (Range range : ranges) { Memtable.FlushablePartitionSet dataSet = current.getFlushSet(range.left, range.right); dataSets.add(dataSet); + boundaries.addAll(dataSet.coordinatorLogBoundaries()); commitLogIntervals.add(dataSet.commitLogLowerBound(), dataSet.commitLogUpperBound()); keys += dataSet.partitionCount(); } @@ -2438,6 +2441,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner 0, repairSessionID, false, + boundaries.build(), commitLogIntervals.build(), new SerializationHeader(true, firstDataSet.metadata(), diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java new file mode 100644 index 0000000000..7b506bd3d5 --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.io.IOException; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.utils.vint.VIntCoding; + +/** + * Max mutation ID present in this SSTable for each coordinator log, to determine whether an SSTable is reconciled or + * not. Once max mutation IDs are reconciled, next compaction can safely mark this SSTabled as repaired. Note that peers + * may have reconciled all mutations included in an SSTable, but {@link StatsMetadata#repairedAt} is dependent on + * compaction timing, so "nodetool repair --validate" may report temporary disagreements on the repaired set. + *

+ * A reference to this class should be treated as immutable. Do not cast to {@link CoordinatorLogBoundariesMap}. + * Iterable over {@link CoordinatorLogId}. + */ +public interface CoordinatorLogBoundaries extends Iterable +{ + int maxOffset(long logId); + MutationId max(long logId); + int size(); + + IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(CoordinatorLogBoundaries boundaries, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_52) + return; + out.writeUnsignedVInt32(boundaries.size()); + for (long logId : boundaries) + MutationId.serializer.serialize(boundaries.max(logId), out, version); + } + + @Override + public CoordinatorLogBoundaries deserialize(DataInputPlus in, int version) throws IOException + { + if (version < MessagingService.VERSION_52) + return CoordinatorLogBoundaries.NONE; + int size = in.readUnsignedVInt32(); + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(size); + for (int i = 0; i < size; i++) + { + MutationId mutationId = MutationId.serializer.deserialize(in, version); + boundaries.add(mutationId); + } + return boundaries; + } + + @Override + public long serializedSize(CoordinatorLogBoundaries boundaries, int version) + { + if (version < MessagingService.VERSION_52) + return 0; + long size = 0; + size += VIntCoding.computeUnsignedVIntSize(boundaries.size()); + for (long logId : boundaries) + size += MutationId.serializer.serializedSize(boundaries.max(logId), version); + return size; + } + }; + + CoordinatorLogBoundaries NONE = new CoordinatorLogBoundariesMap(); +} diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java new file mode 100644 index 0000000000..f9339057cc --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.Iterator; + +import javax.annotation.concurrent.NotThreadSafe; + +import com.google.common.collect.Iterators; + +import org.agrona.collections.Long2ObjectHashMap; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.ShortMutationId; + +@NotThreadSafe +public class CoordinatorLogBoundariesBuilder +{ + private static final MutationId NONE = MutationId.none(); + private static final int NONE_OFFSET = NONE.offset(); + private final Long2ObjectHashMap ids; + + public CoordinatorLogBoundariesBuilder() + { + this.ids = new Long2ObjectHashMap<>(); + } + + public CoordinatorLogBoundariesBuilder add(MutationId mutationId) + { + if (mutationId.isNone()) + return this; + long logId = mutationId.logId(); + + MutationId existing = ids.get(logId); + if (existing == null || ShortMutationId.comparator.compare(existing, mutationId) < 0) + ids.put(mutationId.logId(), mutationId); + return this; + } + + public CoordinatorLogBoundariesBuilder addAll(CoordinatorLogBoundaries boundaries) + { + for (long logId : boundaries) + add(boundaries.max(logId)); + return this; + } + + public CoordinatorLogBoundaries build() + { + return new CoordinatorLogBoundaries() + { + @Override + public int maxOffset(long logId) + { + MutationId id = ids.get(logId); + return id == null ? NONE_OFFSET : id.offset(); + } + + @Override + public MutationId max(long logId) + { + return ids.getOrDefault(logId, CoordinatorLogBoundariesBuilder.NONE); + } + + @Override + public int size() + { + return ids.size(); + } + + @Override + public Iterator iterator() + { + return Iterators.unmodifiableIterator(ids.keySet().iterator()); + } + }; + } +} diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java new file mode 100644 index 0000000000..b96e9233c1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.Iterator; + +import javax.annotation.concurrent.ThreadSafe; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.ShortMutationId; +import org.jctools.maps.NonBlockingHashMapLong; + +/** + * A replica can only receive writes from another replica it shares ranges with, and tracked writes are executed by + * coordinators, so this should contain up to (2*RF - 1) keys. + * Consider wrapping value in AtomicReference to avoid false sharing, see: https://trishagee.com/2011/07/22/dissecting_the_disruptor_why_its_so_fast_part_two__magic_cache_line_padding/ + */ +@ThreadSafe +class CoordinatorLogBoundariesMap extends NonBlockingHashMapLong implements MutableCoordinatorLogBoundaries +{ + private static final MutationId NONE = MutationId.none(); + private static final int NONE_OFFSET = NONE.offset(); + + protected CoordinatorLogBoundariesMap(int size) + { + super(size); + } + + protected CoordinatorLogBoundariesMap() + { + super(); + } + + public void add(MutationId mutationId) + { + long logId = mutationId.logId(); + merge(logId, mutationId, (existing, updating) -> { + if (ShortMutationId.comparator.compare(existing, updating) < 0) + return updating; + return existing; + }); + } + + @Override + public int maxOffset(long logId) + { + MutationId id = get(logId); + return id == null ? NONE_OFFSET : id.offset(); + } + + @Override + public MutationId max(long logId) + { + return getOrDefault(logId, NONE); + } + + @Override + public Iterator iterator() + { + return Iterators.unmodifiableIterator(keySet().iterator()); + } +} diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index ff9a52fefe..532b534aa5 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -583,7 +583,7 @@ public class Keyspace } } - cfs.getWriteHandler().write(upd, ctx, updateIndexes); + cfs.getWriteHandler().write(mutation.id(), upd, ctx, updateIndexes); if (requiresViewUpdate) baseComplete.set(currentTimeMillis()); @@ -628,7 +628,7 @@ public class Keyspace continue; } - cfs.getWriteHandler().write(upd, ctx, true); + cfs.getWriteHandler().write(mutation.id(), upd, ctx, true); } } diff --git a/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java b/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java new file mode 100644 index 0000000000..ffd7f0e948 --- /dev/null +++ b/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import org.apache.cassandra.replication.MutationId; + +public interface MutableCoordinatorLogBoundaries extends CoordinatorLogBoundaries +{ + void add(MutationId mutationId); + + default void addAll(CoordinatorLogBoundaries from) + { + for (long logId : from) + { + MutationId max = from.max(logId); + if (!max.isNone()) + add(max); + } + } + + static MutableCoordinatorLogBoundaries create() + { + return new CoordinatorLogBoundariesMap(); + } + + static MutableCoordinatorLogBoundaries create(int size) + { + return new CoordinatorLogBoundariesMap(size); + } +} diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java index c6361d6c72..1d33e0762a 100644 --- a/src/java/org/apache/cassandra/db/SSTableImporter.java +++ b/src/java/org/apache/cassandra/db/SSTableImporter.java @@ -45,6 +45,7 @@ import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.OutputHandler; @@ -80,6 +81,11 @@ public class SSTableImporter UUID importID = UUID.randomUUID(); logger.info("[{}] Loading new SSTables for {}/{}: {}", importID, cfs.getKeyspaceName(), cfs.getTableName(), options); + // This will be supported in the future + TableMetadata metadata = cfs.metadata(); + if (metadata.replicationType() != null && metadata.replicationType().isTracked()) + throw new IllegalStateException("Can't import into tables with mutation tracking enabled"); + List> listers = getSSTableListers(options.srcPaths); Set currentDescriptors = new HashSet<>(); @@ -110,7 +116,7 @@ public class SSTableImporter { IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, cfs.getPartitioner(), - cfs.metadata().comparator); + metadata.comparator); String keyspace = cfs.getKeyspaceName(); String table = cfs.getTableName(); diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/src/java/org/apache/cassandra/db/TableWriteHandler.java index 7aa709fe52..4cb4d0e7ba 100644 --- a/src/java/org/apache/cassandra/db/TableWriteHandler.java +++ b/src/java/org/apache/cassandra/db/TableWriteHandler.java @@ -19,8 +19,9 @@ package org.apache.cassandra.db; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.replication.MutationId; public interface TableWriteHandler { - void write(PartitionUpdate update, WriteContext context, boolean updateIndexes); + void write(MutationId mutationId, PartitionUpdate update, WriteContext context, boolean updateIndexes); } diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index b93b5bf66b..7396f39b11 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -560,6 +560,7 @@ public abstract class AbstractCompactionStrategy long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -571,6 +572,7 @@ public abstract class AbstractCompactionStrategy repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, cfs.metadata, commitLogPositions, sstableLevel, diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java index 4bc7146f83..e4a5427f42 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java @@ -28,6 +28,7 @@ import java.util.function.Supplier; import com.google.common.base.Preconditions; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -196,6 +197,7 @@ public abstract class AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index ea370b4601..9104f9fef2 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -70,6 +70,8 @@ import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.WrappedExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -1650,7 +1652,8 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, nextTimeUUID(), active, null)) { StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, txn)); + // TODO(aratnofsky): filter coordinatorLogBoundaries to exclude any CoordinatorLogIds we're no longer responsible for, after ownership change + writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, txn)); long lastBytesScanned = 0; while (ci.hasNext()) @@ -1815,6 +1818,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableReader sstable, LifecycleTransaction txn) { @@ -1826,6 +1830,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) .setTransientSSTable(isTransient) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel())) .setSerializationHeader(sstable.header) @@ -1846,14 +1851,16 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan { FileUtils.createDirectory(compactionFileLocation); int minLevel = Integer.MAX_VALUE; - // if all sstables have the same level, we can compact them together without creating overlap during anticompaction - // note that we only anticompact from unrepaired sstables, which is not leveled, but we still keep original level - // after first migration to be able to drop the sstables back in their original place in the repaired sstable manifest + CoordinatorLogBoundariesBuilder boundaries = new CoordinatorLogBoundariesBuilder(); for (SSTableReader sstable : sstables) { + boundaries.addAll(sstable.getCoordinatorLogBoundaries()); + + // if all sstables have the same level, we can compact them together without creating overlap during anticompaction + // note that we only anticompact from unrepaired sstables, which is not leveled, but we still keep original level + // after first migration to be able to drop the sstables back in their original place in the repaired sstable manifest if (minLevel == Integer.MAX_VALUE) minLevel = sstable.getSSTableLevel(); - if (minLevel != sstable.getSSTableLevel()) { minLevel = 0; @@ -1866,6 +1873,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan .setKeyCount(expectedBloomFilterSize) .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(boundaries.build()) .setTransientSSTable(isTransient) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator).sstableLevel(minLevel)) diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java index af5c749929..53285c0f8a 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -226,6 +227,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -251,6 +253,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index d73a3ee6e7..58097fcc7d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; import org.apache.cassandra.db.SerializationHeader; @@ -1411,6 +1412,7 @@ public class CompactionStrategyManager implements INotificationConsumer long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -1427,6 +1429,7 @@ public class CompactionStrategyManager implements INotificationConsumer repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 7336c4543a..e3c25b4433 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.WriteContext; @@ -458,6 +460,13 @@ public class CompactionTask extends AbstractCompactionTask return isTransient; } + public static CoordinatorLogBoundaries getCoordinatorLogBoundaries(Set sstables) + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (SSTableReader sstable : sstables) + builder.addAll(sstable.getCoordinatorLogBoundaries()); + return builder.build(); + } /* * Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java index 45e8753394..6716452c95 100644 --- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -246,6 +247,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -263,6 +265,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java index f548e98a6a..16977ff49a 100644 --- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java @@ -300,6 +300,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -317,6 +318,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, header, indexGroups, diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java index 22913a84f6..26664da355 100644 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@ -80,6 +80,7 @@ public class Upgrader .setRepairedAt(metadata.repairedAt) .setPendingRepair(metadata.pendingRepair) .setTransientSSTable(metadata.isTransient) + .setCoordinatorLogBoundaries(metadata.coordinatorLogBoundaries) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(sstableMetadataCollector) .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable))) diff --git a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java index b0d3f6ec9c..60d9deedc0 100644 --- a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -61,6 +62,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter private final long repairedAt; private final TimeUUID pendingRepair; private final boolean isTransient; + private final CoordinatorLogBoundaries coordinatorLogBoundaries; private final IntervalSet commitLogPositions; private final SerializationHeader header; private final Collection indexGroups; @@ -75,6 +77,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, SerializationHeader header, Collection indexGroups, @@ -87,6 +90,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.commitLogPositions = commitLogPositions; this.header = header; this.indexGroups = indexGroups; @@ -112,6 +116,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter .setKeyCount(forSplittingKeysBy(boundaries.count())) .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTransientSSTable(isTransient) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(metadataCollector) diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java index 2369b3cbb7..03ff8f35d6 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -61,6 +62,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa protected final long minRepairedAt; protected final TimeUUID pendingRepair; protected final boolean isTransient; + protected final CoordinatorLogBoundaries coordinatorLogBoundaries; protected final SSTableRewriter sstableWriter; protected final ILifecycleTransaction txn; @@ -96,6 +98,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables); pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables); isTransient = CompactionTask.getIsTransient(nonExpiredSSTables); + coordinatorLogBoundaries = CompactionTask.getCoordinatorLogBoundaries(nonExpiredSSTables); DiskBoundaries db = cfs.getDiskBoundaries(); diskBoundaries = db.positions; locations = db.directories; @@ -328,6 +331,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa .setTransientSSTable(isTransient) .setRepairedAt(minRepairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .addDefaultComponents(cfs.indexManager.listIndexGroups()) .setCompressionDictionaryManager(cfs.compressionDictionaryManager()); diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java index b439438764..bf42ebea0c 100644 --- a/src/java/org/apache/cassandra/db/memtable/Flushing.java +++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java @@ -244,6 +244,7 @@ public class Flushing ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, + flushSet.coordinatorLogBoundaries(), new IntervalSet<>(flushSet.commitLogLowerBound(), flushSet.commitLogUpperBound()), new SerializationHeader(true, diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java index ce9992b91e..ae7bfdb123 100644 --- a/src/java/org/apache/cassandra/db/memtable/Memtable.java +++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java @@ -26,6 +26,7 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.cassandra.db.CellSourceIdentifier; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -37,6 +38,7 @@ import org.apache.cassandra.db.rows.UnfilteredSource; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.FBUtilities; @@ -183,9 +185,9 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo // Main write and read operations - default long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + default long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) { - return put(update, indexer, opGroup, false); + return put(mutationId, update, indexer, opGroup, false); } /** @@ -202,7 +204,7 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition * and those in {@code update}. See CASSANDRA-7979. */ - long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing); + long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing); // Read operations are provided by the UnfilteredSource interface. @@ -334,6 +336,9 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo /** Statistics required for writing an sstable efficiently */ EncodingStats encodingStats(); + /** The boundaries in coordinator logs for all included tracked mutations */ + CoordinatorLogBoundaries coordinatorLogBoundaries(); + default TableMetadata metadata() { return memtable().metadata(); diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java index 68ac5dfa0c..bf342d96b9 100644 --- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java @@ -32,8 +32,11 @@ import org.github.jamm.Unmetered; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.Slices; @@ -55,6 +58,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -139,11 +143,11 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable * * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; - return shard.put(key, update, indexer, opGroup, assumeMissing); + return shard.put(mutationId, key, update, indexer, opGroup, assumeMissing); } /** @@ -287,7 +291,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable int keyCount = 0; TableMetadata currentTableMetadata = metadata(); - for (Iterator it = getPartitionIterator(from, true, to,false); it.hasNext();) + for (Iterator it = getPartitionIterator(from, true, to, false); it.hasNext(); ) { AtomicBTreePartition en = it.next(); keySize += en.partitionKey().getKey().remaining(); @@ -295,7 +299,15 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable } long partitionKeySize = keySize; int partitionCount = keyCount; - Iterator toFlush = getPartitionIterator(from, true, to,false); + Iterator toFlush = getPartitionIterator(from, true, to, false); + + CoordinatorLogBoundaries flushableBoundaries; + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MemtableShard shard : shards) + builder.addAll(shard.coordinatorLogBoundaries); + flushableBoundaries = builder.build(); + } return new AbstractFlushablePartitionSet() { @@ -336,6 +348,12 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } @@ -361,6 +379,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable private final ColumnsCollector columnsCollector; private final StatsCollector statsCollector; + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); @Unmetered // total pool size should not be included in memtable's deep size private final MemtableAllocator allocator; @@ -376,7 +395,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable this.metadata = metadata; } - public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { Cloner cloner = allocator.cloner(opGroup); AtomicBTreePartition previous = assumeMissing ? null : partitions.get(key); @@ -405,6 +424,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable liveDataSize.addAndGet(initialSize + updater.dataSize); columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); currentOperations.addAndGet(update.operationCount()); return updater.colUpdateTimeDelta; } @@ -514,13 +534,13 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable * * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; synchronized (shard) { - return shard.put(key, update, indexer, opGroup, assumeMissing); + return shard.put(mutationId, key, update, indexer, opGroup, assumeMissing); } } diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java index 1389b1954b..ea138f6528 100644 --- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java @@ -32,8 +32,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -54,6 +57,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ObjectSizes; @@ -86,6 +90,8 @@ public class SkipListMemtable extends AbstractAllocatorMemtable // actually only store DecoratedKey. private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); + private final AtomicLong liveDataSize = new AtomicLong(0); protected SkipListMemtable(AtomicReference commitLogLowerBound, TableMetadataRef metadataRef, Owner owner) @@ -115,7 +121,7 @@ public class SkipListMemtable extends AbstractAllocatorMemtable * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { long initialSize = 0; Cloner cloner = allocator.cloner(opGroup); @@ -143,6 +149,7 @@ public class SkipListMemtable extends AbstractAllocatorMemtable liveDataSize.addAndGet(initialSize + updater.dataSize); columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); currentOperations.addAndGet(update.operationCount()); return updater.colUpdateTimeDelta; } @@ -288,6 +295,9 @@ public class SkipListMemtable extends AbstractAllocatorMemtable } final long partitionKeysSize = keysSize; final long partitionCount = keyCount; + CoordinatorLogBoundaries flushableBoundaries = new CoordinatorLogBoundariesBuilder() + .addAll(coordinatorLogBoundaries) + .build(); return new AbstractFlushablePartitionSet() { @@ -334,6 +344,12 @@ public class SkipListMemtable extends AbstractAllocatorMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java index 29ab18ee6f..ccf10c9bed 100644 --- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java @@ -38,9 +38,12 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.Slices; @@ -69,6 +72,7 @@ import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.metrics.TrieMemtableMetricsView; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.Clock; @@ -183,13 +187,13 @@ public class TrieMemtable extends AbstractShardedMemtable * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { try { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; - long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup); + long colUpdateTimeDelta = shard.put(mutationId, key, update, indexer, opGroup); if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true)) { @@ -447,6 +451,14 @@ public class TrieMemtable extends AbstractShardedMemtable partitionKeySize = keySize; partitionCount = keyCount; + CoordinatorLogBoundaries flushableBoundaries; + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MemtableShard shard : shards) + builder.addAll(shard.coordinatorLogBoundaries); + flushableBoundaries = builder.build(); + } + return new AbstractFlushablePartitionSet() { private final TableMetadata tableMetadata = TrieMemtable.this.metadata(); @@ -489,6 +501,12 @@ public class TrieMemtable extends AbstractShardedMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } @@ -530,6 +548,7 @@ public class TrieMemtable extends AbstractShardedMemtable private final ColumnsCollector columnsCollector; private final StatsCollector statsCollector; + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); @Unmetered // total pool size should not be included in memtable's deep size private final MemtableAllocator allocator; @@ -547,7 +566,7 @@ public class TrieMemtable extends AbstractShardedMemtable this.metrics = metrics; } - public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException + public long put(MutationId mutationId, DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException { BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, allocator.cloner(opGroup), opGroup, indexer); boolean locked = writeLock.tryLock(); @@ -586,6 +605,7 @@ public class TrieMemtable extends AbstractShardedMemtable columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); } } finally diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java index 7482ab0ff2..5f989c481d 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java @@ -74,7 +74,7 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance)) { TrackedDataInputPlus in = new TrackedDataInputPlus(cis); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, inputVersion.format); deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()), session, writer); String filename = writer.getFilename(); String sectionName = filename + '-' + fileSeqNum; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index f4f79a780c..91047eaa0e 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -151,6 +152,12 @@ public class CassandraOutgoingFile implements OutgoingStream return ref.get().getPendingRepair(); } + @Override + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return ref.get().getCoordinatorLogBoundaries(); + } + @Override public void write(StreamSession session, StreamingDataOutputPlus out, int version) throws IOException { diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index 9a6836d34a..53f25b5a2b 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Directories; @@ -87,6 +88,7 @@ public class CassandraStreamReader implements IStreamReader protected final Version inputVersion; protected final long repairedAt; protected final TimeUUID pendingRepair; + protected final CoordinatorLogBoundaries coordinatorLogBoundaries; protected final int sstableLevel; protected final SerializationHeader.Component header; protected final int fileSeqNum; @@ -106,6 +108,7 @@ public class CassandraStreamReader implements IStreamReader this.inputVersion = streamHeader.version; this.repairedAt = header.repairedAt; this.pendingRepair = header.pendingRepair; + this.coordinatorLogBoundaries = header.coordinatorLogBoundaries; this.sstableLevel = streamHeader.sstableLevel; this.header = streamHeader.serializationHeader; this.fileSeqNum = header.sequenceNumber; @@ -135,7 +138,7 @@ public class CassandraStreamReader implements IStreamReader try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version)) { TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, inputVersion.format); deserializer = getDeserializer(cfs.metadata(), in, inputVersion, session, writer); String sequenceName = writer.getFilename() + '-' + fileSeqNum; long lastBytesRead = 0; @@ -176,7 +179,7 @@ public class CassandraStreamReader implements IStreamReader { return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader } - protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat format) throws IOException + protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format) throws IOException { Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize); if (localDir == null) @@ -185,7 +188,7 @@ public class CassandraStreamReader implements IStreamReader StreamReceiver streamReceiver = session.getAggregator(tableId); Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver); ILifecycleTransaction txn = createTxn(); - RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); + RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, coordinatorLogBoundaries, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); return new SSTableTxnSingleStreamWriter(txn, writer); } diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java index 895015bc93..373ce67d18 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java @@ -76,6 +76,7 @@ import org.apache.cassandra.index.internal.keys.KeysSearcher; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -544,7 +545,7 @@ public abstract class CassandraIndex implements Index cell)); Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info); PartitionUpdate upd = partitionUpdate(valueKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.trace("Inserted entry into index for value {}", valueKey); } @@ -590,7 +591,7 @@ public abstract class CassandraIndex implements Index { Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion)); PartitionUpdate upd = partitionUpdate(indexKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.trace("Removed index entry for value {}", indexKey); } diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index 2487e66fd2..f491539387 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -30,7 +30,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Stream; +import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.SerializationHeader; @@ -69,6 +71,9 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable indexGroups = new ArrayList<>(); } + @VisibleForTesting + public abstract long bytesWritten(); + protected void setSSTableFormatType(SSTableFormat type) { this.format = type; @@ -117,10 +122,15 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable protected SSTableTxnWriter createWriter(SSTable.Owner owner) throws IOException { + // This will prevent cassandra-analytics from producing SSTables for tables with mutation tracking enabled. + // We'll eventually support this with coordinated nodetool import + if (metadata.get().keyspaceReplicationType.isTracked()) + throw new IllegalStateException("Can't create writer for table with mutation tracking enabled"); + SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS); if (makeRangeAware) - return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, header); + return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, format, header); SSTable.Owner effectiveOwner; @@ -142,6 +152,7 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, + CoordinatorLogBoundaries.NONE, header, indexGroups, effectiveOwner); diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 552da199ca..5134f2e6b6 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; @@ -161,6 +162,12 @@ public class CQLSSTableWriter implements Closeable .collect(Collectors.toList()); } + @VisibleForTesting + public long bytesWritten() + { + return writer.bytesWritten(); + } + /** * Returns a new builder for a CQLSSTableWriter. * diff --git a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java index 1db8ef679d..b5c0266748 100644 --- a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -45,6 +46,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final long repairedAt; private final TimeUUID pendingRepair; private final boolean isTransient; + private final CoordinatorLogBoundaries coordinatorLogBoundaries; private final SSTableFormat format; private final SerializationHeader header; private final ILifecycleTransaction txn; @@ -53,7 +55,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final List finishedWriters = new ArrayList<>(); private SSTableMultiWriter currentWriter = null; - public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException + public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException { DiskBoundaries db = cfs.getDiskBoundaries(); directories = db.directories; @@ -63,6 +65,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.format = format; this.txn = txn; this.header = header; @@ -74,7 +77,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter throw new IOException(String.format("Insufficient disk space to store %s", FBUtilities.prettyPrintMemory(totalSize))); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, sstableLevel, header, txn); } } @@ -96,7 +99,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter finishedWriters.add(currentWriter); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, sstableLevel, header, txn); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java index 8c9b2f9796..7a15e779d5 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java @@ -61,6 +61,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter private Buffer buffer = new Buffer(); private final long maxSStableSizeInBytes; private long currentSize; + private long syncedSize; // Used to compute the row serialized size private final SerializationHeader header; @@ -100,6 +101,12 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter return previous; } + @Override + public long bytesWritten() + { + return currentSize + syncedSize; + } + private void countRow(Row row) { // Note that the accounting of a row is a bit inaccurate (it doesn't take some of the file format optimization into account) @@ -164,6 +171,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter put(buffer); buffer = new Buffer(); + syncedSize += currentSize; currentSize = 0; } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java index 7b5fb8a89e..0420190eb2 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java @@ -52,6 +52,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter protected PartitionUpdate.Builder update; private SSTableTxnWriter writer; + private long flushedBytes; /** * Create a SSTable writer for sorted input data. @@ -160,6 +161,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter if (writer == null) return; + flushedBytes += writer.getOnDiskBytesWritten(); Collection finished = writer.finish(shouldOpenSSTables()); notifySSTableProduced(finished); } @@ -175,4 +177,10 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter { getOrCreateWriter().append(update.unfilteredIterator()); } + + @Override + public long bytesWritten() + { + return flushedBytes + (writer != null ? writer.getOnDiskBytesWritten() : 0); + } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java index 3b43dcfdf4..cdd7e3fbe3 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Collection; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; @@ -76,6 +77,11 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem return writer.getOnDiskBytesWritten(); } + public long getBytesWritten() + { + return writer.getBytesWritten(); + } + protected Throwable doCommit(Throwable accumulate) { return writer.commit(txn.commit(accumulate)); @@ -108,10 +114,10 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem } @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup - public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header) + public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header) { LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, header, txn); + SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, header, txn); return new SSTableTxnWriter(txn, writer); } @@ -120,6 +126,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat type, SerializationHeader header) { @@ -129,7 +136,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem SSTableMultiWriter writer; try { - writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, 0, 0, txn, header); + writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, type, 0, 0, txn, header); } catch (IOException e) { @@ -148,13 +155,14 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, Collection indexGroups, SSTable.Owner owner) { // if the column family store does not exist, we create a new default SSTableMultiWriter to use: LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, 0, header, indexGroups, txn, owner); + SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, metadata, null, 0, header, indexGroups, txn, owner); return new SSTableTxnWriter(txn, writer); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java index 482b4cf071..38c1e9aae6 100644 --- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.util.Collection; import java.util.Collections; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -117,6 +118,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, TableMetadataRef metadata, IntervalSet commitLogPositions, int sstableLevel, @@ -136,6 +138,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) .setTransientSSTable(isTransient) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTableMetadataRef(metadata) .setMetadataCollector(metadataCollector) .setSerializationHeader(header) diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index ab3171200f..6de1abc839 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -58,6 +58,7 @@ import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.SerializationHeader; @@ -1229,6 +1230,11 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return sstableMetadata.pendingRepair; } + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return sstableMetadata.coordinatorLogBoundaries; + } + public long getRepairedAt() { return sstableMetadata.repairedAt; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 03865c6ff2..2a1e9cb05c 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -38,6 +38,7 @@ import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compression.CompressionDictionaryManager; @@ -77,6 +78,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional protected long repairedAt; protected TimeUUID pendingRepair; protected boolean isTransient; + protected CoordinatorLogBoundaries coordinatorLogBoundaries; protected long maxDataAge = -1; protected final long keyCount; public final MetadataCollector metadataCollector; @@ -100,11 +102,13 @@ public abstract class SSTableWriter extends SSTable implements Transactional checkNotNull(builder.getIndexGroups()); checkNotNull(builder.getMetadataCollector()); checkNotNull(builder.getSerializationHeader()); + checkNotNull(builder.getCoordinatorLogBoundaries()); this.keyCount = builder.getKeyCount(); this.repairedAt = builder.getRepairedAt(); this.pendingRepair = builder.getPendingRepair(); this.isTransient = builder.isTransientSSTable(); + this.coordinatorLogBoundaries = builder.getCoordinatorLogBoundaries(); this.metadataCollector = builder.getMetadataCollector(); this.header = builder.getSerializationHeader(); this.mmappedRegionsCache = builder.getMmappedRegionsCache(); @@ -348,6 +352,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, header, first.retainable().getKey(), last.retainable().getKey()); @@ -454,6 +459,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional private List indexGroups; @Nullable private CompressionDictionaryManager compressionDictionaryManager; + private CoordinatorLogBoundaries coordinatorLogBoundaries; public B setMetadataCollector(MetadataCollector metadataCollector) { @@ -479,6 +485,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional return (B) this; } + public B setCoordinatorLogBoundaries(CoordinatorLogBoundaries coordinatorLogBoundaries) + { + this.coordinatorLogBoundaries = coordinatorLogBoundaries; + return (B) this; + } + public B setTransientSSTable(boolean transientSSTable) { this.transientSSTable = transientSSTable; @@ -569,6 +581,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional return transientSSTable; } + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return coordinatorLogBoundaries; + } + public SerializationHeader getSerializationHeader() { return serializationHeader; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java index 07fc965673..5b9ac4e551 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java @@ -190,7 +190,7 @@ public abstract class SortedTableScrubber imp Refs refs = Refs.ref(Collections.singleton(sstable))) { StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)); + writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, transaction)); scrubInternal(writer); @@ -240,7 +240,7 @@ public abstract class SortedTableScrubber imp // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt; SSTableReader newInOrderSstable; - try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)) + try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, transaction)) { for (Partition partition : outOfOrder) inOrderWriter.append(partition.unfilteredIterator()); diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index 2b214ab56d..208a1cfd62 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -96,6 +96,8 @@ public abstract class Version */ public abstract boolean hasTokenSpaceCoverage(); + public abstract boolean hasMutationTrackingMetadata(); + /** * Records in th stats if the sstable has any partition deletions. */ diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index e6b60c2a06..f3f2754a8b 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -434,7 +434,7 @@ public class BigFormat extends AbstractSSTableFormat= 0 ? MessagingService.VERSION_50 : MessagingService.VERSION_30; + correspondingMessagingVersion = version.compareTo("ob") >= 0 ? (version.compareTo("oa") > 0 ? MessagingService.VERSION_61 : MessagingService.VERSION_60) : MessagingService.VERSION_30; hasCommitLogLowerBound = version.compareTo("mb") >= 0; hasCommitLogIntervals = version.compareTo("mc") >= 0; @@ -503,6 +505,7 @@ public class BigFormat extends AbstractSSTableFormat= 0; hasUintDeletionTime = version.compareTo("oa") >= 0; hasTokenSpaceCoverage = version.compareTo("oa") >= 0; + hasMutationTrackingMetadata = version.compareTo("ob") >= 0; } @Override @@ -589,6 +592,12 @@ public class BigFormat extends AbstractSSTableFormat 0 ? MessagingService.VERSION_61 : MessagingService.VERSION_50; + hasMutationTrackingMetadata = version.compareTo("db") >= 0; } @Override @@ -389,6 +392,12 @@ public class BtiFormat extends AbstractSSTableFormat finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey) + public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey) { assert minClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || minClustering.kind().isStart(); assert maxClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || maxClustering.kind().isEnd(); @@ -481,6 +450,7 @@ public class MetadataCollector implements PartitionStatisticsCollector pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey)); components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality)); diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index 314694a611..6b9f59adf1 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.util.Collections; import java.util.EnumMap; import java.util.EnumSet; @@ -124,17 +125,10 @@ public class MetadataSerializer implements IMetadataSerializer logger.trace("Load metadata for {}", descriptor); File statsFile = descriptor.fileFor(Components.STATS); if (!statsFile.exists()) + throw new NoSuchFileException("Stats component of sstable " + descriptor + " is missing"); + try (RandomAccessReader r = RandomAccessReader.open(statsFile)) { - logger.trace("No sstable stats for {}", descriptor); - components = new EnumMap<>(MetadataType.class); - components.put(MetadataType.STATS, MetadataCollector.defaultStatsMetadata()); - } - else - { - try (RandomAccessReader r = RandomAccessReader.open(statsFile)) - { - components = deserialize(descriptor, r, types); - } + components = deserialize(descriptor, r, types); } return components; } diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index 3ec1a208c5..330efbd14e 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.BufferClusteringBound; import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Slice; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -79,6 +80,7 @@ public class StatsMetadata extends MetadataComponent public final UUID originatingHostId; public final TimeUUID pendingRepair; public final boolean isTransient; + public final CoordinatorLogBoundaries coordinatorLogBoundaries; // just holds the current encoding stats to avoid allocating - it is not serialized public final EncodingStats encodingStats; @@ -122,6 +124,7 @@ public class StatsMetadata extends MetadataComponent TimeUUID pendingRepair, boolean isTransient, boolean hasPartitionLevelDeletions, + CoordinatorLogBoundaries coordinatorLogBoundaries, ByteBuffer firstKey, ByteBuffer lastKey) { @@ -147,6 +150,7 @@ public class StatsMetadata extends MetadataComponent this.originatingHostId = originatingHostId; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL); this.hasPartitionLevelDeletions = hasPartitionLevelDeletions; this.firstKey = firstKey; @@ -207,6 +211,7 @@ public class StatsMetadata extends MetadataComponent pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } @@ -236,6 +241,7 @@ public class StatsMetadata extends MetadataComponent newPendingRepair, newIsTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } @@ -269,6 +275,7 @@ public class StatsMetadata extends MetadataComponent .append(originatingHostId, that.originatingHostId) .append(pendingRepair, that.pendingRepair) .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions) + .append(coordinatorLogBoundaries, that.coordinatorLogBoundaries) .append(firstKey, that.firstKey) .append(lastKey, that.lastKey) .build(); @@ -299,6 +306,7 @@ public class StatsMetadata extends MetadataComponent .append(originatingHostId) .append(pendingRepair) .append(hasPartitionLevelDeletions) + .append(coordinatorLogBoundaries) .append(firstKey) .append(lastKey) .build(); @@ -386,6 +394,9 @@ public class StatsMetadata extends MetadataComponent size += Double.BYTES; } + if (version.hasMutationTrackingMetadata()) + size += CoordinatorLogBoundaries.serializer.serializedSize(component.coordinatorLogBoundaries, version.correspondingMessagingVersion()); + return size; } @@ -509,6 +520,9 @@ public class StatsMetadata extends MetadataComponent { out.writeDouble(component.tokenSpaceCoverage); } + + if (version.hasMutationTrackingMetadata()) + CoordinatorLogBoundaries.serializer.serialize(component.coordinatorLogBoundaries, out, version.correspondingMessagingVersion()); } private void serializeImprovedMinMax(Version version, StatsMetadata component, DataOutputPlus out) throws IOException @@ -654,6 +668,10 @@ public class StatsMetadata extends MetadataComponent tokenSpaceCoverage = in.readDouble(); } + CoordinatorLogBoundaries coordinatorLogBoundaries = CoordinatorLogBoundaries.NONE; + if (version.hasMutationTrackingMetadata()) + coordinatorLogBoundaries = CoordinatorLogBoundaries.serializer.deserialize(in, version.correspondingMessagingVersion()); + return new StatsMetadata(partitionSizes, columnCounts, commitLogIntervals, @@ -677,6 +695,7 @@ public class StatsMetadata extends MetadataComponent pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLogId.java b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java index f7dd55cce7..1beac3fb79 100644 --- a/src/java/org/apache/cassandra/replication/CoordinatorLogId.java +++ b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.Serializable; import java.util.Comparator; +import com.google.common.annotations.VisibleForTesting; + public class CoordinatorLogId implements Serializable { private static final CoordinatorLogId NONE = new CoordinatorLogId(Integer.MIN_VALUE, Integer.MIN_VALUE); @@ -66,7 +68,8 @@ public class CoordinatorLogId implements Serializable return asLong(hostId, hostLogId); } - static long asLong(int hostId, int hostLogId) + @VisibleForTesting + public static long asLong(int hostId, int hostLogId) { return ((long) hostId << 32) | hostLogId; } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index a22ccfebb1..35da81443e 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -654,6 +654,9 @@ public class TableMetadata implements SchemaElement indexes.validate(this); + if (replicationType() != null && replicationType().isTracked() && params.memtable.factory().streamFromMemtable()) + except("Cannot use mutation tracking with persistent memtables"); + for (ColumnMetadata columnMetadata : columns()) { ColumnConstraints constraints = columnMetadata.getColumnConstraints(); diff --git a/src/java/org/apache/cassandra/streaming/OutgoingStream.java b/src/java/org/apache/cassandra/streaming/OutgoingStream.java index 77386a6225..b3fd1d5900 100644 --- a/src/java/org/apache/cassandra/streaming/OutgoingStream.java +++ b/src/java/org/apache/cassandra/streaming/OutgoingStream.java @@ -21,6 +21,7 @@ package org.apache.cassandra.streaming; import java.io.IOException; import java.util.List; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; @@ -48,6 +49,8 @@ public interface OutgoingStream long getRepairedAt(); TimeUUID getPendingRepair(); + CoordinatorLogBoundaries getCoordinatorLogBoundaries(); + String getName(); /** diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java index b83d7863fc..1f3c1b1ec9 100644 --- a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java @@ -74,7 +74,8 @@ public class OutgoingStreamMessage extends StreamMessage session.sessionIndex(), sequenceNumber, stream.getRepairedAt(), - stream.getPendingRepair()); + stream.getPendingRepair(), + stream.getCoordinatorLogBoundaries()); } public synchronized void serialize(StreamingDataOutputPlus out, int version, StreamSession session) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java index ec9d8e83a2..6ffa324e2e 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java @@ -21,6 +21,7 @@ import java.io.IOException; import com.google.common.base.Objects; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -46,6 +47,7 @@ public class StreamMessageHeader public final int sequenceNumber; public final long repairedAt; public final TimeUUID pendingRepair; + public final CoordinatorLogBoundaries coordinatorLogBoundaries; public final InetAddressAndPort sender; public StreamMessageHeader(TableId tableId, @@ -55,7 +57,8 @@ public class StreamMessageHeader int sessionIndex, int sequenceNumber, long repairedAt, - TimeUUID pendingRepair) + TimeUUID pendingRepair, + CoordinatorLogBoundaries coordinatorLogBoundaries) { this.tableId = tableId; this.sender = sender; @@ -65,6 +68,7 @@ public class StreamMessageHeader this.sequenceNumber = sequenceNumber; this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; } @Override @@ -119,6 +123,7 @@ public class StreamMessageHeader { header.pendingRepair.serialize(out); } + CoordinatorLogBoundaries.serializer.serialize(header.coordinatorLogBoundaries, out, version); } public StreamMessageHeader deserialize(DataInputPlus in, int version) throws IOException @@ -131,8 +136,9 @@ public class StreamMessageHeader int sequenceNumber = in.readInt(); long repairedAt = in.readLong(); TimeUUID pendingRepair = in.readBoolean() ? TimeUUID.deserialize(in) : null; + CoordinatorLogBoundaries coordinatorLogBoundaries = CoordinatorLogBoundaries.serializer.deserialize(in, version); - return new StreamMessageHeader(tableId, sender, planId, sendByFollower, sessionIndex, sequenceNumber, repairedAt, pendingRepair); + return new StreamMessageHeader(tableId, sender, planId, sendByFollower, sessionIndex, sequenceNumber, repairedAt, pendingRepair, coordinatorLogBoundaries); } public long serializedSize(StreamMessageHeader header, int version) @@ -146,6 +152,7 @@ public class StreamMessageHeader size += TypeSizes.sizeof(header.repairedAt); size += TypeSizes.sizeof(header.pendingRepair != null); size += header.pendingRepair != null ? TimeUUID.sizeInBytes() : 0; + size += CoordinatorLogBoundaries.serializer.serializedSize(header.coordinatorLogBoundaries, version); return size; } diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..ba7ce3f01ae2e46fae9639e780ebbc8df6f92876 GIT binary patch literal 207 zcmZSJ^@%cZ&d)66OI4? literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..ac900e5814d21443eb7dcf0d5e0387183117ca56 GIT binary patch literal 8496 zcmd5?dvufK9nO~~>4o7d%0=*O(T$<7L6YywtzMdln{Mo!QRO&f-PVSZmbTQUG;L{B zh+Hg)P;S~%(Ym*-gEi>qHLZPbQ z+`6VpU)6%T`m*_frpnUB$|`?dbwmB^rmDuOps&7q_5xpZU~Xyo{MwrOTHowBeqX)6 ztg&`pRb{QewxXe8L1S5YT~(#O++WsEUR&u4ls7lk1_FV(!3tk>ov(U!*_`_V{`=+x z>zjQQP4nx03j(!GHFJZ(>YC=Vc{IF;?N%cOwY9iq*)31b^cd?bVOwn;AE#-%p(fO_ z%(LW70l^WQhcml91zjGGv3|68!pPz?(k;xhblpavS#oev8^!88M$6V4!L&c2OEL-U zVPHB`rrCDgFm_~_Tji!jMZUuFmr7Oe)F#X7bn>q}jM(b95^lJAL#w%-e1vMU69 z1yKrK*oEAPHkC{oybuN`sZ=E5Ks!LY%0#+5fH_i$he80IDsfS0`7%#*YPpwS*T!W3 zfW6VF6@cAhasYzYXt`rLfvi%ZVXkLhqpUaIHZm+S<9RBx2`BEX5 zW6QpicW`mIR2+gh56dB%tGDHi%+Mwx*(PVn@+x+e7P){l6hx=pgfxg-^n|dh={0P0Soo+OY>o-E6& zSwF^${(l>V9Cm9O^(8Jx?|71pR|4=g7;1Z#dDK(8GS}Z@r z)8N~YArjfeNqTuX>%dMiDN^`bksfI~NgO55eWKX6Urf{-+Mx@LJH3Pf8?W+)OKFDzw`bLSf-6d2951;)TGK_}Q0 zcYGH0LmtM)8J198Gen!l-fj#Ob7xZgW@`<$*_suUFtVmOrX`8S6mcJr;aL$IPpsVS z@kBRx#|1qRn|L-i*)r7}^M01!lYaC$_ ztxAHoTei^}CM85go$Zzf$xgUIDTHm)J<)9=X!b0`PZ2$qo{PwZq1i+qX z@aYfm(UJJL>j*wNm5)8_p}8PVh(OD}h9`!l9QBq+TVh;yiMSf*9LoN5AXE19PVmkB}_&>V^JS6AMPXRhpiRXd2Qi<>U37|Wb_-Ddc37LG9SxxfmXz*mO##?$U%lFbbm=$o_ z=%&rNd_O+JXpSnze&+LY0WL<0u|LoEhbQ`IS7vAtb{Fa81MCI^{plBElb#N-n=zA) zY+8MFS>){32(;X2>#S+<7z2f5F=A**p4nQ0I0}*9;s*2nj;}SCPkyEtOeBYh>=eZ) z{__6L6n~)veliU@h@IJqX0Zg$lc7_Q3(|5?qJ=24oF`YJHSn9{03gqWF#sJZ6ANQ3 zFlQ?93mrgLf_O@~#*F#yD{Ht~+H1`2_{RA{pBZ9BZ0>u7J?vFffD@SXx*L6l?FM#IKE;kU}7DkByp% zxr+(F8yhu?fVlOL24g+=-X7P)GM!hqq`A-FH7sfFs9vhM&!oboG4~kSO*QwKhhgsa z-u#uyW*FGtTlSUBKr8Aq#oS{>8FDKL+aJS1zrdc%Y2z#UbR$o!-FB1E#&%+xTu@eY z(N5$*d|aWM3~hX06t@DER5DiPTI5GJiehETd+~Zu9Pi-Jdx23PRV8z&cU*(nnHts< z%A=gC9biG17bDo6O7x-qx8d#=S!Yfgx-8W=%XUljv^+9s4wJRC7IN>{vGjqTaygPO zURK!pr*SJeeScFEO)qYLX6HiB(PeI$Mb~|3k5r<76vCOrIrpa$AYjSkr7Jw(vMK5Eo`AMgdz{ z_$EMU9Um{Ep~3(#dqJ{)D7bhZKu0R^05E5qI9SMEArBv_JIFOkT;iQB(&!V#Kw=MP zh}%bOu68}G;~aW8;VW-5rs8%}9@^Lr`X*7D*)_vyd2TXr?kZ&Bkru^Ttimd`2|G&` zKvwRLViz_Rz0_Q3SOw{8=o-T>FB-P85g%DeJtprks{`m?8st|>C7S$w^?aaqO)Po} z0(8)jk{!{g*4+!#nMVCJKo_U3E97q~{n;Ol;PUVXV-Dr%{ux*2aZhG!por++APg}B z-dOmGyp86;22AG_ci)2BFCP7FK@K|EGg(7e%N;i;?y^1^3I*!M#4Sc8D*|*(L~=y6 zBi_c3&{N``vyu9v{67FylHF_^0nnkwL6fV2In^$(P&Qu#=t?DC0_IK<$8=KQx`Yd&0l7S9uge|I+Abfm|XVY z?hh#$qeV@_q+<6+$dIIR(c4q65*YWQ9?Mg&APao@yup+Dy*^LAH7%%}ln_?rU3i_KzADRhXk_KR};sAis-lW(E|alJDOmmv5(#e@RasPwIgV}hkl3Q zrlg@OE2R$)2uV}w8;;JUD^UKFVdB!+e-5Z!6N_;U0d%B|^Irg+aps508E$dJzKH9S zdWO3(E>HjG86uD9dX3Ho6WLAl*N+->oL?>6Cd#FsHQ+nU8#fF#Z8)tjW}s3CKF8lx z9I9V8k=?@goFDAJa6P0fUNrqIXqd?LSRA(~4O`J;fZtd|w2WTCCs_w#=rmzc1Ba1i z`*%!gPgD+H0qD>M(iBeg0Cj4*!JwW3=;G8_${B9q*yldu8l{}!Cd4H=@pOhrB)N|Z zv@pHcEG3ZWy;hnEyE0qZ7P4H1I}BV(heA@q0$H@FC^_r74EAH1Xl#q6ai7w#6&c=p z3@H+|y^#!qjX)(08`*5a5R_X}k7e+FP#iR(#4yqQg8-eHIE?)izaqIbeV{cTsO}{7 zW91MxZgv>X6N CAqQjt literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db new file mode 100644 index 0000000000000000000000000000000000000000..b1ffb7400f1256619a4a03b4287f055164a30d84 GIT binary patch literal 508 zcmZP)`NO37BY?@sjE$Ko_&rm`n;<4*b2etXFHE&>gbkS)CcR>2SpF)2*}%kti-U!k zVb2R@hKoS_>_q^xp{XSgNbL6`W=6KS05(G-V-pK2S4$NE4i-fRJ{AT>0|s8+1}0HP zCM`y&9U%-%nG6s+>=~Kr7{GS?2iox;$&Po*2T7)OcZPlP{A{K4|a8~h<+nETl2*WzAaqRPxWRa`k&WHhLX zxQYs{RBz@~dZR^^vd3b`H0up!gFcHhTTB*HS*gLSt*QDUt;6@Fg3Ciq{EdPZ78d3` z1Bm3{jRT~9=KksKHn!`NIf->{-0S)C^AmiP7XGjodmCV(ZEqVad>c^_BQcDtf3BaU zt_$Z`M%j(BoU#XH1!Zr_zLZx}UPC#Qas=gQ%1M;dDL+YhC*{4A8!2~F9>XTo`yg>t z+(P*bHtSD$KemXvzsNOCoYlm6xV}V#E!~VQqx$Q12gl`u*d8|Ql?Sku?_jTb8ar?l zdu=)Rg{I5`bu^5kk@uiC>F1b*LEd8ww*)3-Q*?r!Cykg!5S)s?{(OdU3 ziu?AhLS8))h`eSj8##0szwd}y{Ju5yXHosl)*q1{OUCVcDsU3Ti+@ofzns~L+=;A+m6n+*VO-WbIRovkOPz*Z5!{dFlV`qi1jfBY83WKh@nL8@`zvm} z4dX%W4cDL_h)M5)4*zC*5_Hsvu>(4>VBZmF-4*qA=~eHeLw-? z+OePdUj%-g zl&@TzO99R(zIkUq@as>6KR670Wk+Li^ACq-kkIV2(HvIGVokCR!XRa`E zyHt6(k0#{qDK`-Q15M2-wfdoGO-yWDd_tmdB|HOR90+sazr7wv7ndzzu@V++Vbc;e z?t3PAaph>YofrJsnedTK&HQithZbm%3^Jr7AT@91dh*Dpj{owBo0`N_&pPjPq0`dx gr|;RT475n&mWvW($YV z9DER(vkF3U^FwH!`4F1V6+-hrfzW~#5L%c8LW`_`&|*#yTKq1AmUMy8Qg0!&OfQ6% JGl9_ZHvq0G6UYDn literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..55c580d0b98f6579beb4ddcfd542ce4e755c23ae GIT binary patch literal 8451 zcmd5?ZE#f88Q#riHz9@|se~$iEVUVws=d4S-u;Nm?iz(UjFqh&Mgb|Xl1NB^gd`-J z^063@@KMY}N)WLE!Gr-yNPs|NCs}Mqtf*LG6;VKI3rJA`6{}Hi-{-x%dv}BXbcQ%% zX3n$Ex#yhsIq!MSdoQMd$)u&5%#KK|X>psw;gl8C<@R{HN*Qm{pEJ-(IMk3QI zLxmNytBT62XOzq+FASB0%0qKzgl3e^o?8+wsxFulsw$mTLyhX<@QkuqGfN&VudXa9 ze59(nXm+@`bb6?8b~rq{pm=&!;k24Lm2=9b%_yE#R8m|}SU9aD{K(vJsBBJ2LE)^T zqN&9t;qqyZR+Y>RO{*!Wtg0?33WZ8&dhO1UR!pjC^3o-DuK0`JS}wIY%d&W$w&wKN zk)=!gwWoaqw>$mJe9rHC;#t47a)^AyDq<@#Ey$9d*hrvVaWScp`kMV#dlQK2Q>Hi( z*mK~y44%nvm7c5v(Va?M4x+~(qVr1dys5lZgo!PAnDbiOI|J~%#!|nv!Y=R5`y@zl zcQ`Gzw|Gn>OKV&Thm)sXuPww18|U;7HQHoHI<2BXY_jy4d{(LU3`O4GE<2ww_lh5x z@e>sBHC7-rveO`w08c*%P%)^8*LYy6K?F7tn5+1P_(;Ge0dpID5yWQ$06nRl^MQE{ zB0Aqgn0Qp7=RGNUGI{}xrua*W!{Fh8O*hm2vbTLPLi<*i<;(lzQQslUA}zxE?ri^1 zrCo9U%%W+vVlVCOD_DT0pCb(Nwlo67b@aCuDbRE)P(>%N#&1}4ivX%7aS@g_}Qia>GgZJM@NpDUjEHIz^tNlf(Lae}4TmrJ5)&C=3C zoZ{+kWYNGN4lBpFJ9dA<;v~*52jxsTYvbGTem~Jhm2=Xx3h}gs{yc8h^TWU614Ktr z(kPWSQ`+#B{0}gdh+416wrC9qL;C@Mu2kaRfw>JLqIeLPCzbaqFt5SG13PvSC|(<` zRf<2;7#O_*{o6U*w-1el`yW1Qg~;O;ou^BC(DWvn|3uTn%^QL^zT`1e>-V!3F()A? z%M{WU;4oORuYX99dx$noVY>Zi5GLSZO1KH@#qT5-TBti z$?QNr-GOG;Eg2cjpAldS8f7bpKMM&L=;%oHpeEw_qWL8O5Mx_hp@Sjp4Af7oT!k=7_X0xU&~{YF+n)*XKn-3am}?17VO?rD~{JQi*5j>8dPAbfw@w7Yk;{89wJ**4bYQH ztO4dVhyL(i0tw{9 znenG0t-{v;RD6laL4<38sSSzYfn5*GW$+;MJPn}RAOf6Ee@OiVbEKX%^Gfo2FUgt) zZ4sYi^DX^DrFN(Hc5|cHiUaE+R&AxVh#K2k!C~x`BByae$c{BQzb*<}dLx1uIip)k zUZO8ukEEH?NC6Z&1{u=$ql;h1Q*}dWkrp3Dyp$SSd|Y!>NOgMwszy&lcIC4GU8ytN z2+W;ItX~SylS*6$%xe&_ycGef66A9G;lZPiLLSZgeF|^+l6!!5Hw(PF^am6;U>!ka zn43)%MLAY7hZGpzrlb_n(#%3uKyeyeU4hjV6pSJAI-`as71usUpop>!wMc8nBVOv< z#A}1JNS1vfy19&Q74+Zo4Ywc! za<$GrZP|n5*>D>`#o!`lCk$_w=Mu-9|6HD#ZDLn)U|s04d}=L8@z% z$P2>mn3ABg@oYQM*Lc{jcJ0xfr=1L?NBo=m=+3k4{l3n#w-FM^>oelBfbM&r0;p8* ze9;>sJPk~JI59r3XMnj39`?uc3jw-g#1Hi1UiwAh$E;Dj7Rf_#|7y%*o?p~IBx-kh zO4GF0#Cs%1H`Z5+rFf@M9Jhgo$(6+9454XYMhqx_i3rJ`wXVaSnO46I{Kz zl*b~gc8aLw7=+(OZwuIb46dGo-6${<=F>e3gDdsOEWIz8&tJLLRE?e(g-cBr%_c?! z#+9Z!l{XF#^ys_@UykXTeA87`6IdB8{JBufG98QwbHLM2gps{-%{}5I0?V5*50UDy zy`h_b6g4p3|4l#)I>pPD9vnj>ISG`?L{E-By_ccrD^7!}NB@8y;gOATkO>EeVW7D2 z9J~<}bsZ#0jDnif?lQU|`dpW~jS;Xz4smVjNhNZH>NSYe`5LNIgZzR%bK<1WA&pjc zSzgsEm);n%_{MPf+@-rBU=2}q7O$T#aO73}F5E1&z7%sE{rAmu<30QtM>`G#qK7X6 zP;{VKdh-Z16E5@SQ4VtKUS@w6tym@BVn2?SC*JWW7o&+v{slZ{PwZtqRr1|=(orfy zscQ5Otq8JXG;Oo|k~5(1db49@-um*T>%AO`w7GX-e``77t<12#X;4OuN!O9jcYuL8Z1bA1>U+7TD8RGb3;@-vna`eUT^p}%# z#-p;SNL<0pUAfZnEq&eP%oGa$5cz20%MEhsObCK0skg0ZAgZaivO@4&#G9*ElMfuW z6thq%)ug5kh@Dm+-`hWQYInMKl4WRDieVOQ0nURlSczv?nzlgHlecJB0=y5Pw-57v z0!8Un#2JLMtya1q>Bf@O*&G-uU7+fb6p&kufuQ{Htub#&qU$^50;QH=E*GdYB@NEe z{UA0?lr|Ty2C63Y#XN@sboH6%bpYKl=1Kh`)-mc={Q9vq^&*xLljp1c@(ohbMypuM=x6nhn7jSjS4JEkX^5Mhi19I}M z%%c0z{+x&ViA3rSQ?|H)eWw;{`7KA@^;0hYYehpkK1fpJ8(>H7KbfwGll2rhS;gj+ pM28=R*}Ia+0DlMX`fzAAecL3lwPp7Cerrh(|ERz`>g(fA{2wev>X6N CAqQjt literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db new file mode 100644 index 0000000000000000000000000000000000000000..b923a40bac2f0c27c4389cbe380bcfdbb7fb015a GIT binary patch literal 508 zcmZP)`NO37BY?@sjE$Ko_&rm`>mVj$b2etXFHE)1g$mP%~47#-r-L3>+_4+|U7 N-WHQ1l52pr007OIbyffX literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..4311795c8bcc87325a73ddf17f8b56b30971d7d0 GIT binary patch literal 7333 zcmeI$drVtZ7y$5lZXXl|6$)bmHdHnSBUq5~jBE%i0!{<+7^pLHq0I`8M@q9X+`^nJ zCd?o%V}RL1XF@huAOqb1w`sBo>QES2MCTyYR5V*$bpp~2skgKp-*@y6;$JS_lbqc9 z>vwK*&b{ZH`{gnWBVM;YK*HEdIDa5~E#WK6*EjhpVwhFFw{w1wF%cE|ZAz|^E7ob1 z#avYtSFX`>N{!B-Oxb17X6iLsy;hUO=?x`@l8SPzKB=ze`?OA69MEPNkP5hF2-b9~O9 zrOp$_(S@>%vKwW0%5utHlzk|#q`aE)8p>gmqbMg*PN%$u@;1u5DK}B>p*)67sOLdq zsknjiMQnBj<$c&9>i_R{6s^_rlIqWuS{tl{f+#KkzeI1d&-%y+s=fC*s5Qx*~9W{FfQ`1M)O0| z_mV4&i>qBXLQA%eq47#9N~&SprPnkA?W#_lf?gKbkJgh%fno&4Jw+J<(B5&uXg;ry zTUuZ|pyTbE(2qu^cSDDKlb;A3F{0~)PAJ@S3|f6XG9Nnocv%c|aq!tEp?BQ%n1pWl zv^@>FBd0Y8`pf3Jx1eV}98Co#p&mtj(8ZfJ!Szqtv(7o!zpWXu9+JcL&uUE*AA##x z`&1Y&p0CvlKZ$J<1{&K@@ zIDYxJUN_Q{nz;OCPH_`KHMNz14Q@$fuzFw6&HRJR7? zAf7b#Q~yrjXO488hW&ZboDc)&b9uJ9tr4`ZE9xCI0w23Ee!dO(;)(aWV1G@Q!bam3 z<3#cL&|C^|M(OmueZU8v4S8%B`1;nS((pyI2~SMc>VP|ZI-~Y1TBPbOJu$av#yM8J zr~f^tW5G4H)Bj5jXMcw>OcXVKv}f=!A%{aZNXX&P?O^_O4rl)xA%_!kI544v9PUBR z;YiD=Q<}|;Ea-Penz-e{cV{gN#~VZW{3n-%;(XLet*^ba`Hksvr{R%={`2>pB4rt; zuhMbbm3g^O#pmv-)Dr#!P2HKu$j75p(J`@c@d?72a1VreAgqP|_I@B;*tUetO4zW4 zmzMD2eqfRpXO4E)al#)R2_M;M`|!+vXn_XFAVbRhQuAhRC69b!zm`wjB1v@ZtV6cc paIgKGwH<9Wv}MZg&sFZ8B-X-X$r<62TQWqngWJC3=HA8{%U@%v#8?0T literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt new file mode 100644 index 0000000000..5bfa06ac54 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Partitions.db +Rows.db +Statistics.db +TOC.txt diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..867a774cb29e933ed74c0a92c5c411487d82d826 GIT binary patch literal 199 zcmZSJ^@%cZ&d)6Q)?9>LZvWqR_9Ka}V{-e!Md0NE}4W;T#hItke-hnxL@Z=^9J7_!_W{BAOR z_k4eUwN7(%l4Bb7M)=6Q-`tp|+MXw|Nz4LF*6(}ovj6~3ozWmJobgaMPlOeN_U+fTm zXJjMbzlAe1oQ;txh`5B%+>vycMxh3d3KeWKPx5e~P4>RD7jgc0JTXCERSOskYX%cR zD}8a0=v4kHJgHg;rT2HXjR?u*-(E9zNUb6Y%t8KDy?z-d!^AzY}h!U>Sy)}YdaI< z#anntmex2G^#z`g9b_sYv`&x0)Z)DfNve0S-62gb;`^bj{M}k7}CH6|*86NVN^Kf;yltz&Te<*5MELX%7ME)&Wt`bUt z+%6-;W#w(M)GjMw3j_x5JP(5_O37e@Ly1_hF!(79MpGcOyI}CKA~M+EU?V)_eHELL zp=AN_Ua#cx27M`zds^Y;iD+E5S2~s0gj~Y#eNXHjCw4vFLaOR|hd1c2$;4{-f^~}T zX#@7joNme9CKogZ16JDvxpx^>Z^4SElTYEflmox{XtL~ zai5fIv}a-^`X?bFd4E}(kX&q@Ik4$TnwoX}A7#NN2xsz@*uDBUF(E1I;ozpe4<(`8 zxknU4zpnUG$0QfI@ES>IeOUdNoYho)B-xJ0q|80KGV8h9P)AF;V&DC4#eW&)RVm_o z;f60i^z&TsbAu0Ia}*o7Sa!vbxu~f!bK&NQr8}A&Ji}CREPBzPOx58zgAKz{W#>gi;{YvkBrk)M@Og6v!N& z_!P*t$ppFV@DIk?ZXK&cQxH=K!S?5A+I)6p8ifqUQ?Zi*KFdr5Ta$-ikJjApwolAe zV*Om(8okTD0hz2{N}FUY9U3jhZm7wp+45n^OQ4<3)5apsc-!<5J2?M_GU)XCm(N3^*iHFw#UoXZ$rs{Ku9qa=fVeN48 zBHvVu3w@)vybp&-Q}GUmROa@o{?ibmcJ_|FLocqEe+?JiSqV_=RRg9ikxr49e}fb@ zIIuw*y$2sd{Jj^bq(~xpHoSW6S2xXk^U~VC&EWoE#gN?hUNE^=Lt91dE>;TRk79j? zO5yKR!qA~dacD$Yps$c7A{Gb+e~Q(dp)(MtLezqQy~DR*HMyg;S&%yJ=`R4? zp{x5SoAwt*+NB-BevLB3CQcvqY-Vc7GRBuMTAi96S^YLM?(FV~=?lh=+ z_XL*ozapZF4ue=Ij~0BIQ6)Eks?~3Ss^1q9Rn$R+KM}PCwx3u&tuwXy?H&?uUFhrQ zlH64uK*jF$QDai)VDq_h*Ha6r9E6>HZn8t;y8Ls%hT0x!6r?<$OZi*bN%vgJ-)s9= z%J8{t(ZjO6(UGdV*86!!qmTX8?D3<|l;W z>GcOj(2nA(ovJrU6INY4iYLR`BuzGwaqi|zl2M`GAzY&wT_N-vgt|eP#_r}#=`_%O z3sMoR6zyQFDg{Zl_8zoO|lul0}xg0G*sk(04iy4Hj(SQs1wgOL=-7Xl1=mU6XG zqA3u9UCz437P3>x*v`U|r0e-a#CDc_X|ILzuRekEtBQ@i6ZlxDW&5z4vnYs@yO79q zv_2#`0l$gWE8?qv5ZaV?O=~EKi*LbVULoH#r2G0_z=yXH*TAo72WVdT1fRhFeS1P( zJ5r(91rb-XKxoj}Kml#h^Ab-xcu8K2yPa!-&mlO88YSGt6Tt6Sv`D!IA^E8uscCgZ zZWy?1ftDR0IBEhgR!{S@L;O5uEBOwu9!_>f((&%PTJ@O*aHx-5PcrU}S?+bT1qxbE zKCg2}%$B*YR{bW3a~IEEjG&@@bbaA+Zbl%q%LVc5hv>8C25S9D9}Qh65^sgh%svXSE{oRn&shf~Z5^CoE)-^Nph*_E~M5 z#o%1i45zX88PGT^lVFEs#cx6BG|p-#v%_~flz=4w&fuI~a9EK;X{44!wIC7S^N3GZ zZ=!Y13p97kTDlvA^qBcNC@y900=G=LSHdayP38g`_xbXS;)vU^3eBO#78GpKK;PE0BdK-uAMz+I#Q738>6mVGP zNq!v;7l;>`1LyNbeXI8V3OtUan*y{wVuG>HPc0*5SE44yMst=!?Ppz1CN{B$GcID@ znPM_BI=jSOL>jJ^FK{cqsRMo>;qeCwsT+50oIO^YJ~lX_P>F1z0sXR|FN&Y4lagG~&-RC&t z`imVT9*nOt2;Vq+NWu?vnuG_U)Yoej>{!0xwsH@SI@LD=6Bsq` zS_MjG^LG&VEBlS+M?GR%0douK>X_--+|(^m^%pd&ehXaJn$6!O1H2m5fF%TO`+JBX z*?w=$gQs{6d0OivQ$ZgEXNhEMKFwA_78_W`1Pq2PUcvVEY%=9JQd#ei=-w7dfjHqc zN%!{A$=@pV0-v=cqSHO7z&|(GuKn+jYFzd=2v?PVP4dQ`WSv^+wqOF zrxJTPo10o?YQPk~1*3CU`xUK&>ohgy13=2k&IA_trg05ZEkHQy#gdXu_Rhb9p-?lC zX?V7gLZ8B5*n$IXyE>48w~|J^3DWy7_H z*^TrGu#Etb(vN~t2g|8p_Ps}=tez}h`sh@%z4Ob{5qtgZ!c{JI(@=}0oNme)su*$Ul(!m!}pDk@l*`_k-#@h zIAzp-{Kg^czi+Qu|NiK;;%W8;I+X@QRRAZB^gjKP@x1ql2Maq(w?yZtNNZ3O4w4~2=ZsA&#|ftMbxKI2r;#c_k(#XDwS~Nwk0di5NQiq zC$IAa88Ae8fdx#-=zJf!ONw;ohu|Z)AcSK1Y^IVF;{5tye*iQaurLUL(a)KPrAIoZM}+#^mLP0 z&M6ygc?FHJbpA$GJ%T)~;=CkvxTU;@_;h`6fm%K`&Jbb^H~)CceLH{R;R~;;#|9_~qU-v>X6N CAqQjt literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db new file mode 100644 index 0000000000000000000000000000000000000000..43d466285234459b9b112e2f5fe2a9865365ab0d GIT binary patch literal 508 zcmZP)Ze~(v4`4DfV`FCW<6}yk6vSj~&cK3gojIpg!Z)rQe?`Y!UUoPI0+}!)) zJH0vQ-t+i!VHidnx+PG;*h?gTBK%RoA1r^o!XF}rxrbeNDWQ&u(i?Uuxf-stT&pbQ zYHPXbG6Sb9D>o|B_87G}hBB=|Tb9cij1|TTeYMu0Zff{0vsdwa@x}2D{zidTt99Aa zpGXc~JwRF)?_C(^X8S*~NaidvUeAs!&+=6|`NLLhZGeTgt!=RIZA3+k#4yhOIsYtm zUO3J!lwB#iQ}&=NqwGytL3usp4U{)gj-VVvIfZf-<;N-SqP&-K8|4AY)7XT1A0&Z_ z8!4Z`X8kDd#}-ld7dgj?qnfx7*O$a%OSfUWQ2lkih2!pH*dAu=wFj`}Z(^@|5<6fD zdt(%K=uvEC5q9EX?6lL^xxU!N{@AsTVK;NwUDWs;)8M!{$YJ|DB0;~SI4LT?K8(jF zF@J^PY(AdP?C02`qAyW>u^5kk@rCm!F8Qiyk^-4SyqO}qqpWK z6juzbLtZ}{fV^Qk4|&rBe%}#G_?gbN z28;*xymlG-zPPM@=&-MMq(DbamiIy@7w_# zI}^Gm|4=aW7adKnLNC5Ql>tn`Jxhk5OViWg{3qSHr=0WOR*hI+m%;hZYE82rg5R@q zC$={Li;To{5Lmo@yj)&v9I#1Kp3hFnN435nKANh9XbQkcMV{iAt_L|N`OeL?>Z= z|C-aW;5W9*|4R;M-$NNDhU!0BGx(U0!=VQxnrQ3s%`n@LTCThKLN!(zZn1k literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..f30e3ceb791e51673e5b45c6240fe7591b1b8cd4 GIT binary patch literal 47 ocmZSJ^@%cZ&d)6z`>oCpRbUVSOmmXj0}v14Jv{x2PSk$FbEn+Fz9@fl3)-p cmSC_s*tP&DWFoftNp8;l z_4|77IrpB^b5cTxY}M*;Ik{6}#fpkYg?Mn{@v3<6glL~XrdZcN;;L<14ML4jZZR9m zh1y!7%48D^CX3yWvD0qOwVBK|vnfxo*(>Z7)m3I&T2uY^S>3u_rI$w9#S4YIT&@NG zP%7VdZ7*$|z2|ygC)ay=UOqiP8Svcsg()%Rpm?~8yA80gcDD@{v5lz6kO-0bCw;7t zPCT^_V!(Md++fa<1bO<}+6n0(^c4;Vf?UUHe0(J*8eus@X?$jew{qNYth1iGi_@vIS zQJgEl^O^e`n=k$n<;!Gv{L3z0Kymq}Ey#-BI+2wHE@a;YGxCy!KxDNEk4KI1CluEW z1S5w`g&{AW$VXl=ir;t49Dd&kt!GjG`sp8#A4|vWdpgX4;?`fZ$S>yhARoAyjNE@7 zKcC5d^gO95;0Da6X<6=d;BKLf5y$UBJfS1!;-R^ylj!%A+poR^<9t{h8Xx??v%WAc ztMlCeEiais{Z&*~)WNtq!}-sd9aA5I?{m{9Hq-<2 zcIwy%EL%CM&4Heq+YR53{EyZ)xPJJo4xC8``O5Zp>)!)b<-R>+1NPg}@+<7W`diHv z*niE%>^?2X4-%Sd;Cc{xJZ;bb;`+~KwD5hcyq1;I4C1jx&Pcc)NKCjoqz3V{i5~}d z06%`P=LBreW@kzw9M4tx#*S8y-&EE=JPmyKyU9}>z-Nvg>w)ccoQoMxT8vZeOBd!d zfOD)jZtVen<%y^VMu9Jvv{_>p&84iHX|w=$>AKf6ESgW~wH}>cG?Dr${ie%3G09RY zC!Kie|H~duYIcZp;;FrRIP^d`Fvi>g?;Z|49^&1@q37Y%|mY)7gclPIpUrr_@`k((yDl=3H gwpxp@%}`YMXma7s8nbvie!H;kKj(k^;Eut+0T4{7qW}N^ literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..0476db58d0b7149e5371d2fca91467d84e31324e GIT binary patch literal 47 ocmZSJ^@%cZ&d)6R literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..6bf3d6ff53fffd2951ae4297dc49b442e80ee8f7 GIT binary patch literal 132 zcmey&z`*d4lY!BoK|zCsfkB0Vv4J61(lzwkYsU)*7#JBQGBU84a!nT%zrn}wAE=g* zk%7^$K|za!;lP9rLk13oIV6#oV2GcXx42m(3w2b*>pGO%li d^-X3FG6AYM$q7`k;i?AEBoJU^6#llv4gmg=9v}b! literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..687d787888 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +3514184066 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-Qe4gseI;!9H5#KPTpTyRInX-G1Z z;Vy?<^SrXg=s$J7{9rSs|~QwcC`%_zKy7ekr+nkpKw?z z?6_+$%HEWHDa$C!DF;wiQeHuMCFM1gW8D_|kx0c;DQ8oDp7Ku0dnvb2?x#G7O{n`p zo}}Vd%IC4!<&^hfi>UL9gn8ml6BprpNfNenGqxAiU+;T3?mL1lb6`KVA6xMrcJPbX zp%d5<@!0E5V5^F;liRV=&tc~UVV8tpS3if{$YFO-^LIpxOgYv~xr(9|N?% zp}2A|7h-X^K$u} zBQPG;{mymh)f=+=prgOvo(dg5Zs>tdDXBXSt-Y489XkI+r5d^{>f@)Oci;7!fo}S| zD+{{2;AkZD*KG}lpl3gt$OI&!0No3)^eI7(0=?6eo(e zFVCj~=NNC_+XsB$+33f|fUoUnF~%*KP1!V4ZvgIA_9WIVS(MagJT<>$M(C^XPM5f2 zf`wF8*m2j7${J4ScA~K3uF*S#KjvA(p#k!&;m~j}o;4gA56>FTvxWl`;?Y{ek=9XT zY9u4;f1i=6ecOU$5fAF%Z)5nsTpko*Xf(DTBRUUlZ551a{tFEj?!tGQuifV-Y6qXs zXF-E&Go&&kvuO54+W2RS=OqkDARFrE+*f()i243~<%Qp$IvBw)>;L=Dq*7HSXQ?)D r6{@1bts4vXRO$K4iFR`pPyAnghIyCCs<&jAD|S_yTt8fOcXj)}kJPaV literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..5a82349dd2e22fcbf900172acbc7f84c8e531651 GIT binary patch literal 47 pcmZSJ^@%cZ&d)6}Gdo_GhxmOm-)`lPW%d*lT5{ zW!@lp0(-$=;}ZzlIw`*5T)uPQel`Gp9s#?v@V?`qJDZ6j&P_^1m1{nhCJo9|usoY+ z--uEaRA+ugGZ_cbSdFxze#oe1yi@~`DyEW2Qk6W)Mam+^Cy`32jHX6ODdQrcSxz&b zheOGRVWFoqF=I_Dsmv%YiflCH1NRaD_APabV?h^MH5YEzxz&a?(hh9-4LAsTaMW+w z>*ft~sUK{>L&*5LP}_lb1U$7*9XboMcZ|Q@!t4@kG|vd0tlkLzd8=2vt2x`}1ihuW cCRSOBtF>}rmHD$($oz?5zivnI<@@97ACAIZvj6}9 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..0c820c94b7 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +1960712659 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-QemcqW1`-GYO%22&v20|yTRDi=++7Bw0~yXZ z1EC^aiscuFTA)81<}?)u%E>bDlY`xQI2zRF&UIY>~a5eYLNFyOH1A+}y024=p}; zXD?kh_2}L9CPCY0Gvd*iVfR-q&W>;?N4VXxS;_!6(w4HpjVmMCL`W3H*N^|$fj_Yx z9SJ)Tb|LIaSVGu?u#E63!m9}f5)LICK{$$V65$sJ=MvsUxSnu3;UR3A?+&HTVoLDW&#Q9<+wnH?wBT-+c2RQE1i|smr{m5Qy>ATq8FJk)+ zVm}s!z4kb^A`4r46g%!LcB&V4why-HdF*NiyO9{bz416c;g3xF|BjuNfqfK@PkQ2e z6c?o9=UMO-wlM1(lrIwD@h`e^8O6n4)*w4fHX%Eu&mlX{vdAlDJ&`4PJRUvbe?f6s zmpAgN5ntrhLutr?{kXqFr*VHP>&~P6=+U2$pNYlg+v+=p;`zTykto4 zbaVRQ0O+q9s%oL9J|0W}rq{XUbV2JjZGz7~-IRI;KL7OG+Yzq(bF5t zfrVyz>>#jcO}{i5dU|>n>>u$Tb@g!ma18Mrj|KTo4IhtiR-% z$1PZYk1I*-QjqV(RF}Z{z~@v~yJ<@s_mgl93NHrYKMQQPkbs)bgr?Yny_}GubXB&afojBPF%R6=P%vv91(r0iY8Sua~&!{PQ@Uc;gGSYE>|ui?Oe zSYE^ZpKCb!P+xvj0Oi{L0p*}}Y48fHoj-bi7itLnu!DQ;viSc`%%$j?L0fOU&vl(; z{+*wLEji$Pk~_z9F*lQg$(H}$lADjM-%BT3-dAv0(4b5dUFegLIdwaB;BzjI+h~do zr&Sfx3tIf!Dwc)*B|Cp}!^zr7X`pupweH_vIOQk`8KcR-6euz?Hm}bpD`B}eJYu%g zY-H@dLb#JDsO-0O|BP2l3Rue**t~B#Yql#dxyY=Huw;hFZGE}Dx8xfkBt*njD&nE6 zmGTXDTg|oESX|B)+0_Y|Ce}EgSit5oy0YzNCZ~Wk$u&BSTBFrGsZnYp)jKpQO@w-f zR;5*I!z~f+T&+=QRdNl>rRX*44Yo>T|LEbn(B_!n6GIj{TU9w`s}K)EUH9+v`}S`@ CA^rgX literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..efba218e453bea0341c0b0660ad3093b637513b3 GIT binary patch literal 207 zcmZSJ^@%cZ&d)6hX3K!k z9I6nSa{`3sR)^3$+aWYx3WOG5htPt>5L%cYLW}H&&|+y2TKpe`mfQlNr4t~uEE9y5 JTLGaJd;sOl6LJ6m literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..fbc8ce33b263b7425c41d8bd947ba3d3cc871375 GIT binary patch literal 8568 zcmd5?ZFCgX8Qx8@NeJRC2%=HDR3j=PyE`+xJHaHoimfOjiwe|&#Uvzwkgp{lkQ^zi zkrJ#Z29M-WqlPIUtICQNR@9ai6qnKRI$XInENav2WlM8*toGTe9qq1Q2481tYf5Ek z*;3!KeR%|TxO|+s#h171exI%OLg9doX**gHYFg^+y0O%6tF?s)|$3+k6K8d=Tt=i@jbLCA->r1*@>Y`yKsF9ZDTDvk~RPl{Ex0D5quqj zYn63?rC9MB!kgC>oQPd7;gW5#>|%~u0Ra>58^p(NOCGR(Gu%;s2|(%b*kA-FbPS+7 zN1X{w0-Ih2=rJ=N-T=^RX0H4lK-tV}nL!|H5NYx!+0iuq%C$F;2oBL{X11O7+p6vD z$~DUIfDCb_PwTmoYonzLLD1%Ll7<4X>Y>C?rbBdjlPQbY2R7xDDnZ*h1VU&@Qckh- zB>GJ~_|F^yEXthy6#a0E#>tpVsZGj~1!&|_wH@*MM;nca7SDVv#Z~1sj zqm=+XX6E5V0KI1Bv3mi^X67e6J5P>QRxv;7qSbN2!K3ZVj79qoH{{uBSZ)l++cXBi z^*ZqMX#m*(P6tr4r1|gX)p0qvv2ZBE(xm)`G9~YiA0~zv?c=PEu=U*Ytd=~RqhoKL zt?D_e#p+UxaTDr}Vr4%`SwST(rR!AY1Y6Z?Zc3`vY^tOJ7ruA`1&my&B6UUdZK|?o zQeamdE*@BA%M@SuvN4cA=tt>OUu+3w_`*LM23T}jV=sW*S!aMs>-AL-YQNetrpDLd za>tR!{yCIE$G}W>IG@KWym4Hq1*jaym6!N!xdHCdkJ^^oA)Yes|L|*WW8A$|^KDTk z!@0up(jP>13W-B-sv&N5D8P(&m_d3n{H5XPQ`)xX1Tqjl5Z-@KcWHyrNpY z7}yr#vY4%YH@6di*tR1vROyghrQ{oXSZ|WjqsITY8xaVhm;kn)TxJh~`%n`=QRiaQ zk5mGbbS6Sp3(RdMHqQd+(TN!S1XVp!!OUF4>ziz5u0wTWrDXJb*-fcR4@PhJ(qJ-e zjzJrI=`l3XR8X3(+Hu)47;;5CKh}qy!DOm^qDtLs6O*lx>0d|cs$4_Bf8-ic%ps=b)7Ju=kRA zD*cN`PPBpIjSx7!oNZKjKwh_;zbB9IZhl zZTa_|3BT=^bT^Ay>|m-|6n`=5@ZlGhXilUu$l)V{c@-l=A>kCV^a%SE#Y6L-R$t_T z8xS5zYXKZ1ufk)@;jhC}%;7H?j>nk8|MUzVHHU9?T>c?F###L3WJNW0sWW@Anhq~q z+fw?ibim4zRgaI1e)Idb|KkbNE^Xz#=3v&H-e~dJeBoD0cx)xtXlo;Tf_KV;`MHja z=|CKWr?ml@DUIl4BjZ{Civ~K5=sE<+G6H+1=nIYDxR3EBcrc@(uqe+wpq{vq+@idO zS&$p!?*dgeP=V%+$1ESi?UnObPkJN38?P|<3^jYN1vI0%Kfs-&^McRc;@a>#B!|$o zexn&nSWXaSyj#fTl~lGfRcV5Oyl$s`*yU@IlqR+YA6rskHdkVESDa6eX))%W!J5^G zp$M}($LS;BHIZ9B`!27E6y2_R%^~s{n)bqEr3t$GBa;urnN4IKb;@)SO4DLXft=+R z0~K4-+=DqpFl1jq9{r1Pn1a*P>&xv>Da=0 zF16OPhk4qK0NhPi5qQsbo^>NwR!Th%!{k2Ad<=9oOS^GfiMh}20j|kOo_OkLvY*|j zfl7w4m}+k?P&ZbkC@PoGxB30XW8fmaam}%NIrYH+z9TzxZkj)a%M|;x?98y7(Nx;^ zbb_cP$m^_U+4)GHX6yr(W!9Jawb}rzcI;mT6-1|iAT+a$$!Br0d#UGv0X-QT3k|0B zXfC9aRS9|*BRYaU6W-BxFQ$s#hx&>^krqIg*fbyUOOjh3hZXWuqQipl|d37Sd zV?6!j7*p%a&h3BWG3fG?u`|Ur*bLj4gw3XbJ=zJjey;mI>_ zlb5=vSdQbaB#;%3Phu~}mc+B)R!uvzxBD`Gu3mnG=Z89ZHkmFz zbXcy}q`P}%Vo2BS`3J8Tx$LSzQ`Ml~{RE84`?+N*o5Ll|MYh~?AGfb4`?i#+K=TXW zUoFFFXG$LP*`_;+CTiv-g{&j&&M)=1z`VK#&H66$>K?U+pA1(f|HPBa==GLeS-$Qe z6XLw9d(=D7MUYbU)seDUytfe#YI~vKI1(>|DxK7qfIPc;4Nx~xZw~M;ofi!o)6ccy zpFH)ys>y2c>%Mw@Vo25D@*IFS%WPt@G96nP`vA7${nDyTXLC^#W$c=NIFRG0&iO=T zEf)|e8^v;#J*fJXVKB;^MBn}4SARFmL^=<-Me{2Iou8I&nfh$qujE{;NW2JjQClbZ zRn91&l3^^?IU1llZk_&gpdO9-f!gvl-7`b|LxH*F^JoJ3&rcA61aBYp3AyZc`gbY> zn!xQ_K7nUefZwojxHtPXwGep>EhR;Uj@8`Dz*ZHR z)Qu1dhROwDCsBs!{xyYvFOcHuu!ua(Z`t;iRiMKcQ!nangbF%)H{WA@FeeVsxQOlQT(ii{A>p3Iey z-ZzXVB}NR@?8`t`l_+!|NP=-bdQBl+TPa5jYy`AqBvzr`xY`^g{HueT78P45jhs); zL{b<=LZ4KCUgIS6x{BjK=%8-4a~sCO-teaWgz1fmwpKU8hVLD Y-bGp~`Sv3|TUh}ACxf;6+O94C1u8$UAOHXW literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..f0bd564de9 --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +2672104528 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-QeF2c3;# zpTcYGEFzM|Dui4}+&6LZF2yxo?tH)9PpU^Cm${vJ-}%1Zo1I_TI$wrQ&z$|@xwGZV zz4Pa1^Tqasi>7;= zwNMMSP%DO7!)JG=eZ|GLXiKyu+7fN`9c?vAjmjmOr(Beaa^)!3l{?eE;$o-4%!quE zFY?v5e65sg&pMHDF)qfHV_aA7Od;2{w+Vt5c!5_8ylTj`O}S7DwNNXDTEn&5Q^>XL zeS+WxUf>l2uTi-~^QeVds1-x4;YL#~FBAkX@B*(Gc-4JHPjZ14Xn|G?w1%6vm;2Au zmvZe{En*jTVOI>hDzvN;K?}4%D+XG_!QabU=gPxBPk&U(3xAaI^5s%q{k4?Wc1zj$ zzLeKrDdml?N_p!{Dertz%6os6^6}ecYqosqLtWU2hj@q=L%jB_pt>ok3)?qjd_$)1 zZ^*P6YqYbw>cT#!3p?<36^HRSQN7MhnK6JNP$S({uMTjT_j7UDy@F zuF<$`N}?8Op;ipFzI$X`y0AApF_4+b%(7-cH7;G)S?3I8P-e5Q*q^qR`&+uOV+qGT#|VimyAos6>|q4E|+OrQ5UvbnP*TvgW8m)_fb_YUD&9FTBy}KtJ?7m zA56-n3mdzz3%eQ{dLNC(r3+igCFBxv<%C?HmuX*77dGXhT$HPK(Q3yxDwj8?QZC9x zxpI{2%7bZNQ5QDlqFj`#DNXOAm2&CA1})G6t=>`9j&Hd7U<$c(VS^WVfmee=@1q)W z>B2@W)IzP^S=EkjxORUExpZNJ7kGhJgG2A5QMt66*;ll$XkRgBU-3p$E-w@WFYp4d z7(SOJ_R<@g^geMgsWe8!nQ!NK#Ex)J?ZJ6 z{qt(br3;&KQ7+1rLo!|1l#6mvuE;lAjmo79+rA=(V|Wb1bzz&XrfbY}%~{Kx#*x;2 zMNe`uE(nM47=#bHun`Nf5G#gQP35X^MTT?jS1?+~cK|r6&gxOS{m1IU-u~sK-*sUV zHo`{OGKEbSwz+HW2F~5hby{86gbTDmI}qCSDX`gv%|*B0To7|q5aZhD_1e&d?V9yf zAzu|rB=l%pHYKeXtr)EsV^)m1uo)NQVqB@lr3)Ls@C(0k_@xUQu@DQfa){NAv}Rm$ zUDz@%8JCPJCgakDZMvGSG1Il_hLJ98*Fdii>o_yc%u!K}F6^T|8MDT$ac%S*fYy*p z7q)A(=WEnm#9id^8LA6g$R*?waz#FYJ}Q?tsM=S=a14)OxGrqd)pU)SuGY+l*UT&B z(uMu_650iF_yX~1_%wQ(AG)v^7vo}Fu_2c(?A~rEG>7K6CXe3h)`d;EC>Q0*RW4oF z_7yQ4!($k(3)^%xU1O%J1mSRkQ1=x*$;G%J9KvG|t_xcPdUOQp27mYl@6+h()2PA~ W8P2(o`nr&u59i~wG~gIr*#7`;-v7P; literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..ae9b9467a7bde20843d3742e74d79f02dc7a706d GIT binary patch literal 7333 zcmeI$e@v8R902h5dG7~z{J6LSQ9#L5L>V2&FM-96z(b(rAP$tuikG{a2g#2eH$sx( z47Y}j$VJGeShmQ(!30Ynvo;MXEs$KK1*1mPbV)g-Q;6$5?oPhnr~Y95mFs=BcklDL z@AKZ?=Y5|a-@OdOh}UlnlrWAG#-9j(6!8bkAFuF-h+$Ux-O&9gW5UbKJJnn{r#BhZ zdaj~^D>axowZUXjr|z~Gv&;sg*=We-%$6ccQCX?coLpD^Lwcw0iv^c2H1mHHXtUYo zz5I#f;LZJ{apu8WJ?(7wM{|;ixiRnO&(DwZO%C&iz1Z&o7TWfEgN46GRK!RO7Nm)TzN!ge33d*Y}ucaJHIg)Y$ABnn%ai0wx8S2l~|?t|E#)7TH}$5y?G?e`RR zzzFv0aP0L*vDNw5u}82|&tPXS!!Gd0u6P2wp2KdV#_ymO$ESmwcFZFZ^gD`^{9NoK zczlxSZ%~}o;rYydfi24a3bhxD@%R^Cx`^VE&l-@WzqKRFbT(x9yb;-B-UnG>z~fP= z{RPE+`}~ksj0YgE8qGmoJA~hNs1?8OsK$>``^1T#kRMCI_xEJLB#IaRszTnI)rH)0 zCl0yyJbpf7z36!ockkPRny?k&n3fcA+EM&o(8 z;?5x$5A1mD8uUZa8Qst!-)>KU4j(pkLdO@>v_fmIYPLh?94(1~)(3y^D0JmruPNw; zPutU>J9I5;pucRcdlh=-{gE_avcapc4_d!@Go1gVJ^Qq4{@a@o+v^HA|5@YY_=n(n zcH%@*HL%D+CJzFO*A1yMp{>?E@cl^sXl#P}hucPEc`}aAxuC*?qtVwuU@B1blTzQ*qd$+4x7MYE8f$zMYXZixx$77ayBjG~?P zOE`0diQA>l%iR)}ySv;-_zyI7CpDUg$f)R;O|fzDLQQxE!Z;A-!hd@`kS;7+!eS*X z*uthIY~1%v^5V+S?m8>{*_rT>fJ**1{zD5iNCp{F;-8i`b3JAF6UViD;-(?dwN|I> sso`Enowc9s$EmIF8vb1Ap||NG7-s1i;gVZ2L{!P`T5@xF;_BAF0NSjhW=n_A z97+(HvmZipYd~n89T1u?6+#Q}Kxn~r5L(y=LW}%?&|-@qw1gFemV5`HrDsBDS*W<& J0|;NC82}QZ6ubZc literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..75270dc694f0bf7196224ef5618b54c59bbc844b GIT binary patch literal 8623 zcmd5?Yj6}*8l6tkc_Ca;qkzZ&8}S9qOn1*C2$?Bl5#==~3%V)A5EDYkYY0h5RCW+p zQm&{46a)kxBqX8&CRs&fC7B7RuohUXBC?2RQOHuR1uP;OvG+T-XQsPH%PN0pjXzS= zXU=_n=YIEm_oBsTv8X8)!5+-EEN!#f9g?g#U2acKrbVUy)~b&Mm!(+JgTb-|wFQ-d z`Gw^bRdb3;0)dK>KtW;2-1_+i#ew<7<@GgnmDN>6>b7gjGQD=w;;T`;?< zU~WZzL3N<0uyjFLeMMnaetCK6!?h0w0)-1I>+9sq?h?rEpbR@vGdflNNn)RvTr;PPeO zC1-L8Zg+S&bF(-1twyg{eXVpvWRVqF7G>IYFDB3`J2`1Bwf#%5VVSpU>^vWFmGzb= z7qGAEZa2YJrz^%5e4=g2U~pYJ7vMdaV7krB`#%QLV`e_kMIhUjgUL6h9?L-M{4G9l zfmPa-bK1{!xHKfM-9q|o=YnvEuVJkknZI8e58DQDN*)P$L2=Dgy04h2cV8?3!oo42aMu09ebJt>kZZos91fa*v?7oLU z<{m-0z)sLqG0wZAAcSH!Gyge+3(=WPn(>3cx3+?3yt12e>8#%ab*}s9Um_PLa+VQ9 z;ykS`=9}+$5i9{S=tf zOgy>=pi3vB_3>8$y3NclRs-~ynWr8lkX;z2u3|nMb%O0o0S~r!KrGnL-j^#bU}IuH z?(<^-+^z$y3_uQmOMdPqmMhDfR-lJi(GH z+O~a>D^_k7EP_Mx#U>m(s=%&HSwSaUO4oVm4zY3;_ax0}nkwjg3uWC$0V7i?PhAo1 zO%wKFf4Eh!fr<`pJ_*zv#~uDYP){7!-r*B-{oJ7^?6i!9bn1v-@6{2hNp#J48`UXr zt&p<*w~^c@4zfvhEx;`{Pmn7iMeBgB?=7SNJV|3TlLVuSY56S3-Kd2=%MlG(`);1< z3;Cu4P@-LqyB(j-=;wr?_;f`-CtZP0suZASi*4!!e0rjv_a3EB_IkwbPh|+5IkuT0 z*uC7DvqQ_1RrMK%f08HE*?XGZhw@(fZ36derEQbTun!NgiRqmbRBzYi(>%1YW46%A z4k3vSVAp#?vz|c^q;idZJh_u^@q${!qKgqvrLfOIO<+|vrAFHd!A#q{2>_jXOYHY4 zz+Czu*w_^$4~1PP0<5|opvTOtr7o)3R+#-kHYvT6h}L?at(q*xOkDIqFq6EVEaolE z>(i(AnaqVTd;MPSC%%wyFfnv#R~#i|9Q)YOBz2D_z@ozlgj7TTJ4`mS53zmx9{^>Y zi;(@b37}$N68OXu0G(#$QvrZ3Gjk;!d?bUJxrQHc9-Rph#}kp2kkjvH<5JZ<7<>J2 zgUz(s#2sF8kX&vlI4Q-q@eR+}t=W6SmheWdhsoYUJWafzjHbkpqs>!@uv2M_*q;-; zqOJE6SV?o5x|hvO=9TdtzawBL$aaTJ0qE~SJBG81Z^;u2X)SD1Y8T^oxjm^LCS#|boJB|WFv+gp zL0mUi?NF z^`>yIYJ^fTs6*!`%R@`2-V!_G>ak$D^icp_hhpwpx0!i8ujM>C6XK$jOEdEZeUI`$ z>C9QACDQZX{P)wJBod;Qq}2!Ep(lPZYQLMVZORs2r}kpq=}Wi>ZO?KM6vr@O30uRf z>|UA*Lr)=W(#w$s15%c78HVw~R5s8N&Rg4v%q&@0bc;7UaxL0A&+rPm7o)atw~-rx zy5gEjj{|iZ#zBWJ6o6Jw9Cy?-LfL016x1a4R{9E1bE&eUqCmR}QPuML(uhet?&mJF zcm7j@xH{1*j?0a*_j1IPc9tfHHtx+-BB-fsSE|~8W9sdDX&qM2ElFwvTaTBTRG^_8 ztnLqe=sPV&-8(p9Q*toOtoD(56ZlkkZ1&IjR5*XO=2Iuhr)b=3lhp=D?+cH=BF?D7 zg?B;@sU%IG3kg~{8K~T%%^U5Z$6S7HD!1yB09tk(xK0BX$#nxjSA_bd_O^DY^2gI$ zFVovvkLF9^K*GFux;N1qjliy=zkDBdXwN4jDH)WOp(i)E+OLG`=N^E8zl2D+A}omL8A%(FUgus+Bh{$O(;L1|@CiLUU=ZO&`MV<7>w%edftI5Int91C3BSB=5n=MJ~8oujSU@3Hh z<2`XWzoD;S8&Y`coEgYvP?{`4l#E=M{At9p40KJ1uc_Q1A({<*QsV1Wpt6CDke)kD zXvDEvZQmRO$9YoJN1~SBd<4`L*YdkJfw~Q&U;UnDf`{!wgOY)VT&Zl#66 zvBqn+rgb-VkKnmvx;4>O85OA?rtL`#;o4mP=Eubtc2mYQE#`lE87}4ipJf_*muV`HZ>uS7)7N+JhuQzPQx@YdC%xzCZ%VDc|xo9{B2vb7a?=C z=Zxqx+L|ExNbciQB8*{E=!=2l69-nMBe!>!caBlV>RS&rgXAAfO*ks=DEyE!M z`W@%EAo6X7gOmj6?Tt8>fvyR1$F~T!WIVP=*E{jp5c95yom`N(vvJ~HpiW~;n!r># z*+>$@IGF5KfNoBmq}6z>9Xr3`X=JMLB1!?ufs3)}o?}pqV{ihG6 zf8sV4cn-~wy5|WAqL6afqOkGYc=E88G#+>7cl>P6VF2c3;# zpTa9FEdr9lDui&7xbIvx@7V;B-MO9L-6z$M5O!|veZKR3o_BVBWpbf(pKqP}>V-`HCwx_e5is4JB46W{uc}-#qj}0jxhPkTa(%c|u6;u;?7}YW zieXo8T&RUwsD)ZF)at&tx7=5p?HFwtZ5eGDZH;@h)hN{~m(e`sqFj_KN4c)vUG6K+ zb{otbkuUN^zQ!$I3+39^Co(R^#kg{e>)PGrkZZ@=1i=fuz$*q`b;z|txljwWP%DO7 z-Ss=mA=i%g34#}RfmaN?dgU^jM=jJstr%)`H=A;Kp&)pH7kI_MtL`g$k_)sz3$$XO z)!n)?KX|UblxttNh+Wu)T`}yc(9$J>7HEN1479q#f9AK(m!m(=ep1Ryf0gp;l~P{+ zy_7fhO4Jj9D3Ui(&1-ISDt z?He+_Av5l8$h3-ew7a*+!ajeL%qVe#dQmUx6{B9gaCvqWwNMMSVyN}qWcl3vw!{tW z!Y=HJVOMWlHYHIDwNNXDT0cB8E?L;d^TzYW^Ere6YFx6gQ46(DD~DRYOy&m@S=cd# zoU^i=sh98$YXiM;narDUnQ@tM#oWQWD`mN@IQ2Dc@B%OJih);EE?L;aYOTge<79=c zbBx-3v{Wux*jCm$R~A-d^{^Gy8<#9>eWE^5pBU38c3+m|zM?GbVQ&x=go0RM>y-Ll zxs2v17v-W{Im&hQ!E#?w7PcGWI&X*+lVT2AQ48ggg+1&gR>X?f3R~xGvAgzQIpmUs zZE3A@X@NCZ4_i@n$R!JV*lVncRk0Pe&ii6_{r+;uB@5fqTIbRNYp@=+qI%_$Zf0N6 zzM_4_oPEWcO}V^K5WK((ykg*0_Z2Jj9D3UTb{nrlc%vO~5*vfVR3EX|RW= zEm_!vi*OOHRN<0^jbHeMUorfWg*}AEYLB!>*45CXhknFmQ<8BpF2)sOT(Yp;8Hc_z z@=yLbM3q(Jl7&6Ql~u7S*18(zz96p*Cs{TU)(x$-*|{GUGDiikWfA!d9-z zHKtr?>~W2)$|VbXSgp0qp z>JoK{C!t8|{*1b>=t(Zd#kd$(j&aGtMl8fatQcZ7ZL7lNwe#==U+@)!uWpqr?Afm` zzb^|L@emL3Vu<%YVUvZecGYfR?Y1gDYe%xM2^VOCb|AFtQ()V&u<;AO@GFL2vame_ z<{>Z-fh9JYy>Z!;WL%7kam5&yENsTbxENQeamm8QFZ{x<9Dd2dMl8fatQ=ysGp#eO zsVr4u}-y{cTYu(ga@MlE9w4`pE^7Gfb*4zb#q_QoX( z+mK7wtZRjIh<=hf}`V4 z=D8ZK#%bRrUnvWF=eJk>l!Z;$2peI`6gFAdYFF(B*6!JLb6MDg3$#Hy5Zd)AFj?5A zeWY-9Z_R%4(>O4cg^gIgD&(s|iFEIc%ci7`QOBrbjOiFZFH_8^z8S=SuXHOs5WvEN?77PNjPMSykbR9L_!d$MUbIG2q2-)o-<`#tK>3{{i|X{;~i7 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..1d4529efcccbb5b82966f893b5fd8e4a79dabab6 GIT binary patch literal 7333 zcmeI$drVtZ7y$5l?rlqXRa!bWU_;#oYy?ZCFd#AzC@-S{d2HyWaw*LK$0MbgfFmrE z{UI8}MPxI3=uF6j1u{?vh|^>f#7$v@h?_!~K{O+-x@^ccq~6kYeBaSOh<~|wPjYk5 zuixp-J@=l+m&-7WBxtk0l(Cmk;Y5U^SU6bWcwIO+hFR-#J^M#F6IO28rQs`hLy=Kq z;43TnvO*KDDJ(K;606O|3{#=eWGu|&P3B^Aae0~16klI+HMK+ia{kzb7U4qvR;zW< z)0apO-Z(&-<{rG+-Ntr(x*(lenDBb>{Nkk04Z2ScH2-MS{dI&i*-n zmN_eqvkPT8WjD$mlogbfl+~2iP+muQ1LY9P;gn-3r%~QUc^Boql$$AcQy#}A)cYV? zsJNN(S!{MS<^9+kb$!k`P8`i7xwySlhb`NR?Lzfeeiz5x2C+RX*bf}QR=tbu^DMUC zDE32P*g?m!HF?-kN3j$8u`^d;=lfz;K8@YLW4BV{cTkVxmH>zC^N0lej^ZRQ2m2@< zpTzPlinG~xKC@q8b9vvO_7Vvm|B_1=QC#|EBeLwbHe`9W71?#si0r=TjjSld<58*q z1;y39KFDh({gBs_l(~(ikJMVLf)6riG28W zG;+^*{Cp;Q(DNj2UbkR-5}#_@2ksS8xBlck2o-YSoEDl5K7-a*_HD%&jB|cfXnb(J z&$+_5q{=l8TDo%_^;cG2Tm|DUJ=1g0uKMJg(Cz_!Xg+!7sYYPji%TDX_Kpff<9W5> z_8}PeZ-4U|^urNpUC_bbZI6Wx8!765j>)e*2CcuM-42~~yfhNp5cu)q(0lH9&OkSQ z(UuC`o_%;d^w%x*Z$Qs|G@1-dHhC8GLK~8j;QS|TnP;5y-`0#+-%`N&&l;yEKLP(| zr%uJy0CQ$CeGpi(aaffOJwLw}z8~oyP0eusaM|oVlK|SwTTa(}2<(>e-hc_%Xq7%4@Ly%1de8D$srv-%tVf1K*SJeHsuC7@ASR_qFjxYI*~Rhvr%WVLlM4n;lSq zc>MU!eJ=q&d8G3cyq_JGm`FID%d+&XO`v^!LC@e6@Ub5z`dfj|o_N0#-tY9ekkP2+ zI8nWFaUl^nz2w&2{lKq175vCB@Rgm-C85h^V;-BSD*|p;cZAn2o6~iboLE>ki;E&v+tn{6HfIXtrQ#o3#coqx{+=wICsO3XCzqw-eAV&q$Yzt;Q{aB>3^8QWKY!mT(3J9~ z$|8QZCO0P~I;XnANQ4hG^{2JkN5gdykz1moW5k;H48(CD&V~Q>dLUa`w#3CsT(HGW zOWe5co8-lrpxtp)gtH^zBSB-rZ^DNbXpjstq|`S#cWyRe~+?5wjEc?hl2lH>u#{_5U#uOjBv#*8N%)1cdxj4zq)$pFAx*JmjD0& literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|!G^+%JW;+d` zIf@}Pmnekh-UXp~vmrD;D})x91)+s>A++#W2rU{3p~db&XbGshqaxA*HS5lAE|LOiC&$Hpxs-s%%YJiyE4com;N5WIJckf1+%u84)AAWRhumBQMI zf4}(ppTE)3Dj0-U3_|*>N3+g;nr9V$BnSoqH@VM?fE-v?^kT2@>u{f#;p!THvR5?q ziO#UQ*~09l*Ne2?C+z5mi5c#*y#u|XF(x{j+j)9H-6H!mEEXUB8j#cM9O)m4Ts#EiJ!Xy!j6^1B2g zvN^NQIpNZRQaT8&dmq&dz~N1KI7-OHIFJ<`8mpE4w3@Y)oA#wZUpx)N%Ld4By;b(7 zKwthP46iuG4F^)N)m6Z*+{bkZreI%P5A2#MhLy^ugZmt5vw0JIs#yl;x{8=+YPUGO zt`ux-7O>whJdb5LEZq(80pY06`QsgQeW#thwB06qk21*`oDDaKJ$Ki(%vOt7Yn6`r zJ`b@-TD~Z6KGQ2DxT>yXu3^nYd7MH&K*d>Mnk~v<>4gCaR#vUcJ;>JQ$S#rPvXL!^+dSq?B%K+ zF)d<|-G|ILDC2_P-H*o;`(q-OxlM{kZ?@LzJbevj51AttBmJvMNeXtR8{WK$n60-; zJ8X%{5KDh67e3$31VbL=qOkxIRKdEUqY4)8)C5y}r4TG#$}w?CdJoZlw&p9MiGs&} z%0T|_g6Y>#fdJ0;f+^Tb6t`x3+eUuJJ95ZohV)PA zeAmkl0(#;JmDJUFw5pQ&vt^9b>q~n?6OkHTIlLAV4i?4;_GJkr;Z}=O##aPN)eTv+ zVEP1qq_)yTx6o|2#Ck-b)pFP%7~E>LnKVqnx*}S;p7$B; z`CqO-ipCR`;hJ!O3EY;w9ETjNI4=qxSOnaGLb?Oi5X*wgC*JqZOtPe4{s1IfuPXLM zVpPR`PZcbg65a0O#zbP1^b-AzjMpc$O}rnw4lU(D=2?mr=`MUrXt&I5%bx*qua)p9 z^XlM;LB|~c-=O?Kg-BL(V60BP3GA&*9%a51Y+)SO#0`%1r(kD%h+K2d6jMVq1?*}9 zIO8M-2UEbxPqFdXBMd8@nwCgxmgc3=!QfS}5>Tmzhp5M;!y4&gZU@v>Ib4%)nd;}Z zewju!du+2$_B5;6aMt$_k5Q7*(@v#-H?LjA-tVQ|Uc3IpWoj$=&S)~7%Uz|(fX8x>E94uATwn(BYrNje{*lOqSAo3S ztWrD@`Ry(si;z1=+Tcj!l2#yRs`Y-0ocN10Q)C`9f0s4TvduHQz&Nvjiq1u0zjPE; zb`S)=3A=tu14t5T$svNj8YF@pU4LNh)Z8YoTS7lIDBUvc=NI+ySJ{myrW9e?VRL#t znps1g)wyCzT+Ef~tz56Rl`F8#oQ;%iDD^ST?&Oj5XM37*dn7%jDzs+7s0!Wpgr<8|$Lh0_Xw@;_#Wf0~V4tV2nC%besZ^(6 zHxJNi)+5ZYIRkELVeRzLRL`bk{Oc7~N{RCjn*#JG*SWuh8}4M7w-rh+)bG-R}Nl9Ju(f%EkbuzMwS zTr_t`aXzuY-&05x_F6N!6{QZu?YWSlZo7>4fUlf;Hj@da?*SL7f^~F`Dphn|}gw`cfdB|3MJj;btp^gY4HKV6XcX zT6FxXvDS(vBai}(ZNzSGRkPbD2X$ENwiVEKH?n&y#q%GLzD)Ys3DXz88y9n&9@ixJ z*eMN|n(^ttLonI$1SLC)&ameTp;u<)3j?ezl2%^`t!_l-yeiB5gLowOndI5Jo>+c(O{>U z*5lI)MxG`1{52h2d)w!iMv8VCAu!=%t3Z}z|H$F+=Ev~#^Z*koC!h_7VD1m__DlzP zOaBOt1SXEd>8&hT)QpLXb0emMFx7Re%|aU8<#lPc(2qnikYyab`zRp6sP|KLComcd zed64<@J|sXRWwsYagDtm%^PNY>;|}#EFuvJ)06#T1uG&s z%ALbgx`C3H?HjOlX(^p6Sz)MAh~yRmbGby9)7Zx|xjIXDxXfFcJLk4=0sGT<$V4XJ zF_L}lN#fkqLM>#x!E#2xG$|;S;l|yGp38nclPMuBU~dvd6v^ezf*Ab$;s#X;`kvxC zB$P=TS*%UNaSAT-Qn6aI#~5w?7clyA4b6OXNfoya#yP#~6zqP5My<{&dvyLVN{gsd z%W!oY=$50A@apjJcjg|T(~c0D?lP|$#Ny0f5M8p5`_B(YK6+lDFHf?R=gIyQGyYYy zVjo|5(3)0dlLI;@49z~L{~1m`2ySrB31G|YP{bmZ=TX4)hFAz(-%?)5Bw;fx`-lxi z`&C8e%pOgV?(ER;BT?AbkNQ$C{o2&e_!49zha# zbrl6UEeA9-GDO?Gv^cTmAWcLX4htt_sPgk*i$y$WmDc3qVA3x#`i*4c>wV&c5ETe- zB@yLv4ey8g5Scdy^`Sf8-fl5g*-}Q3NLeXpob?3d|3<6qK8YEWA7w_t+{lAh;i|)& zS{))~4-?G73xhR1c|P2#)m|7ZDWwEhXQjH~9T;W@>aS&)?0#f2O32ELVPBCqSU%fb zoLCiNQ|gKh{b!gy$p<`_o9UCM>a*T7sy?Oq5y=0KaA6XeE&|4!GS{z_ucQ9E7mT4Y9 z%x`sq*QwNM^E#kI^AKyfoAaxy0nxEAx!0}|>+#NkR{T^3%8XIG>^NRCtIX<5;sEbo zW8sx@tPj!hPukCdf{an9e2AQ3uj^!Ch1&8zA;$jiAN{*_rp564fgf5_t5W=~eN-vx zbngeAU>8j?d!$BV;yS!K&Blh;+4F&PoEnT8ka`OfLk^7*#q>4F3i$R04i^j%WS(%cd zccOO%ec92xkx~hz9iPq;9VgC$>1_%o8bM^g<^paVMGW1#k${T`7RP&~WNo;jG&nCI zM(k}aW`@=G=XpTfQL+8m|0f_UstA|zM-`#XrBx8c^)N9vR+>V8zW^03lY3_S37SSw zC&J1JEJ&~7OSr6yqF}a=dNRs2^cLmT!8x~=&18Uat9ahmfr;EJgS3?S+g6KU6pR=l zbYQT;y3AA5Mqt8gQj$Z&|I!#k)u55CMA8#7_Qkto%+b-o=#u>@rrL0dPpI<-`2oEj zSEf6^s)fOz4kA$R5jE_9#m;?P0Eah`7>65}*eU-z6-PPDpOzY&iw_Y8wi$KBhTX+- zcon*LLx}yVaL;=`xy3Y5pVjDrjRIpEB}Y;*G@JzwsyQISwmvene1O#T$mJJ=_2gfcXRo<9XrNYB5y zQcW>YRuy7ebniUrV#@Gee^B4Y2%qyD`iFeeL;14)XCK^~Hg{GRp z!u?j${QTq(3$@ho%M&SE>t1xCc{h=Whv-K$yDzpVei-?Ax8a-+cdb-@H?fY?R^S zqsKmd=2*F~v$3%`nQomtb$YgK<#*ukz+MO}Hg1SDYMsO?timd+8aJytr7m5V@7}#Ch z#?e&F@!XZCm&P@5sEA$IgU^u%%c`+p;ipFu75G#{i9`Elc{7&vL)G)Y>hkF z>Xd56C7EYjjEix_7}s#-x&4sqbcdN0`66HBYuxfxm1`=Rr(Beaa^)!3xv%z=Ye&e1 zUD$OI3%X7u)mS{_~CE5~gjXT=vlxoT)nx|Zpi*n^C*Trv_ z=Ze!=hnW@mB46Zd-14NlE62Dl&z2$AmZu4V7kGhJ47_T{wMDs53$;)y zhFZhb@0TIhmgfnA7kGhJ47{3hiRMuYwNNXDTEmUk_La*61;Govz$*q`^<2@NT%ZM7 zpcMnH;nt7y-M{NYxpqv8*o9r#6~nFyEmICtrOdu5<<%ETdHwTJ-acB&J0F(v?r)`haJC%T9R5763mfqe5AkA%*FP0h z4<&VB`-F^7$c+0FGJVHd&9)a^*njE54t%@u$Lr&$>-}cigv-6F21Wy;fidPC{L*L3 zo4__TZeSO7VOI>hnsGUlL@m@ptr%)uIleS5UD)fL7|6_IW?4I+8ka8Ytk(=>P-eaV zVy}NW-~C+|c8nn-7GkY8vHA{Z#wD4Taml!3Truz9!bD}xXoES4F4(B(P=ZdWcT-wc?D>_$n zu9$PKc;n2za(SR2c!3vq#lWkcE4q^lv_K2AVxTqLy0usJqYGQcCF7ED^$cn3t14Wk zM8ZY52v?49-99_N^U?q8!k)~3J)sL5@emL3Vu;r@vK~t6!Zv0aGmV*@3S(dC!X{jV zi*Th1mo9Al!Y}-a;g>FKBcc(}h&cAa@Sy8!5tlKc8SB!D#!e(5Ii*X%fG-Ll- zH7;G)_=R8iHFj3@>(hmeScrvKod`LrW?Z_kWn3~Y8COikr3>43wOwPjYmN>b#+6p( z(uHkgG%^|)4}&+4=6mSEMl8fatVctW!~Iv9ap}Sqa+#V<%`sE6E^OP?c8%GthpRMW z|87mWJVBLmVK|1zFnrL3E#;DONx8K>D`s>Y=Y8^7=izjCOh3mdTz3$b#D z)vvS}mo97>myAos6_au4!nR#)*O=|vb;D}5U6o50wvo}uXk^Ucp)PF1LM+6}Ay&WA zW?Z_kgB1J$3+aXQIbhU<&A1pB-G67E--h4Ds-KcSR2TNvji27rg-zH98)3^7HeJ~EuDu($ch}d=bzu`O&<5>5 zXxF>IbYZW0q_Dnk&3^J#3=DN)BbJW}`KVAL-J5Yalr%A#7)^{Z6QeF{#>Kc8SE_O8 z!p1NB!mk{D>B2@V#6qkbV)ZMn8P`M?wv0>0CF6?8xO8FLuC{B;cI~=hr3>5Nz$)J8 z<5gYlS5+6bkW0uVa&_)T|5JcC}q&wripoJ-%yh%H;{FlncW# zJci-Aux(e{HDNlEAlZ5HRRHTO}Qu+)P;>$ jh=o`=#OhaCZ{&(R5A8pK)oMQt!0YSv{jXp=L>KnI{L>Dv literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..7398ca55ce0775a051b7710784ba8c15808cae0e GIT binary patch literal 7342 zcmeI%e@sy4Lo)Qqv{nlu(J+y}bA2`#rVg`YYEx+u6Mz zzURK3bMHCles^IQM!I%=pp3DXaQ;B}TEbVBuW#{H!Z0iRZsh)`U?M6_+tgeYr!yMV zIHvP^n|$)L~XOy&}ENo9q>l-N-FLwdLGt3{X2wemj{XtUYo zJ^hJn-_5pXHeckc^lUofj zRNPGYG&Z}8@*Zpn_4^X%JaIIW=HvDLE$Y!*LnZS!ZS{U+;GecLnj#5u?U#=(wV~BhaZ=GzHK(N6Vw3bs--=0lnj{=M;3) z7oF+QUAb++&=*=8-iDs}Xe_ABA~XAP4RpMdLG z>+ytIV2PPb?gN&t8Bt|I&(7|G&qwxqa|?WbxUBb?N&@W_ttVUN&d((SXO`Z+w+Hymr$Zkb0lu=er8IoeY~15h^+w<>-|on|MN723rN`zL%{a%3 z_w>K!bS$_=IQ_rmaQ1g7!$eZ!M|%bz6LUCpgTx#T-45no=WzC)5py^(hXWHz%;6s9 z9F83Ppzg&4M)d~6$m6$oU$*!!9LG1y`A;qj#rdcc+d2w!j8|WG8Xn2IzyH2dtS;wF z)kdyNou9WkHg9K@f$$$_8ct|58zQw)(Hmo8z`>oCpRbUVSOmmXj0}v14Jv{x2NlI67zB+Z7<4Y&mtYVu dmSC_sRPqYQGLc{~J!}#s!4LpezTnQ`Q~wd`Y3VXfe(z$lwMv-o}dqJV0 zvVAw3XzT1lw|X61&lmHu>G{c!SI;g?2`Pt#>M5Q+z{1+oH&}!|q9R2iL>!;^XFu`6 z+xjyOU@T`G%vi}-#W>7su^-_~d?n-6jN=%uXPm}3hj9_(ZH#v@Ze!fbcpRHD?}I+h z#O;jFU~?-N@4=QZ_m_zC#G589#QCx$Y`;y|{>*p-?&7$77(3X7{Xi48`aSIBFJMQE zVLzOJz2+FUwir9*2zKUaY~wQQ5)F3EbJ)#1b_X+mhxIt_ibAGQzhf5{U?0Krle#XW zIG2ytGxr^~r1%2LmrC*cmtH)N;<9gAko|shAP3~TkpmYj$UzIC$VxMwk1GAoC?3|o z965X{0(s?l9`foD{J!Jn@cT|`{SxJGn*I^_=?vVz=OdgbUjB<3d3SC%@}WB$k^9c# z=QG)do+p)u+=ls7m+jdH?ibpy>co8r7xdg=7*&Jr9c>$ z)&{0S%eIcA@%mMj)xx-cpK}&EP@i=RIw)!YttUmXdKAV(B!(gA(3BW7pI0dF9E9=6 z&UddvKe8dG2Re3QOB!^-sI?0^wWR(iwEk-17U;ZV70J-1n9rVs-tnhm2D;^IM>ce4 z{-IUS-?ulu13mlcSQao{t0?Vz1-TRer{c{%Q)aT&y-1N!xI$(*N zI`;ufqes*R=()LF@cqdCXl;Z0hyS|JnGBF0(Ef4VC&2RD_lIo2!LPLZ3dgU!qPh;p zuezAis|NYY_~vT3A81bK2DBg^_3exrzOU$;*@k8ik1uq^!1F+I(v2Y{i0j6G8h8!( znZw;DVSiq9r6$AqT$!iuXa)I=rG3NGz(=o5p6&oXbNqvD*k9*h+*ry|oT}eAKc5L~ zD8GGo5Afb+V;>y>zPh!oJbuYs>Ju{!R^ZOCuJ!dxmL&C*AD>?`5yvWir^~!C(PAnm zUU=L8%Nb7WcDQ)qZGC4r^guW;$2PY!vo|tEzgoO6B{Y$4XqfZL)05qc*PN&I)W_V0H~)Gt$mEVT60-K6 le_$%rR`9kOE5BV^Sn$-wf}PbCp$n27{PzExpH><@@i)k7vZMe2 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|R literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..060f1a3fa278f6ab2cdf5a53aff211832389d760 GIT binary patch literal 139 zcmey&z`*d4lY!BoK|zCsfkB0Vv4J61(lzwkYsU)*7#JBQGBU8)gbCiLkha)T# zei$0WDY7Md(U}m01tO>eoYQ2Js++>dGTZ`$B^AvUtu7lfht#LgXMDdG6aRp5$>pBU z^SwU3yXWqCE)T;n;!T^wCCpNa;|s#)3O=)Zev8i{hEX=DFUuTEyw$c-!_{yGvq@v% zYHK-*(Z*?vX1gY1kKL4KGn#BBV?JlISJ*487L!faSa&VENA-HyrSt9lg~B}^&w@-% zBnNLck=EG7OV9IjJ3d$cT(O36JIrPF|IF3!j^8s_M`gie;3CCo!G%{><5~#l^`lk9HKo`oN3kmOIK8xMVVRumT*Qv*GcO){2{1dyh2>U3WpTvC` z#o0o&*?R^j2OQd0a=RNQTn$_JqhnVE3?ldk+zaQzd{dvdt`S(9t(3-Eh(`ea%i zu*gna2Z6;KN0hnHb8~y){gM3D+6Lbrew#yQ(m{Ry_D||Q1rE&naM%VM{Bq0hu>bNa zitDidiiG0l*IWbN2kK7UkOssfznf9Q`?c|Ac5X9>#}&Jy;C>)E>Bg`e#C7Ap z47~>Y^pW0^ustuhwaIWkTMG0Yt)PBm`G9j8_}Ikc=?>sCCqC+h?RA}v9ZOk`6XhEh z<}-kED{tT35B%mc(T|J(U)|AG8Mka!`}j8m(CDr?BVKhz!2OaCTKwt)HtxcEn=gIWCo1ng ziGPnnga2kowK}VK_D1^X*Gm^A3`rnc8s>cRwq)PZG3zb8x$0fE`Jad&gC|DIFfsrA zfvH?m&DmQ_6k;KZ@dA=U1g-3XHexqBJCn__x3ibYW~=xBVrSf^3apY;ZDYfG2;bE5#(VNi;P84;Er4(ktbs|@X4c)C8;d= zxRRt)IZ!GTB$z5CC4m@4Mc~N5Ist$!O-)nRpam_%g6rkoYC{`Y2i6@Ews{YB`y1M> zT|t*Qd=>6pA(sVX6J8MTKtHuppDyL{oi)yF?AP=vem*_^{R68^V2%I) literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..a38308e51f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +383358291 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-QeKgVL;Njxfw{Bv}|Ai05x9(yZ`_I literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..163fede2849dfb54d03f6728b221b1f4503a3a7d GIT binary patch literal 5806 zcmeI$eNYr-7y$6y`{0`k4*URwN=1aZ4&uI~5ClP>se$+qDod8Tg@gE-<1!!^$ncM_ zA+iXa5iFZw3>=uC5=gBX4Kgi|oJa?VW(XUZ!W<<`f_C8+p69LR?!RUnc4lYych9qX z@7M18-cb}K*%Ig@rIt1Ycc8gFm)k|${sy;;DN5cTVSG1G!9}JVB~#4kjjU47l$0=q z1{0$+7|qJ)GBcZKGO#Arki?kGx#rxWLe>;nQ~pz2yXPyJS1vYkH}Y9nSeSS9qNRs! z9iZ!H9=zSvD(XBlCmo*~b$jvr{1}&Vl-v1@F9Y02^JRk@S4Ol+kSI#1pYXAraAG;y z6Lui%MA(I}jIcXlPr~a7`x4$nIEZj4;c&w7gm)9pAzVSYfp8b$VQgCH4@*1KS|V;H zd=^`@j_`hLv2cIO$6{feSS(4y`BD|O-F9qyqP`CIaNMaM+hrR2fdkm`_psMKi|svx z{ZKIWmSfn;3~XH!cJyiNq&3)?Uf3m1W7jg+EyVckkHPV2KV;hPckGN*>?S-u>FMuL zT$F;JXVKT#;*4)mzC?n@zvS{I6qkNghirGV71<$W0oie$MRuO|K$aQscyy2X1;ssk z)*`PT^G5a^PDb7|fcraW7WcQR{&SSSef($SC!%oqp7EYQ@w{K<$kmA*$VcvOL+(D0 z+h?>JwI}W5b_eFuk#YPyu)NV#8&52QJE1S03xO5|oI>wc(YvxMFfR5kMdL%<^SmRB zOG+Ihprw0;QGM--a!X;{zI$Q@+A${fHng)}FM6I_GvtFX?j}y?gZ9w*qw%~>cK0xh z`?S4v1Nvcgd?$3k59#61!Gp$j=&;PnX6Tq}A?eV`#|kvidjC&%K=1q0brQPn%hovP zwv;0qp}%RYc@ui(lc88(daG+z543*gPWb%Otx2cg^G`3lBZJSsh@BYw3|uc7KN(RD zEH={-9A`pd4m z-+=XZzZ~Bs2l;E5+G02#c%6vsRf4$Rg-JQ=ug$mO5^6zQkv8oQ*8>{WRG$pQBZsf| zz6|{2(TokZvVT%U`=R z7Y&?{cjw-I;Db*EJTd@$ZBIjWbg#|6J3F zz%~5of34x9mOSAoT!!8;+=r`cIH6^hjt!WUt!p^TWvgpA+No`YH*YWf7 z;vCK8@attfzj6^U!3w`8xg5)P(ZT$hlgokz<)G*Suh_Jisi?s(xIAv7DLRB!SIt_) z7pi*cT*MzO=ak>u*DL{`;R!S;_*&RAOZEm1(KFwxyO8vs?|N zY^mAA*nEX}ACq4_VD0{?FBRvr{1;fgZwhO+DKDYWtP15b0~OZ3+}2wPjSv_p;VKn+ zU)DJmpWTrJz=d7_d{LND}QwQ E8(x$IC;$Ke literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|> requestedRanges = Arrays.asList(new Range<>(sstable.getFirst().minValue().getToken(), sstable.getLast().getToken())); CassandraStreamHeader partialSSTableStreamHeader = @@ -173,7 +174,7 @@ public class ZeroCopyStreamingBench partialStreamReader = new CassandraStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, - null), + null, CoordinatorLogBoundaries.NONE), partialSSTableStreamHeader, session); } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index ab03d6d55f..56f35fe180 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -846,7 +846,7 @@ public class ColumnFamilyStoreTest { @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, Group opGroup, boolean assumeMissing) { return 0; } diff --git a/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java new file mode 100644 index 0000000000..93b9c41833 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.MemtableParams; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.assertj.core.api.Assertions; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +@RunWith(Parameterized.class) +public class CoordinatorLogBoundariesLifecycleTest +{ + private static final AtomicInteger keyspaceNumber = new AtomicInteger(); + + static + { + DatabaseDescriptor.daemonInitialization(); + MutationJournal.instance.start(); + } + + @BeforeClass + public static void setupClass() + { + SchemaLoader.prepareServer(); + } + + @Parameter + public String memtableConfig; + + @Parameters(name="{0}") + public static Collection memtableConfigs() + { + List memtableConfigs = new ArrayList<>(); + for (String config : DatabaseDescriptor.getMemtableConfigurations().keySet()) + { + try + { + MemtableParams.get(config); + memtableConfigs.add(config); + } + catch (ConfigurationException e) + { + // skip if the config is meant to validate config parsing + } + } + + memtableConfigs.sort(String.CASE_INSENSITIVE_ORDER); + return memtableConfigs; + } + + private static Mutation createMutation(TableMetadata tableMetadata, int k, int v) + { + DecoratedKey key = tableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes(k)); + MutationId mutationId = MutationTrackingService.instance.nextMutationId(tableMetadata.keyspace, key.getToken()); + SimpleBuilders.MutationBuilder builder = new SimpleBuilders.MutationBuilder(mutationId, tableMetadata.keyspace, key); + PartitionUpdate.SimpleBuilder partition = builder.update(tableMetadata); + partition.row().add("v", v); + Mutation mutation = builder.build(); + Assert.assertFalse(mutation.id().isNone()); + return mutation; + } + + private static MutationId applyMutation(TableMetadata tableMetadata, int k, int v) + { + Mutation mutation = createMutation(tableMetadata, k, v); + mutation.apply(); + return mutation.id(); + } + + private static void assertEmptyMemtable(View view) + { + Assert.assertEquals(0, view.getCurrentMemtable().operationCount()); + } + + private static void assertNonEmptyMemtable(View view) + { + Assert.assertTrue(view.getCurrentMemtable().operationCount() > 0); + } + + private static void assertNumSSTables(View view, int numSSTables) + { + Assert.assertEquals(numSSTables, view.liveSSTables().size()); + } + + private static String nextKeyspaceName() + { + return "ks_" + keyspaceNumber.getAndIncrement(); + } + + /** + * Test that mutation ids go from the memtable to sstables, and are combined during compaction + */ + @Test + public void mutationIdLifecycleTest() + { + String ks = nextKeyspaceName(); + String tbl = "tbl"; + TableMetadata tableMetadata = TableMetadata.builder(ks, tbl) + .addPartitionKeyColumn("k", Int32Type.instance) + .addRegularColumn("v", Int32Type.instance) + .build(); + + TableParams tableParams = tableMetadata.params; + tableParams = tableParams.unbuild().memtable(MemtableParams.get(memtableConfig)).build(); + tableMetadata = tableMetadata.withSwapped(tableParams); + + SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1, ReplicationType.tracked), tableMetadata); + + // TODO: check against all memtable types (David Capwell style) + Keyspace keyspace = Keyspace.open(ks); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tbl); + cfs.disableAutoCompaction(); + + // pre-apply + { + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 0); + } + + MutationId id1; + MutationId id2; + // apply 1 + { + id1 = applyMutation(cfs.metadata(), 1, 1); + id2 = applyMutation(cfs.metadata(), 2, 2); + + View view = cfs.getTracker().getView(); + assertNonEmptyMemtable(view); + assertNumSSTables(view, 0); + + Memtable memtable = view.getCurrentMemtable(); + CoordinatorLogBoundaries boundaries = memtable.getFlushSet(null, null).coordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + + // flush 1 + { + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 1); + + SSTableReader sstable = Iterables.getOnlyElement(view.liveSSTables()); + CoordinatorLogBoundaries boundaries = sstable.getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + + MutationId id3; + MutationId id4; + // apply 2 + { + id3 = applyMutation(cfs.metadata(), 3, 3); + id4 = applyMutation(cfs.metadata(), 4, 4); + + View view = cfs.getTracker().getView(); + assertNonEmptyMemtable(view); + assertNumSSTables(view, 1); + + Memtable memtable = view.getCurrentMemtable(); + CoordinatorLogBoundaries boundaries = memtable.getFlushSet(null, null).coordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + + // flush 2 + { + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 2); + + List sstables = Lists.newArrayList(view.liveSSTables()); + sstables.sort(Comparator.comparing(sst -> sst.descriptor.id.asBytes())); + { + CoordinatorLogBoundaries boundaries = sstables.get(0).getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + { + + CoordinatorLogBoundaries boundaries = sstables.get(1).getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + } + + // compaction + { + cfs.forceMajorCompaction(); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 1); + + SSTableReader sstable = Iterables.getOnlyElement(view.liveSSTables()); + CoordinatorLogBoundaries boundaries = sstable.getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + } +} diff --git a/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java new file mode 100644 index 0000000000..a62efd3484 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationId; +import org.assertj.core.api.Assertions; + +import static accord.utils.Property.qt; + +public class CoordinatorLogBoundariesTest +{ + private static final Gen LOG_ID_GEN = rs -> { + int hostId = rs.nextInt(1, 4); + int hostLogId = rs.nextInt(1, 11); + return CoordinatorLogId.asLong(hostId, hostLogId); + }; + + private static final Gen SEQUENCE_ID_GEN = rs -> { + int offset = rs.nextBiasedInt(1, 10_000, 1_000_000); + return MutationId.sequenceId(offset, offset); + }; + + private static final Gen MUTATION_ID_GEN = rs -> new MutationId(LOG_ID_GEN.next(rs), SEQUENCE_ID_GEN.next(rs)); + + private static final Gen COORDINATOR_LOG_BOUNDARIES_GEN = rs -> { + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + int numIds = rs.nextBiasedInt(0, 10, 1000); + for (int i = 0; i < numIds; i++) + boundaries.add(MUTATION_ID_GEN.next(rs)); + return boundaries; + }; + + @Test + public void roundtripSerde() + { + qt() + .forAll(COORDINATOR_LOG_BOUNDARIES_GEN) + .check(boundaries -> { + try (DataOutputBuffer outputBuffer = DataOutputBuffer.scratchBuffer.get()) + { + CoordinatorLogBoundaries.serializer.serialize(boundaries, outputBuffer, MessagingService.current_version); + byte[] bytes = outputBuffer.toByteArray(); + try (DataInputBuffer inputBuffer = new DataInputBuffer(bytes)) + { + CoordinatorLogBoundaries deserialized = CoordinatorLogBoundaries.serializer.deserialize(inputBuffer, MessagingService.current_version); + Assertions.assertThat(boundaries).isEqualTo(deserialized); + Assertions.assertThat(bytes.length).isEqualTo(CoordinatorLogBoundaries.serializer.serializedSize(boundaries, MessagingService.current_version)); + } + } + }); + } + + @Test + public void monotonicAdd() + { + qt() + .forAll(Gens.lists(MUTATION_ID_GEN).ofSizeBetween(3, 100)) + .check(ids -> { + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(); + for (MutationId id : ids) + { + int originalOffset = boundaries.maxOffset(id.logId()); + boundaries.add(id); + int updatedOffset = boundaries.maxOffset(id.logId()); + Assertions.assertThat(updatedOffset).isGreaterThanOrEqualTo(originalOffset); + Assertions.assertThat(updatedOffset).isEqualTo(Math.max(originalOffset, id.offset())); + } + }); + } + + @Test + public void monotonicMerge() + { + qt() + .forAll(COORDINATOR_LOG_BOUNDARIES_GEN, COORDINATOR_LOG_BOUNDARIES_GEN) + .check((left, right) -> { + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + boundaries.addAll(left); + boundaries.addAll(right); + CoordinatorLogBoundaries merged = boundaries; + for (Long logId : merged) + { + int leftOffset = left.maxOffset(logId); + int rightOffset = right.maxOffset(logId); + int mergedOffset = merged.maxOffset(logId); + Assertions.assertThat(mergedOffset).isGreaterThanOrEqualTo(leftOffset); + Assertions.assertThat(mergedOffset).isGreaterThanOrEqualTo(rightOffset); + Assertions.assertThat(mergedOffset).isIn(leftOffset, rightOffset); + } + }); + } + + @Test + public void builderEquivalentToMutable() + { + qt() + .forAll(Gens.lists(MUTATION_ID_GEN).ofSizeBetween(3, 100)) + .check(ids -> { + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(); + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MutationId id : ids) + { + boundaries.add(id); + builder.add(id); + } + + CoordinatorLogBoundaries fromBuilder = builder.build(); + Assertions.assertThat(fromBuilder).hasSize(boundaries.size()); + for (Long logId : boundaries) + Assertions.assertThat(fromBuilder.max(logId)).isEqualTo(boundaries.max(logId)); + }); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java index 33f1a63238..4008472cee 100644 --- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java @@ -101,6 +101,7 @@ public class SerializationHeaderTest .setTableMetadataRef(TableMetadataRef.forOfflineTools(schema)) .setKeyCount(1) .setSerializationHeader(header) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .setMetadataCollector(new MetadataCollector(schema.comparator)) .addDefaultComponents(Collections.emptySet()) .build(txn, null)) diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index cb55129909..2067b9b4ef 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -277,10 +277,11 @@ public class AntiCompactionTest private SSTableReader writeFile(ColumnFamilyStore cfs, int count) { + assert !cfs.metadata().replicationType().isTracked(); File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { for (int i = 0; i < count; i++) { diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index 79ac2bde96..8270378b25 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -43,6 +43,7 @@ import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SerializationHeader; @@ -1444,7 +1445,7 @@ public class LogTransactionTest extends AbstractTransactionalTest DecoratedKey key = MockSchema.readerBounds(generation); SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, CoordinatorLogBoundaries.NONE, header, key.getKey().slice(), key.getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) @@ -1480,7 +1481,7 @@ public class LogTransactionTest extends AbstractTransactionalTest DecoratedKey key = MockSchema.readerBounds(generation); SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, CoordinatorLogBoundaries.NONE, header, key.getKey().slice(), key.getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java index 6031e4e611..a64dbdb064 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -167,6 +168,7 @@ public class RealTransactionsTest extends SchemaLoader .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) .addDefaultComponents(cfs.indexManager.listIndexGroups()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs)); while (ci.hasNext()) { diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java index 03ac14f364..e4d99f295a 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java @@ -63,6 +63,7 @@ import static java.util.Collections.singleton; public class TrackerTest { + private static final int COMPONENT_STATS_SIZE_BYTES = 4805; private static final class MockListener implements INotificationConsumer { @@ -156,7 +157,7 @@ public class TrackerTest if (reader instanceof KeyCacheSupport) Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); - Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(17 + 121 + 9 + (COMPONENT_STATS_SIZE_BYTES * 3), cfs.metric.liveDiskSpaceUsed.getCount()); } @Test @@ -181,7 +182,7 @@ public class TrackerTest Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); } - Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(17 + 121 + 9 + (COMPONENT_STATS_SIZE_BYTES * 3), cfs.metric.liveDiskSpaceUsed.getCount()); Assert.assertEquals(1, listener.senders.size()); Assert.assertEquals(1, listener.received.size()); Assert.assertEquals(tracker, listener.senders.get(0)); @@ -220,8 +221,8 @@ public class TrackerTest tracker.dropSSTables(); LogTransaction.waitForDeletions(); } - Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount()); - Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.totalDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.liveDiskSpaceUsed.getCount()); Assert.assertEquals(1, tracker.getView().sstables.size()); } if (!invalidate) @@ -249,7 +250,7 @@ public class TrackerTest Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(2)).deleting); Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(3)).removed.size()); Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(3)).added.size()); - Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.liveDiskSpaceUsed.getCount()); readers.get(0).selfRef().release(); } else @@ -313,7 +314,7 @@ public class TrackerTest listener.received.clear(); if (reader instanceof KeyCacheSupport) Assert.assertTrue(((KeyCacheSupport) reader).getKeyCache().isEnabled()); - Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 10, cfs.metric.liveDiskSpaceUsed.getCount()); // test invalidated CFS cfs = MockSchema.newCFS(); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java index 3cb982b89b..081a10d48f 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Queue; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.BeforeClass; import org.junit.Test; @@ -163,7 +164,7 @@ public class CassandraEntireSSTableStreamWriterTest .withTableId(sstable.metadata().id) .build(); - CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null), header, session); + CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null, CoordinatorLogBoundaries.NONE), header, session); SSTableTxnSingleStreamWriter sstableWriter = (SSTableTxnSingleStreamWriter) reader.read(new DataInputBuffer(serializedFile.nioBuffer(), false)); StreamingLifecycleTransaction stt = new StreamingLifecycleTransaction(); diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java index 372cd1603a..3ed00434b1 100644 --- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java @@ -44,6 +44,7 @@ import org.junit.runner.RunWith; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; @@ -234,7 +235,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest concurrentMutations.get(3, TimeUnit.MINUTES); session.prepareReceiving(new StreamSummary(sstable.metadata().id, emptyList(), 1, 5104)); - StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null); + StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null, CoordinatorLogBoundaries.NONE); try (DataInputBuffer in = new DataInputBuffer(serializedFile.nioBuffer(), false)) { diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java index 5833819267..b937ac0fd4 100644 --- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java +++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java @@ -71,6 +71,7 @@ import org.apache.cassandra.index.TargetParser; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -469,7 +470,7 @@ public class CustomCassandraIndex implements Index cell)); Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info); PartitionUpdate upd = partitionUpdate(valueKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.debug("Inserted entry into index for value {}", valueKey); } @@ -515,7 +516,7 @@ public class CustomCassandraIndex implements Index { Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion)); PartitionUpdate upd = partitionUpdate(indexKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.debug("Removed index entry for value {}", indexKey); } diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java index a9a304787b..bbf9d1a29c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java @@ -1454,10 +1454,10 @@ public abstract class CQLSSTableWriterTest } CQLSSTableWriter writer = builder.build(); - int rowCount = 30_000; + final long MiB_1_17 = 1_226_833; // Max SSTable size is 1 MiB - // 30_000 rows should take 30_000 * (4 + 37) = 1.17 MiB > 1 MiB, i.e. producing 2 sstables - for (int i = 0; i < rowCount; i++) + // write 1.17 MiB of data, producing 2 sstables + for (int i = 0; writer.bytesWritten() < MiB_1_17; i++) { writer.addRow(i, UUID.randomUUID().toString()); if (closeWriterOnFirstProduced && !produced.isEmpty()) diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 2c80779a29..c9049c2939 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -118,7 +118,7 @@ public class LegacySSTableTest // Get all versions up to the current one. Useful for testing in compatibility mode C18301 private static String[] getValidLegacyVersions() { - String[] versions = {"da", "oa", "nb", "na", "me", "md", "mc", "mb", "ma"}; + String[] versions = {"ob", "oa", "db", "da", "nb", "na", "me", "md", "mc", "mb", "ma"}; return Arrays.stream(versions).filter((v) -> v.compareTo(BigFormat.getInstance().getLatestVersion().toString()) <= 0).toArray(String[]::new); } diff --git a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java index d80377f5cc..c6601491d2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.BeforeClass; import org.junit.Test; @@ -75,6 +76,7 @@ public class RangeAwareSSTableWriterTest 0, null, false, + CoordinatorLogBoundaries.NONE, DatabaseDescriptor.getSelectedSSTableFormat(), 0, 0, diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java index 8c07d3bec6..15e692d84a 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java @@ -39,6 +39,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.SerializationHeader; @@ -133,7 +134,8 @@ public class SSTableFlushObserverTest .setMetadataCollector(new MetadataCollector(cfm.comparator).sstableLevel(0)) .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS)) .setSecondaryIndexGroups(Collections.singleton(indexGroup)) - .addDefaultComponents(Collections.emptySet()); + .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE); assertThat(observer.beginCalled).isFalse(); assertThat(observer.isComplete).isFalse(); @@ -218,6 +220,7 @@ public class SSTableFlushObserverTest .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS)) .setSecondaryIndexGroups(List.of(indexGroup1, indexGroup2)) .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(transaction, null) ).withMessage("Failed to initialize"); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 69414329cc..d6c6c747d2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; @@ -939,7 +940,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount; for ( ; i < end ; i++) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index c3729901ef..59cb4d14ce 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -232,8 +232,10 @@ public class SSTableUtils File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : dest.fileFor(Components.DATA); TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname); ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); + if (cfs.metadata().replicationType().isTracked()) + throw new IllegalStateException("Can't create writer for table with mutation tracking enabled"); SerializationHeader header = appender.header(); - SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, header); + SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, header); while (appender.append(writer)) { /* pass */ } Collection readers = writer.finish(true); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java index 49523811cb..23e5811d77 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java @@ -35,6 +35,7 @@ import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionManager; @@ -168,6 +169,7 @@ public class SSTableWriterTestBase extends SchemaLoader .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) .addDefaultComponents(cfs.indexManager.listIndexGroups()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs); } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java index b439b9c980..0e239e2c34 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.Collection; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.Assert; import org.junit.BeforeClass; @@ -72,7 +73,7 @@ public class SSTableWriterTransactionTest extends AbstractTransactionalTest private TestableBTW(Descriptor desc) { - this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, + this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))); diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java index f641be4149..e4b977e7d4 100644 --- a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java @@ -64,6 +64,7 @@ import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.CounterMutation; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -842,6 +843,7 @@ public class ScrubTest .setMetadataCollector(collector) .setSerializationHeader(header) .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs); return new TestMultiWriter(writer, txn); diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 4bb7478587..ac0a676039 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -50,6 +51,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.Throwables; @@ -154,7 +156,10 @@ public class MetadataSerializerTest collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("cba"), withNulls ? null : Int32Type.instance.decompose(234))); ByteBuffer first = AsciiType.instance.decompose("a"); ByteBuffer last = AsciiType.instance.decompose("b"); - return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last); + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + boundaries.add(new MutationId(1, 12345)); + boundaries.add(new MutationId(2, 56789)); + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, boundaries, SerializationHeader.make(cfm, Collections.emptyList()), first, last); } private void testVersions(List versions) throws Throwable diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 860f3e6ee5..dc9fa5705d 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -21,6 +21,7 @@ package org.apache.cassandra.schema; import java.io.IOException; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -40,6 +41,7 @@ import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; @@ -53,6 +55,7 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.sstable.format.big.BigTableReader; @@ -62,6 +65,7 @@ import org.apache.cassandra.io.sstable.format.bti.PartitionIndex; import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; import org.apache.cassandra.io.sstable.keycache.KeyCache; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.DataOutputStreamPlus; @@ -184,6 +188,7 @@ public class MockSchema public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime, long timestamp) { + assert !cfs.metadata().replicationType().isTracked(); SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.getKeyspaceName(), @@ -193,7 +198,7 @@ public class MockSchema if (BigFormat.is(format)) { - Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); + Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC, Components.STATS); for (Component component : components) { File file = descriptor.fileFor(component); @@ -208,17 +213,26 @@ public class MockSchema collector.update(DeletionTime.build(timestamp, minLocalDeletionTime)); BufferDecoratedKey first = readerBounds(firstToken); BufferDecoratedKey last = readerBounds(lastToken); - StatsMetadata metadata = - (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), - 0.01f, - UNREPAIRED_SSTABLE, - null, - false, - header, - first.retainable().getKey().slice(), - last.retainable().getKey().slice()) - .get(MetadataType.STATS); + Map metadataComponents = collector.sstableLevel(level) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), + 0.01f, + UNREPAIRED_SSTABLE, + null, + false, + CoordinatorLogBoundaries.NONE, + header, + first.retainable().getKey().slice(), + last.retainable().getKey().slice()); + StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS); + try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE)) + { + new StatsComponent(metadataComponents).save(descriptor); + out.flush(); + } + catch (IOException ioe) + { + throw new RuntimeException(ioe); + } BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) .setDataFile(fileHandle.sharedCopy()) @@ -226,7 +240,7 @@ public class MockSchema .setIndexSummary(indexSummary.sharedCopy()) .setFilter(FilterFactory.AlwaysPresent) .setMaxDataAge(1L) - .setStatsMetadata(metadata) + .setStatsMetadata(statsMetadata) .setOpenReason(SSTableReader.OpenReason.NORMAL) .setSerializationHeader(header) .setFirst(first) @@ -257,7 +271,7 @@ public class MockSchema BufferDecoratedKey first = readerBounds(firstToken); BufferDecoratedKey last = readerBounds(lastToken); StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, first.retainable().getKey(), last.retainable().getKey()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, CoordinatorLogBoundaries.NONE, header, first.retainable().getKey(), last.retainable().getKey()) .get(MetadataType.STATS); BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) diff --git a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java index 81645d90ea..5b872f9ddb 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; @@ -464,7 +465,8 @@ public class StreamReaderTest fakeSession, fakeSeq, System.currentTimeMillis(), - pendingRepair); + pendingRepair, + CoordinatorLogBoundaries.NONE); } private static CassandraStreamHeader streamMessageHeader(int...tokens) @@ -503,9 +505,9 @@ public class StreamReaderTest super(header, streamHeader, session); } - protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat format) throws IOException + protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format) throws IOException { - return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, format); } @Override diff --git a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java index 635970024c..840ddd6241 100644 --- a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java +++ b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java @@ -27,7 +27,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -63,7 +64,7 @@ public class StreamingInboundHandlerTest @BeforeClass public static void before() { - DatabaseDescriptor.daemonInitialization(); + SchemaLoader.prepareServer(); } @Before @@ -115,7 +116,7 @@ public class StreamingInboundHandlerTest public void StreamDeserializingTask_deserialize_ISM_NoSession() throws IOException { StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, nextTimeUUID(), true, - 0, 0, 0, nextTimeUUID()); + 0, 0, 0, nextTimeUUID(), CoordinatorLogBoundaries.NONE); ByteBuffer temp = ByteBuffer.allocate(1024); DataOutputPlus out = new DataOutputBuffer(temp); @@ -134,7 +135,7 @@ public class StreamingInboundHandlerTest StreamResultFuture future = StreamResultFuture.createFollower(0, planId, StreamOperation.REPAIR, REMOTE_ADDR, streamingChannel, MessagingService.current_version, nextTimeUUID(), PreviewKind.ALL); StreamManager.instance.registerFollower(future); StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, planId, false, - 0, 0, 0, nextTimeUUID()); + 0, 0, 0, nextTimeUUID(), CoordinatorLogBoundaries.NONE); // IncomingStreamMessage.serializer.deserialize StreamSession session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, header.sendByFollower);