From e4e19e33faf9ac7cf27a9779c8083a7f5c5b865a Mon Sep 17 00:00:00 2001 From: Branimir Lambov Date: Fri, 14 May 2021 16:13:35 +0300 Subject: [PATCH] Add memtable API (CEP-11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Branimir Lambov; reviewed by Andrés de la Peña and Caleb Rackliffe for CASSANDRA-17034 --- .build/build-rat.xml | 2 +- CHANGES.txt | 1 + NEWS.txt | 4 +- build.xml | 4 +- conf/commitlog_archiving.properties | 9 + pylib/cqlshlib/test/test_cqlsh_output.py | 1 + .../org/apache/cassandra/config/Config.java | 12 + .../cassandra/config/DatabaseDescriptor.java | 7 + .../cassandra/config/InheritingClass.java | 78 ++ .../config/YamlConfigurationLoader.java | 4 + .../statements/schema/TableAttributes.java | 5 + .../db/CassandraKeyspaceWriteHandler.java | 43 +- .../cassandra/db/ColumnFamilyStore.java | 566 +++++++++------ .../cassandra/db/DiskBoundaryManager.java | 58 +- .../org/apache/cassandra/db/Keyspace.java | 8 +- .../org/apache/cassandra/db/Memtable.java | 682 ------------------ .../db/PartitionRangeReadCommand.java | 23 +- .../apache/cassandra/db/SimpleBuilders.java | 6 + .../db/SinglePartitionReadCommand.java | 25 +- .../org/apache/cassandra/db/StorageHook.java | 2 +- .../apache/cassandra/db/SystemKeyspace.java | 7 +- .../AbstractCommitLogSegmentManager.java | 17 +- .../db/commitlog/CommitLogArchiver.java | 28 +- .../db/commitlog/CommitLogPosition.java | 18 + .../db/commitlog/CommitLogReplayer.java | 42 +- .../db/compaction/CompactionController.java | 7 +- .../db/compaction/CompactionTask.java | 2 +- .../db/lifecycle/LifecycleTransaction.java | 4 +- .../cassandra/db/lifecycle/Tracker.java | 12 +- .../apache/cassandra/db/lifecycle/View.java | 4 +- .../memtable/AbstractAllocatorMemtable.java | 306 ++++++++ .../db/memtable/AbstractMemtable.java | 222 ++++++ .../AbstractMemtableWithCommitlog.java | 125 ++++ .../cassandra/db/memtable/Flushing.java | 217 ++++++ .../cassandra/db/memtable/Memtable.java | 431 +++++++++++ .../cassandra/db/memtable/Memtable_API.md | 177 +++++ .../db/memtable/ShardBoundaries.java | 129 ++++ .../db/memtable/ShardedSkipListMemtable.java | 560 ++++++++++++++ .../db/memtable/SkipListMemtable.java | 371 ++++++++++ .../db/memtable/SkipListMemtableFactory.java | 49 ++ .../db/partitions/AbstractBTreePartition.java | 5 + .../cassandra/db/partitions/Partition.java | 5 + .../db/partitions/PartitionUpdate.java | 17 +- .../repair/CassandraValidationIterator.java | 11 +- .../db/repair/PendingAntiCompaction.java | 2 +- .../org/apache/cassandra/db/rows/Row.java | 8 + .../org/apache/cassandra/db/rows/Rows.java | 2 +- .../UnfilteredRowIteratorWithLowerBound.java | 2 +- .../cassandra/db/rows/UnfilteredSource.java | 69 ++ .../db/streaming/CassandraStreamManager.java | 4 + .../db/streaming/CassandraStreamReceiver.java | 10 +- .../apache/cassandra/db/view/TableViews.java | 4 +- .../apache/cassandra/db/view/ViewBuilder.java | 2 +- .../org/apache/cassandra/index/Index.java | 1 + .../index/SecondaryIndexManager.java | 32 +- .../index/internal/CassandraIndex.java | 8 +- .../cassandra/index/sasi/SASIIndex.java | 2 +- .../index/sasi/conf/ColumnIndex.java | 2 +- .../io/sstable/format/SSTableReader.java | 19 +- .../io/sstable/format/SSTableWriter.java | 13 + .../io/sstable/format/big/BigFormat.java | 9 + .../io/sstable/format/big/BigTableReader.java | 16 +- .../sstable/format/big/BigTableScanner.java | 7 +- .../cassandra/metrics/TableMetrics.java | 34 +- .../MemtableDiscardedNotification.java | 2 +- .../MemtableRenewedNotification.java | 2 +- .../MemtableSwitchedNotification.java | 2 +- .../SSTableAddedNotification.java | 2 +- .../repair/consistent/LocalSessions.java | 2 +- .../cassandra/schema/MemtableParams.java | 183 +++++ .../apache/cassandra/schema/SchemaEvent.java | 6 + .../cassandra/schema/SchemaKeyspace.java | 14 +- .../schema/SystemDistributedKeyspace.java | 9 +- .../cassandra/schema/TableMetadata.java | 7 + .../apache/cassandra/schema/TableParams.java | 19 + .../cassandra/service/StorageService.java | 32 +- .../service/disk/usage/DiskUsageMonitor.java | 2 +- .../uncommitted/PaxosUncommittedIndex.java | 6 +- .../uncommitted/PaxosUncommittedTracker.java | 2 +- .../cassandra/streaming/StreamSession.java | 2 +- .../cassandra/utils/memory/HeapPool.java | 7 +- .../cassandra/utils/memory/MemtablePool.java | 3 +- .../cassandra/utils/memory/NativePool.java | 4 +- .../cassandra/utils/memory/SlabPool.java | 4 +- test/conf/cassandra.yaml | 42 ++ .../distributed/fuzz/FuzzTestBase.java | 3 +- .../distributed/fuzz/SSTableGenerator.java | 3 +- .../cassandra/distributed/impl/Instance.java | 9 +- .../distributed/test/FailingRepairTest.java | 3 +- ...yInspectorCorruptSSTableExceptionTest.java | 7 +- .../distributed/test/PaxosRepairTest.java | 4 +- .../distributed/test/PaxosRepairTest2.java | 4 +- .../distributed/test/RepairTest.java | 4 +- .../SSTableLoaderEncryptionOptionsTest.java | 4 +- .../format/ForwardingSSTableReader.java | 13 +- .../apache/cassandra/cql3/ViewLongTest.java | 4 +- .../db/compaction/LongCompactionsTest.java | 2 +- .../LongLeveledCompactionStrategyCQLTest.java | 3 +- .../LongLeveledCompactionStrategyTest.java | 4 +- .../compaction/CompactionAllocationTest.java | 10 +- .../test/microbench/CacheLoaderBench.java | 2 +- .../test/microbench/CompactionBench.java | 4 +- .../ZeroCopyStreamingBenchmark.java | 2 +- .../test/microbench/instance/ReadTest.java | 205 ++---- .../instance/ReadTestSmallPartitions.java | 20 +- .../instance/SimpleTableWriter.java | 209 ++++++ .../test/microbench/instance/WriteTest.java | 92 +++ .../cluster/OnInstanceFlushAndCleanup.java | 3 +- .../cassandra/simulator/paxos/Ballots.java | 55 +- .../org/apache/cassandra/ServerTestUtils.java | 3 +- test/unit/org/apache/cassandra/Util.java | 37 +- .../batchlog/BatchlogManagerTest.java | 6 +- .../cassandra/cache/AutoSavingCacheTest.java | 2 +- .../org/apache/cassandra/cql3/CQLTester.java | 3 +- .../cassandra/cql3/GcCompactionTest.java | 4 +- .../cassandra/cql3/KeyCacheCqlTest.java | 7 +- .../cassandra/cql3/MemtableQuickTest.java | 139 ++++ .../cassandra/cql3/MemtableSizeTest.java | 93 ++- .../apache/cassandra/cql3/OutOfSpaceTest.java | 6 +- .../cql3/ViewComplexDeletionsPartialTest.java | 16 +- .../cql3/ViewComplexDeletionsTest.java | 38 +- .../cql3/ViewComplexLivenessLimitTest.java | 5 +- .../cql3/ViewComplexLivenessTest.java | 6 +- .../cassandra/cql3/ViewComplexTTLTest.java | 24 +- .../cassandra/cql3/ViewComplexTest.java | 14 +- .../cql3/ViewComplexTombstoneTest.java | 10 +- .../cql3/ViewComplexUpdatesTest.java | 50 +- .../cassandra/cql3/ViewFiltering1Test.java | 38 +- .../cassandra/cql3/ViewFiltering2Test.java | 17 +- .../apache/cassandra/cql3/ViewRangesTest.java | 3 +- .../org/apache/cassandra/cql3/ViewTest.java | 13 +- .../apache/cassandra/cql3/ViewTimesTest.java | 14 +- .../statements/DescribeStatementTest.java | 2 + .../miscellaneous/CrcCheckChanceTest.java | 12 +- .../SSTableMetadataTrackingTest.java | 15 +- .../cql3/validation/operations/AlterTest.java | 136 ++-- .../validation/operations/CreateTest.java | 161 +++-- .../cql3/validation/operations/TTLTest.java | 3 +- .../org/apache/cassandra/db/CleanupTest.java | 4 +- .../cassandra/db/CleanupTransientTest.java | 2 +- .../cassandra/db/ColumnFamilyMetricTest.java | 6 +- .../cassandra/db/ColumnFamilyStoreTest.java | 28 +- .../cassandra/db/DeletePartitionTest.java | 4 +- .../org/apache/cassandra/db/ImportTest.java | 41 +- .../org/apache/cassandra/db/KeyCacheTest.java | 10 +- .../org/apache/cassandra/db/KeyspaceTest.java | 22 +- .../cassandra/db/MultiKeyspaceTest.java | 6 +- .../org/apache/cassandra/db/NameSortTest.java | 2 +- .../cassandra/db/PartitionRangeReadTest.java | 6 +- .../cassandra/db/RangeTombstoneTest.java | 46 +- .../apache/cassandra/db/ReadCommandTest.java | 44 +- .../db/RecoveryManagerFlushedTest.java | 7 +- .../apache/cassandra/db/RemoveCellTest.java | 3 +- .../org/apache/cassandra/db/RowCacheTest.java | 2 +- .../apache/cassandra/db/RowIterationTest.java | 8 +- .../cassandra/db/SchemaCQLHelperTest.java | 1 + .../org/apache/cassandra/db/ScrubTest.java | 12 +- .../cassandra/db/SecondaryIndexTest.java | 8 +- .../db/SinglePartitionReadCommandCQLTest.java | 4 +- .../db/SinglePartitionSliceCommandTest.java | 22 +- .../org/apache/cassandra/db/SnapshotTest.java | 2 +- .../org/apache/cassandra/db/TimeSortTest.java | 4 +- .../org/apache/cassandra/db/VerifyTest.java | 6 +- .../SSTableReverseIteratorTest.java | 3 +- .../db/commitlog/CommitLogCQLTest.java | 2 +- .../db/commitlog/CommitLogReaderTest.java | 5 +- .../CommitLogSegmentManagerCDCTest.java | 5 +- .../cassandra/db/commitlog/CommitLogTest.java | 35 +- .../AbstractCompactionStrategyTest.java | 2 +- .../compaction/AbstractPendingRepairTest.java | 3 +- .../db/compaction/ActiveCompactionsTest.java | 13 +- .../compaction/AntiCompactionBytemanTest.java | 2 +- .../db/compaction/AntiCompactionTest.java | 6 +- .../db/compaction/CancelCompactionsTest.java | 2 +- .../compaction/CompactionControllerTest.java | 12 +- .../db/compaction/CompactionIteratorTest.java | 2 +- .../CompactionStrategyManagerTest.java | 2 +- .../db/compaction/CompactionTaskTest.java | 21 +- .../db/compaction/CompactionsBytemanTest.java | 8 +- .../db/compaction/CompactionsCQLTest.java | 35 +- .../db/compaction/CompactionsPurgeTest.java | 52 +- .../db/compaction/CompactionsTest.java | 16 +- .../CorruptedSSTablesCompactionsTest.java | 2 +- .../DateTieredCompactionStrategyTest.java | 16 +- .../LeveledCompactionStrategyTest.java | 24 +- .../db/compaction/NeverPurgeTest.java | 7 +- .../db/compaction/OneCompactionTest.java | 2 +- .../compaction/SingleSSTableLCSTaskTest.java | 7 +- .../SizeTieredCompactionStrategyTest.java | 5 +- .../db/compaction/TTLExpiryTest.java | 35 +- .../TimeWindowCompactionStrategyTest.java | 16 +- .../writers/CompactionAwareWriterTest.java | 3 +- .../lifecycle/LifecycleTransactionTest.java | 9 +- .../cassandra/db/lifecycle/TrackerTest.java | 21 +- .../cassandra/db/lifecycle/ViewTest.java | 2 +- .../cassandra/db/memtable/TestMemtable.java | 35 + .../AbstractPendingAntiCompactionTest.java | 3 +- ...onManagerGetSSTablesForValidationTest.java | 3 +- .../db/repair/PendingAntiCompactionTest.java | 6 +- .../rows/ThrottledUnfilteredIteratorTest.java | 8 +- ...assandraEntireSSTableStreamWriterTest.java | 3 +- .../streaming/CassandraOutgoingFileTest.java | 3 +- .../streaming/CassandraStreamHeaderTest.java | 3 +- .../streaming/CassandraStreamManagerTest.java | 3 +- ...StreamConcurrentComponentMutationTest.java | 3 +- .../db/transform/DuplicateRowCheckerTest.java | 2 +- .../db/view/ViewBuilderTaskTest.java | 5 +- .../cassandra/index/CustomIndexTest.java | 10 +- .../index/internal/CustomCassandraIndex.java | 9 +- .../cassandra/index/sasi/SASIIndexTest.java | 39 +- .../cassandra/io/DiskSpaceMetricsTest.java | 3 +- .../CompressedSequentialWriterReopenTest.java | 4 +- .../io/sstable/IndexSummaryManagerTest.java | 6 +- .../IndexSummaryRedistributionTest.java | 2 +- .../io/sstable/LegacySSTableTest.java | 2 +- .../SSTableCorruptionDetectionTest.java | 12 +- .../io/sstable/SSTableLoaderTest.java | 10 +- .../io/sstable/SSTableMetadataTest.java | 20 +- .../io/sstable/SSTableReaderTest.java | 30 +- .../io/sstable/SSTableRewriterTest.java | 4 +- .../io/sstable/SSTableScannerTest.java | 13 +- .../io/sstable/SSTableWriterTest.java | 10 +- .../format/RangeAwareSSTableWriterTest.java | 4 +- .../big/BigTableZeroCopyWriterTest.java | 12 +- .../cassandra/repair/ValidatorTest.java | 7 +- .../consistent/PendingRepairStatTest.java | 3 +- .../cassandra/schema/MemtableParamsTest.java | 200 +++++ .../schema/MigrationManagerDropKSTest.java | 3 +- .../schema/MigrationManagerTest.java | 10 +- .../apache/cassandra/schema/MockSchema.java | 4 +- .../service/ActiveRepairServiceTest.java | 2 +- .../cassandra/service/ClientWarningsTest.java | 5 +- .../uncommitted/PaxosMockUpdateSupplier.java | 2 +- ...axosUncommittedTrackerIntegrationTest.java | 4 +- .../reads/range/RangeCommandIteratorTest.java | 2 +- ...SSTableStreamingCorrectFilesCountTest.java | 3 +- .../streaming/StreamTransferTaskTest.java | 6 +- .../streaming/StreamingTransferTest.java | 6 +- .../StandaloneSplitterWithCQLTesterTest.java | 2 +- .../StandaloneUpgraderOnSStablesTest.java | 2 +- .../StandaloneVerifierOnSSTablesTest.java | 2 +- 241 files changed, 5637 insertions(+), 2018 deletions(-) create mode 100644 src/java/org/apache/cassandra/config/InheritingClass.java delete mode 100644 src/java/org/apache/cassandra/db/Memtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/AbstractMemtableWithCommitlog.java create mode 100644 src/java/org/apache/cassandra/db/memtable/Flushing.java create mode 100644 src/java/org/apache/cassandra/db/memtable/Memtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/Memtable_API.md create mode 100644 src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java create mode 100644 src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java create mode 100644 src/java/org/apache/cassandra/db/memtable/SkipListMemtableFactory.java create mode 100644 src/java/org/apache/cassandra/db/rows/UnfilteredSource.java create mode 100644 src/java/org/apache/cassandra/schema/MemtableParams.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/instance/WriteTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java create mode 100644 test/unit/org/apache/cassandra/db/memtable/TestMemtable.java create mode 100644 test/unit/org/apache/cassandra/schema/MemtableParamsTest.java diff --git a/.build/build-rat.xml b/.build/build-rat.xml index 3c7736b1ef..5632664486 100644 --- a/.build/build-rat.xml +++ b/.build/build-rat.xml @@ -79,7 +79,7 @@ - + diff --git a/CHANGES.txt b/CHANGES.txt index 412e7964f8..b0c0ba4f97 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * Add a pluggable memtable API (CEP-11 / CASSANDRA-17034) * Save sstable id as string in activity table (CASSANDRA-17585) * Implement startup check to prevent Cassandra to potentially spread zombie data (CASSANDRA-17180) * Allow failing startup on duplicate config keys (CASSANDRA-17379) diff --git a/NEWS.txt b/NEWS.txt index d5a05a77a4..1280d02379 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -56,6 +56,8 @@ using the provided 'sstableupgrade' tool. New features ------------ + - Added API for alternative memtable implementations. For details, see + src/java/org/apache/cassandra/db/memtable/Memtable_API.md - Added a new guardrails framework allowing to define soft/hard limits for different user actions, such as limiting the number of tables, columns per table or the size of collections. These guardrails are only applied to regular user queries, and superusers and internal queries are excluded. Reaching the soft limit raises a client warning, @@ -88,7 +90,7 @@ New features - Whether querying with ALLOW FILTERING is allowed. - Add support for the use of pure monotonic functions on the last attribute of the GROUP BY clause. - Add floor functions that can be use to group by time range. - - Support for native transport rate limiting via native_transport_rate_limiting_enabled and + - Support for native transport rate limiting via native_transport_rate_limiting_enabled and native_transport_max_requests_per_second in cassandra.yaml. - Support for pre hashing passwords on CQL DCL commands - Expose all client options via system_views.clients and nodetool clientstats. diff --git a/build.xml b/build.xml index eb5e3f187c..343f7e87a0 100644 --- a/build.xml +++ b/build.xml @@ -1369,10 +1369,12 @@ - + + + diff --git a/conf/commitlog_archiving.properties b/conf/commitlog_archiving.properties index 393259c8ed..1488ced5c4 100644 --- a/conf/commitlog_archiving.properties +++ b/conf/commitlog_archiving.properties @@ -44,5 +44,14 @@ restore_directories= # or equal to this timestamp will be applied. restore_point_in_time= +# Snapshot commit log position override. This should not be normally necessary, unless the snapshot used a method other +# than sstables to store data (e.g. persistent memtable was restored from snapshot). +# Format: segmentId, position +# +# Recovery will not replay any commit log data before the specified commit log position. It will NOT exclude the +# intervals covered by existing sstables from the replay interval, i.e. it will replay data that may already be in +# sstables. +snapshot_commitlog_position= + # precision of the timestamp used in the inserts (MILLISECONDS, MICROSECONDS, ...) precision=MICROSECONDS diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 14c2b258f1..c430f46766 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -658,6 +658,7 @@ class TestCqlshOutput(BaseTestCase): AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND memtable = 'default' AND crc_check_chance = 1.0 AND default_time_to_live = 0 AND extensions = {} diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 1dedb6b817..9bf9dfffe5 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -174,6 +175,17 @@ public class Config public SmallestDataStorageMebibytes memtable_offheap_space; public Float memtable_cleanup_threshold = null; + public static class MemtableOptions + { + public LinkedHashMap configurations; // order must be preserved + + public MemtableOptions() + { + } + } + + public MemtableOptions memtable; + // Limit the maximum depth of repair session merkle trees @Deprecated public volatile Integer repair_session_max_tree_depth = null; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index a264cbc847..5c661321a3 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3326,6 +3326,13 @@ public class DatabaseDescriptor return conf.memtable_cleanup_threshold; } + public static Map getMemtableConfigurations() + { + if (conf == null || conf.memtable == null) + return null; + return conf.memtable.configurations; + } + public static int getIndexSummaryResizeIntervalInMinutes() { return conf.index_summary_resize_interval.toMinutesAsInt(); diff --git a/src/java/org/apache/cassandra/config/InheritingClass.java b/src/java/org/apache/cassandra/config/InheritingClass.java new file mode 100644 index 0000000000..c0bc41f513 --- /dev/null +++ b/src/java/org/apache/cassandra/config/InheritingClass.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.cassandra.exceptions.ConfigurationException; + +public class InheritingClass extends ParameterizedClass +{ + public String inherits = null; + + @SuppressWarnings("unused") // for snakeyaml + public InheritingClass() + { + } + + public InheritingClass(String inherits, String class_name, Map parameters) + { + super(class_name, parameters); + this.inherits = inherits; + } + + @SuppressWarnings("unused") + public InheritingClass(Map p) + { + super(p); + this.inherits = p.get("inherits").toString(); + } + + public ParameterizedClass resolve(Map map) + { + if (inherits == null) + return this; + ParameterizedClass parent = map.get(inherits); + if (parent == null) + throw new ConfigurationException("Configuration definition inherits unknown " + inherits + + ". A configuration can only extend one defined earlier or \"default\"."); + Map resolvedParameters; + if (parameters == null || parameters.isEmpty()) + resolvedParameters = parent.parameters; + else if (parent.parameters == null || parent.parameters.isEmpty()) + resolvedParameters = this.parameters; + else + { + resolvedParameters = new LinkedHashMap<>(parent.parameters); + resolvedParameters.putAll(this.parameters); + } + + String resolvedClass = this.class_name == null ? parent.class_name : this.class_name; + return new ParameterizedClass(resolvedClass, resolvedParameters); + } + + @Override + public String toString() + { + return (inherits != null ? (inherits + "+") : "") + + (class_name != null ? class_name : "") + + (parameters != null ? parameters.toString() : ""); + } +} diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java index 0e147aaff0..b40bb9139e 100644 --- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java +++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java @@ -324,6 +324,10 @@ public class YamlConfigurationLoader implements ConfigurationLoader TypeDescription seedDesc = new TypeDescription(ParameterizedClass.class); seedDesc.putMapPropertyType("parameters", String.class, String.class); addTypeDescription(seedDesc); + + TypeDescription memtableDesc = new TypeDescription(Config.MemtableOptions.class); + memtableDesc.addPropertyParameters("configurations", String.class, InheritingClass.class); + addTypeDescription(memtableDesc); } @Override diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java index caeb720f46..fd31e43ec0 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/TableAttributes.java @@ -20,6 +20,7 @@ package org.apache.cassandra.cql3.statements.schema; import java.util.Map; import java.util.Set; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -29,6 +30,7 @@ import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.CompressionParams; +import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableParams; import org.apache.cassandra.schema.TableParams.Option; @@ -121,6 +123,9 @@ public final class TableAttributes extends PropertyDefinitions builder.compression(CompressionParams.fromMap(getMap(Option.COMPRESSION))); } + if (hasOption(Option.MEMTABLE)) + builder.memtable(MemtableParams.get(getString(Option.MEMTABLE))); + if (hasOption(Option.DEFAULT_TIME_TO_LIVE)) builder.defaultTimeToLive(getInt(Option.DEFAULT_TIME_TO_LIVE)); diff --git a/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java index efba11f1a4..f2cf93cc7e 100644 --- a/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java +++ b/src/java/org/apache/cassandra/db/CassandraKeyspaceWriteHandler.java @@ -18,9 +18,14 @@ package org.apache.cassandra.db; +import java.util.HashSet; +import java.util.Set; + import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.schema.TableId; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -46,8 +51,7 @@ public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler CommitLogPosition position = null; if (makeDurable) { - Tracing.trace("Appending to commitlog"); - position = CommitLog.instance.add(mutation); + position = addToCommitLog(mutation); } return new CassandraWriteContext(group, position); } @@ -61,6 +65,41 @@ public class CassandraKeyspaceWriteHandler implements KeyspaceWriteHandler } } + private CommitLogPosition addToCommitLog(Mutation mutation) + { + // Usually one of these will be true, so first check if that's the case. + boolean allSkipCommitlog = true; + boolean noneSkipCommitlog = true; + for (PartitionUpdate update : mutation.getPartitionUpdates()) + { + if (update.metadata().params.memtable.factory().writesShouldSkipCommitLog()) + noneSkipCommitlog = false; + else + allSkipCommitlog = false; + } + + if (!noneSkipCommitlog) + { + if (allSkipCommitlog) + return null; + else + { + Set ids = new HashSet<>(); + for (PartitionUpdate update : mutation.getPartitionUpdates()) + { + if (update.metadata().params.memtable.factory().writesShouldSkipCommitLog()) + ids.add(update.metadata().id); + } + mutation = mutation.without(ids); + } + } + // Note: It may be a good idea to precalculate none/all for the set of all tables in the keyspace, + // or memoize the mutation.getTableIds()->ids map (needs invalidation on schema version change). + + Tracing.trace("Appending to commitlog"); + return CommitLog.instance.add(mutation); + } + @SuppressWarnings("resource") // group is closed when CassandraWriteContext is closed private WriteContext createEmptyContext() { diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 5db768ca6e..2bdac2b0e4 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -28,9 +28,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -42,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -61,6 +64,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -75,7 +79,6 @@ import org.apache.cassandra.cache.RowCacheKey; import org.apache.cassandra.cache.RowCacheSentinel; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.FutureTask; -import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.commitlog.CommitLog; @@ -87,6 +90,9 @@ import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.filter.ClusteringIndexFilter; import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.memtable.Flushing; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.ShardBoundaries; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.lifecycle.SSTableSet; @@ -102,6 +108,7 @@ import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.StartupException; @@ -112,6 +119,7 @@ import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableIdFactory; import org.apache.cassandra.io.sstable.SSTableMultiWriter; @@ -159,14 +167,11 @@ import org.apache.cassandra.utils.MBeanWrapper; import org.apache.cassandra.utils.NoSpamLogger; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.WrappedRunnable; -import org.apache.cassandra.utils.concurrent.AsyncPromise; import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.concurrent.Promise; import org.apache.cassandra.utils.concurrent.Refs; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; -import org.apache.cassandra.utils.memory.MemtableAllocator; import static com.google.common.base.Throwables.propagate; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -180,7 +185,7 @@ import static org.apache.cassandra.utils.Throwables.maybeFail; import static org.apache.cassandra.utils.Throwables.merge; import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; -public class ColumnFamilyStore implements ColumnFamilyStoreMBean +public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner { private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class); @@ -209,6 +214,35 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations(), DatabaseDescriptor.useSpecificLocationForLocalSystemData()); + /** + * Reason for initiating a memtable flush. + */ + public enum FlushReason + { + COMMITLOG_DIRTY, + MEMTABLE_LIMIT, + MEMTABLE_PERIOD_EXPIRED, + INDEX_BUILD_STARTED, + INDEX_BUILD_COMPLETED, + INDEX_REMOVED, + INDEX_TABLE_FLUSH, + VIEW_BUILD_STARTED, + INTERNALLY_FORCED, // explicitly requested flush, necessary for the operation of an internal table + USER_FORCED, // flush explicitly requested by the user (e.g. nodetool flush) + STARTUP, + DRAIN, + SNAPSHOT, + TRUNCATE, + DROP, + STREAMING, + STREAMS_RECEIVED, + VALIDATION, + ANTICOMPACTION, + SCHEMA_CHANGE, + OWNED_RANGES_CHANGE, + UNIT_TESTS // explicitly requested flush needed for a test + } + private static final String[] COUNTER_NAMES = new String[]{"table", "count", "error", "value"}; private static final String[] COUNTER_DESCS = new String[] { "keyspace.tablename", @@ -243,6 +277,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean private final String oldMBeanName; private volatile boolean valid = true; + private volatile Memtable.Factory memtableFactory; + /** * Memtables and SSTables on disk for this column family. * @@ -287,6 +323,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean @VisibleForTesting final DiskBoundaryManager diskBoundaryManager = new DiskBoundaryManager(); + private volatile ShardBoundaries cachedShardBoundaries = null; private volatile boolean neverPurgeTombstones = false; @@ -344,48 +381,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean compactionStrategyManager.maybeReload(metadata()); - scheduleFlush(); - indexManager.reload(); - // If the CF comparator has changed, we need to change the memtable, - // because the old one still aliases the previous comparator. - if (data.getView().getCurrentMemtable().initialComparator != metadata().comparator) - switchMemtable(); - } - - void scheduleFlush() - { - int period = metadata().params.memtableFlushPeriodInMs; - if (period > 0) - { - logger.trace("scheduling flush in {} ms", period); - WrappedRunnable runnable = new WrappedRunnable() - { - protected void runMayThrow() - { - synchronized (data) - { - Memtable current = data.getView().getCurrentMemtable(); - // if we're not expired, we've been hit by a scheduled flush for an already flushed memtable, so ignore - if (current.isExpired()) - { - if (current.isClean()) - { - // if we're still clean, instead of swapping just reschedule a flush for later - scheduleFlush(); - } - else - { - // we'll be rescheduled by the constructor of the Memtable. - forceFlush(); - } - } - } - } - }; - ScheduledExecutors.scheduledTasks.scheduleSelfRecurring(runnable, period, TimeUnit.MILLISECONDS); - } + memtableFactory = metadata().params.memtable.factory(); + switchMemtableOrNotify(FlushReason.SCHEMA_CHANGE, Memtable::metadataUpdated); } public static Runnable getBackgroundCompactionTaskSubmitter() @@ -481,14 +480,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean this.sstableIdGenerator = sstableIdGenerator; sampleReadLatencyNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) / 2; additionalWriteLatencyNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) / 2; + memtableFactory = metadata.get().params.memtable.factory(); logger.info("Initializing {}.{}", keyspace.getName(), name); - // Create Memtable only on online + // Create Memtable and its metrics object only on online Memtable initialMemtable = null; + TableMetrics.ReleasableMetric memtableMetrics = null; if (DatabaseDescriptor.isDaemonInitialized()) - initialMemtable = new Memtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()), this); - data = new Tracker(initialMemtable, loadSSTables); + { + initialMemtable = createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition())); + memtableMetrics = memtableFactory.createMemtableMetrics(metadata); + } + data = new Tracker(this, initialMemtable, loadSSTables); // Note that this needs to happen before we load the first sstables, or the global sstable tracker will not // be notified on the initial loading. @@ -519,7 +523,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean indexManager.addIndex(info, true); } - metric = new TableMetrics(this); + metric = new TableMetrics(this, memtableMetrics); if (data.loadsstables) { @@ -615,6 +619,26 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return dataPaths; } + public boolean writesShouldSkipCommitLog() + { + return memtableFactory.writesShouldSkipCommitLog(); + } + + public boolean memtableWritesAreDurable() + { + return memtableFactory.writesAreDurable(); + } + + public boolean streamToMemtable() + { + return memtableFactory.streamToMemtable(); + } + + public boolean streamFromMemtable() + { + return memtableFactory.streamFromMemtable(); + } + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker) { MetadataCollector collector = new MetadataCollector(metadata().comparator).sstableLevel(sstableLevel); @@ -924,17 +948,30 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return newDescriptor; } + /** + * Checks with the memtable if it should be switched for the given reason, and if not, calls the specified + * notification method. + */ + private void switchMemtableOrNotify(FlushReason reason, Consumer elseNotify) + { + Memtable currentMemtable = data.getView().getCurrentMemtable(); + if (currentMemtable.shouldSwitch(reason)) + switchMemtableIfCurrent(currentMemtable, reason); + else + elseNotify.accept(currentMemtable); + } + /** * Switches the memtable iff the live memtable is the one provided * * @param memtable */ - public Future switchMemtableIfCurrent(Memtable memtable) + public Future switchMemtableIfCurrent(Memtable memtable, FlushReason reason) { synchronized (data) { if (data.getView().getCurrentMemtable() == memtable) - return switchMemtable(); + return switchMemtable(reason); } logger.debug("Memtable is no longer current, returning future that completes when current flushing operation completes"); return waitForFlushes(); @@ -947,11 +984,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * not complete until the Memtable (and all prior Memtables) have been successfully flushed, and the CL * marked clean up to the position owned by the Memtable. */ - public Future switchMemtable() + @VisibleForTesting + public Future switchMemtable(FlushReason reason) { synchronized (data) { - logFlush(); + logFlush(reason); Flush flush = new Flush(false); flushExecutor.execute(flush); postFlushExecutor.execute(flush.postFlushTask); @@ -960,33 +998,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } // print out size of all memtables we're enqueuing - private void logFlush() + private void logFlush(FlushReason reason) { // reclaiming includes that which we are GC-ing; - float onHeapRatio = 0, offHeapRatio = 0; - long onHeapTotal = 0, offHeapTotal = 0; - Memtable memtable = getTracker().getView().getCurrentMemtable(); - onHeapRatio += memtable.getAllocator().onHeap().ownershipRatio(); - offHeapRatio += memtable.getAllocator().offHeap().ownershipRatio(); - onHeapTotal += memtable.getAllocator().onHeap().owns(); - offHeapTotal += memtable.getAllocator().offHeap().owns(); + Memtable.MemoryUsage usage = Memtable.newMemoryUsage(); + getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage); for (ColumnFamilyStore indexCfs : indexManager.getAllIndexColumnFamilyStores()) - { - MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator(); - onHeapRatio += allocator.onHeap().ownershipRatio(); - offHeapRatio += allocator.offHeap().ownershipRatio(); - onHeapTotal += allocator.onHeap().owns(); - offHeapTotal += allocator.offHeap().owns(); - } + indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage); - logger.info("Enqueuing flush of {}: {}", - name, - String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap", - FBUtilities.prettyPrintMemory(onHeapTotal), - onHeapRatio * 100, - FBUtilities.prettyPrintMemory(offHeapTotal), - offHeapRatio * 100)); + logger.info("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", keyspace.getName(), name, reason, usage); } @@ -996,14 +1017,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * @return a Future yielding the commit log position that can be guaranteed to have been successfully written * to sstables for this table once the future completes */ - public Future forceFlush() + public Future forceFlush(FlushReason reason) { synchronized (data) { Memtable current = data.getView().getCurrentMemtable(); for (ColumnFamilyStore cfs : concatWithIndexes()) if (!cfs.data.getView().getCurrentMemtable().isClean()) - return switchMemtableIfCurrent(current); + return flushMemtable(current, reason); return waitForFlushes(); } } @@ -1021,10 +1042,18 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean // and this does not vary between a table and its table-backed indexes Memtable current = data.getView().getCurrentMemtable(); if (current.mayContainDataBefore(flushIfDirtyBefore)) - return switchMemtableIfCurrent(current); + return flushMemtable(current, FlushReason.COMMITLOG_DIRTY); return waitForFlushes(); } + private Future flushMemtable(Memtable current, FlushReason reason) + { + if (current.shouldSwitch(reason)) + return switchMemtableIfCurrent(current, reason); + else + return waitForFlushes(); + } + /** * @return a Future yielding the commit log position that can be guaranteed to have been successfully written * to sstables for this table once the future completes @@ -1034,15 +1063,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean // we grab the current memtable; once any preceding memtables have flushed, we know its // commitLogLowerBound has been set (as this it is set with the upper bound of the preceding memtable) final Memtable current = data.getView().getCurrentMemtable(); - return postFlushExecutor.submit(() -> { - logger.debug("forceFlush requested but everything is clean in {}", name); - return current.getCommitLogLowerBound(); - }); + return postFlushExecutor.submit(current::getCommitLogLowerBound); } - public CommitLogPosition forceBlockingFlush() + public CommitLogPosition forceBlockingFlush(FlushReason reason) { - return FBUtilities.waitOnFuture(forceFlush()); + return FBUtilities.waitOnFuture(forceFlush(reason)); } /** @@ -1052,12 +1078,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean private final class PostFlush implements Callable { final CountDownLatch latch = newCountDownLatch(1); - final List memtables; + final Memtable mainMemtable; volatile Throwable flushFailure = null; - private PostFlush(List memtables) + private PostFlush(Memtable mainMemtable) { - this.memtables = memtables; + this.mainMemtable = mainMemtable; } public CommitLogPosition call() @@ -1075,11 +1101,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean CommitLogPosition commitLogUpperBound = NONE; // If a flush errored out but the error was ignored, make sure we don't discard the commit log. - if (flushFailure == null && !memtables.isEmpty()) + if (flushFailure == null && mainMemtable != null) { - Memtable memtable = memtables.get(0); - commitLogUpperBound = memtable.getCommitLogUpperBound(); - CommitLog.instance.discardCompletedSegments(metadata.id, memtable.getCommitLogLowerBound(), commitLogUpperBound); + commitLogUpperBound = mainMemtable.getCommitLogUpperBound(); + CommitLog.instance.discardCompletedSegments(metadata.id, mainMemtable.getCommitLogLowerBound(), commitLogUpperBound); } metric.pendingFlushes.dec(); @@ -1102,7 +1127,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean private final class Flush implements Runnable { final OpOrder.Barrier writeBarrier; - final List memtables = new ArrayList<>(); + final Map memtables; final FutureTask postFlushTask; final PostFlush postFlush; final boolean truncate; @@ -1126,6 +1151,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean */ writeBarrier = Keyspace.writeOrder.newBarrier(); + memtables = new LinkedHashMap<>(); + // submit flushes for the memtable for any indexed sub-cfses, and our own AtomicReference commitLogUpperBound = new AtomicReference<>(); for (ColumnFamilyStore cfs : concatWithIndexes()) @@ -1133,10 +1160,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean // switch all memtables, regardless of their dirty status, setting the barrier // so that we can reach a coordinated decision about cleanliness once they // are no longer possible to be modified - Memtable newMemtable = new Memtable(commitLogUpperBound, cfs); + Memtable newMemtable = cfs.createMemtable(commitLogUpperBound); Memtable oldMemtable = cfs.data.switchMemtable(truncate, newMemtable); - oldMemtable.setDiscarding(writeBarrier, commitLogUpperBound); - memtables.add(oldMemtable); + oldMemtable.switchOut(writeBarrier, commitLogUpperBound); + memtables.put(cfs, oldMemtable); } // we then ensure an atomic decision is made about the upper bound of the continuous range of commit log @@ -1147,7 +1174,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean // since this happens after wiring up the commitLogUpperBound, we also know all operations with earlier // commit log segment position have also completed, i.e. the memtables are done and ready to flush writeBarrier.issue(); - postFlush = new PostFlush(memtables); + postFlush = new PostFlush(Iterables.get(memtables.values(), 0, null)); postFlushTask = new FutureTask<>(postFlush); } @@ -1167,17 +1194,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean logger.trace("Flush task for task {}@{} waited {} ms at the barrier", hashCode(), name, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start)); // mark all memtables as flushing, removing them from the live memtable list - for (Memtable memtable : memtables) - memtable.cfs.data.markFlushing(memtable); + for (Map.Entry entry : memtables.entrySet()) + entry.getKey().data.markFlushing(entry.getValue()); metric.memtableSwitchCount.inc(); try { + boolean first = true; // Flush "data" memtable with non-cf 2i first; - flushMemtable(memtables.get(0), true); - for (int i = 1; i < memtables.size(); i++) - flushMemtable(memtables.get(i), false); + for (Map.Entry entry : memtables.entrySet()) + { + flushMemtable(entry.getKey(), entry.getValue(), first); + first = false; + } } catch (Throwable t) { @@ -1195,14 +1225,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean logger.trace("Flush task task {}@{} finished", hashCode(), name); } - public Collection flushMemtable(Memtable memtable, boolean flushNonCf2i) + public Collection flushMemtable(ColumnFamilyStore cfs, Memtable memtable, boolean flushNonCf2i) { if (logger.isTraceEnabled()) logger.trace("Flush task task {}@{} flushing memtable {}", hashCode(), name, memtable); if (memtable.isClean() || truncate) { - memtable.cfs.replaceFlushed(memtable, Collections.emptyList()); + cfs.replaceFlushed(memtable, Collections.emptyList()); reclaim(memtable); return Collections.emptyList(); } @@ -1214,13 +1244,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean List sstables = new ArrayList<>(); try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.FLUSH)) { - List flushRunnables = null; + List flushRunnables = null; List flushResults = null; try { // flush the memtable - flushRunnables = memtable.flushRunnables(txn); + flushRunnables = Flushing.flushRunnables(cfs, memtable, txn); ExecutorPlus[] executors = perDiskflushExecutors.getExecutorsFor(keyspace.getName(), name); for (int i = 0; i < flushRunnables.size(); i++) @@ -1239,7 +1269,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } catch (Throwable t) { - t = memtable.abortRunnables(flushRunnables, t); + t = Flushing.abortRunnables(flushRunnables, t); t = txn.abort(t); throw Throwables.propagate(t); } @@ -1294,9 +1324,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } } - memtable.cfs.replaceFlushed(memtable, sstables); + cfs.replaceFlushed(memtable, sstables); reclaim(memtable); - memtable.cfs.compactionStrategyManager.compactionLogger.flush(sstables); + cfs.compactionStrategyManager.compactionLogger.flush(sstables); logger.debug("Flushed to {} ({} sstables, {}), biggest {}, smallest {}", sstables, sstables.size(), @@ -1316,7 +1346,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean public void runMayThrow() { readBarrier.await(); - memtable.setDiscarded(); + memtable.discard(); } }, reclaimExecutor); } @@ -1328,6 +1358,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } + public Memtable createMemtable(AtomicReference commitLogUpperBound) + { + return memtableFactory.create(commitLogUpperBound, metadata, this); + } + // atomically set the upper bound for the commit log private static void setCommitLogUpperBound(AtomicReference commitLogUpperBound) { @@ -1345,86 +1380,29 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } - /** - * Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately - * queues it for flushing. If the memtable selected is flushed before this completes, no work is done. - */ - public static Future flushLargestMemtable() + @Override + public Future signalFlushRequired(Memtable memtable, FlushReason reason) { - float largestRatio = 0f; - Memtable largest = null; - float liveOnHeap = 0, liveOffHeap = 0; - for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) - { - // we take a reference to the current main memtable for the CF prior to snapping its ownership ratios - // to ensure we have some ordering guarantee for performing the switchMemtableIf(), i.e. we will only - // swap if the memtables we are measuring here haven't already been swapped by the time we try to swap them - Memtable current = cfs.getTracker().getView().getCurrentMemtable(); - - // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF, - // both on- and off-heap, and select the largest of the two ratios to weight this CF - float onHeap = 0f, offHeap = 0f; - onHeap += current.getAllocator().onHeap().ownershipRatio(); - offHeap += current.getAllocator().offHeap().ownershipRatio(); - - for (ColumnFamilyStore indexCfs : cfs.indexManager.getAllIndexColumnFamilyStores()) - { - MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator(); - onHeap += allocator.onHeap().ownershipRatio(); - offHeap += allocator.offHeap().ownershipRatio(); - } - - float ratio = Math.max(onHeap, offHeap); - if (ratio > largestRatio) - { - largest = current; - largestRatio = ratio; - } - - liveOnHeap += onHeap; - liveOffHeap += offHeap; - } - - Promise returnFuture = new AsyncPromise<>(); - - if (largest != null) - { - float usedOnHeap = Memtable.MEMORY_POOL.onHeap.usedRatio(); - float usedOffHeap = Memtable.MEMORY_POOL.offHeap.usedRatio(); - float flushingOnHeap = Memtable.MEMORY_POOL.onHeap.reclaimingRatio(); - float flushingOffHeap = Memtable.MEMORY_POOL.offHeap.reclaimingRatio(); - float thisOnHeap = largest.getAllocator().onHeap().ownershipRatio(); - float thisOffHeap = largest.getAllocator().offHeap().ownershipRatio(); - logger.info("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}", - largest.cfs, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap), - ratio(flushingOnHeap, flushingOffHeap), ratio(thisOnHeap, thisOffHeap)); - - Future flushFuture = largest.cfs.switchMemtableIfCurrent(largest); - flushFuture.addListener(() -> { - try - { - flushFuture.get(); - returnFuture.trySuccess(true); - } - catch (Throwable t) - { - returnFuture.tryFailure(t); - } - }); - } - else - { - logger.debug("Flushing of largest memtable, not done, no memtable found"); - - returnFuture.trySuccess(false); - } - - return returnFuture; + return switchMemtableIfCurrent(memtable, reason); } - private static String ratio(float onHeap, float offHeap) + @Override + public Memtable getCurrentMemtable() { - return String.format("%.2f/%.2f", onHeap, offHeap); + return data.getView().getCurrentMemtable(); + } + + public static Iterable activeMemtables() + { + return Iterables.transform(ColumnFamilyStore.all(), + cfs -> cfs.getTracker().getView().getCurrentMemtable()); + } + + @Override + public Iterable getIndexMemtables() + { + return Iterables.transform(indexManager.getAllIndexColumnFamilyStores(), + cfs -> cfs.getTracker().getView().getCurrentMemtable()); } /** @@ -1465,6 +1443,44 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } + @Override + public ShardBoundaries localRangeSplits(int shardCount) + { + if (shardCount == 1 || !getPartitioner().splitter().isPresent() || SchemaConstants.isLocalSystemKeyspace(keyspace.getName())) + return ShardBoundaries.NONE; + + ShardBoundaries shardBoundaries = cachedShardBoundaries; + if (shardBoundaries == null || + shardBoundaries.shardCount() != shardCount || + shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion()) + { + DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this); + Set> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges(); + List weightedRanges; + if (localRanges.isEmpty()) + weightedRanges = ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken()))); + else + { + weightedRanges = new ArrayList<>(localRanges.size()); + for (Range r : localRanges) + { + // WeightedRange supports only unwrapped ranges as it relies + // on right - left == num tokens equality + for (Range u: r.unwrap()) + weightedRanges.add(new Splitter.WeightedRange(1.0, u)); + } + weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left)); + } + + List boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false); + shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1), + versionedLocalRanges.ringVersion); + cachedShardBoundaries = shardBoundaries; + logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), boundaries); + } + return shardBoundaries; + } + /** * @param sstables * @return sstables whose key range overlaps with that of the given sstables, not including itself. @@ -1634,7 +1650,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { Instant creationTime = now(); String snapshotName = "pre-scrub-" + creationTime.toEpochMilli(); - snapshotWithoutFlush(snapshotName, creationTime); + snapshotWithoutMemtable(snapshotName, creationTime); } try @@ -1973,20 +1989,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return metadata().comparator; } - public TableSnapshot snapshotWithoutFlush(String snapshotName) + public TableSnapshot snapshotWithoutMemtable(String snapshotName) { - return snapshotWithoutFlush(snapshotName, now()); + return snapshotWithoutMemtable(snapshotName, now()); } - public TableSnapshot snapshotWithoutFlush(String snapshotName, Instant creationTime) + public TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime) { - return snapshotWithoutFlush(snapshotName, null, false, null, null, creationTime); + return snapshotWithoutMemtable(snapshotName, null, false, null, null, creationTime); } /** * @param ephemeral If this flag is set to true, the snapshot will be cleaned during next startup */ - public TableSnapshot snapshotWithoutFlush(String snapshotName, Predicate predicate, boolean ephemeral, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) + public TableSnapshot snapshotWithoutMemtable(String snapshotName, Predicate predicate, boolean ephemeral, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) { if (ephemeral && ttl != null) { @@ -2175,40 +2191,47 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean * Take a snap shot of this columnfamily store. * * @param snapshotName the name of the associated with the snapshot - * @param skipFlush Skip blocking flush of memtable + * @param skipMemtable Skip flushing the memtable * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed * @param rateLimiter Rate limiter for hardlinks-per-second * @param creationTime time when this snapshot was taken */ - public TableSnapshot snapshot(String snapshotName, boolean skipFlush, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) + public TableSnapshot snapshot(String snapshotName, boolean skipMemtable, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) { - return snapshot(snapshotName, null, false, skipFlush, ttl, rateLimiter, creationTime); + return snapshot(snapshotName, null, false, skipMemtable, ttl, rateLimiter, creationTime); } /** * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup - * @param skipFlush Skip blocking flush of memtable + * @param skipMemtable Skip flushing the memtable */ - public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipFlush) + public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipMemtable) { - return snapshot(snapshotName, predicate, ephemeral, skipFlush, null, null, now()); + return snapshot(snapshotName, predicate, ephemeral, skipMemtable, null, null, now()); } /** * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup - * @param skipFlush Skip blocking flush of memtable + * @param skipMemtable Skip flushing the memtable * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed * @param rateLimiter Rate limiter for hardlinks-per-second * @param creationTime time when this snapshot was taken */ - public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipFlush, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) + public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipMemtable, DurationSpec ttl, RateLimiter rateLimiter, Instant creationTime) { - if (!skipFlush) + if (!skipMemtable) { - forceBlockingFlush(); + Memtable current = getTracker().getView().getCurrentMemtable(); + if (!current.isClean()) + { + if (current.shouldSwitch(FlushReason.SNAPSHOT)) + FBUtilities.waitOnFuture(switchMemtableIfCurrent(current, FlushReason.SNAPSHOT)); + else + current.performSnapshot(snapshotName); + } } - return snapshotWithoutFlush(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime); + return snapshotWithoutMemtable(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime); } public boolean snapshotExists(String snapshotName) @@ -2414,6 +2437,108 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } + public void writeAndAddMemtableRanges(TimeUUID repairSessionID, + Supplier>> rangesSupplier, + Refs placeIntoRefs) + { + @SuppressWarnings("resource") // closed by finish or on exception + SSTableMultiWriter memtableContent = writeMemtableRanges(rangesSupplier, repairSessionID); + if (memtableContent != null) + { + try + { + Collection sstables = memtableContent.finish(true); + try (Refs sstableReferences = Refs.ref(sstables)) + { + // This moves all references to placeIntoRefs, clearing sstableReferences + placeIntoRefs.addAll(sstableReferences); + } + + // Release the reference any written sstables start with. + for (SSTableReader rdr : sstables) + { + rdr.selfRef().release(); + logger.info("Memtable ranges (keys {} size {}) written in {}", + rdr.estimatedKeys(), + rdr.getDataChannel().size(), + rdr); + } + } + catch (Throwable t) + { + memtableContent.close(); + Throwables.propagate(t); + } + } + } + + private SSTableMultiWriter writeMemtableRanges(Supplier>> rangesSupplier, + TimeUUID repairSessionID) + { + if (!streamFromMemtable()) + return null; + + Collection> ranges = rangesSupplier.get(); + Memtable current = getTracker().getView().getCurrentMemtable(); + if (current.isClean()) + return null; + + List> dataSets = new ArrayList<>(ranges.size()); + long keys = 0; + for (Range range : ranges) + { + Memtable.FlushablePartitionSet dataSet = current.getFlushSet(range.left, range.right); + dataSets.add(dataSet); + keys += dataSet.partitionCount(); + } + if (keys == 0) + return null; + + // TODO: Can we write directly to stream, skipping disk? + Memtable.FlushablePartitionSet firstDataSet = dataSets.get(0); + SSTableMultiWriter writer = createSSTableMultiWriter(newSSTableDescriptor(directories.getDirectoryForNewSSTables()), + keys, + 0, + repairSessionID, + false, + 0, + new SerializationHeader(true, + firstDataSet.metadata(), + firstDataSet.columns(), + firstDataSet.encodingStats()), + DO_NOT_TRACK); + try + { + for (Memtable.FlushablePartitionSet dataSet : dataSets) + new Flushing.FlushRunnable(dataSet, writer, metric, false).call(); // executes on this thread + + return writer; + } + catch (Error | RuntimeException t) + { + writer.abort(t); + throw t; + } + } + + private static final LifecycleNewTracker DO_NOT_TRACK = new LifecycleNewTracker() + { + public void trackNew(SSTable table) + { + // not tracking + } + + public void untrackNew(SSTable table) + { + // not tracking + } + + public OperationType opType() + { + return OperationType.FLUSH; + } + }; + /** * For testing. No effort is made to clear historical or even the current memtables, nor for * thread safety. All we do is wipe the sstable containers clean, while leaving the actual @@ -2425,7 +2550,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean for (final ColumnFamilyStore cfs : concatWithIndexes()) { cfs.runWithCompactionsDisabled((Callable) () -> { - cfs.data.reset(new Memtable(new AtomicReference<>(CommitLogPosition.NONE), cfs)); + cfs.data.reset(memtableFactory.create(new AtomicReference<>(CommitLogPosition.NONE), cfs.metadata, cfs)); return null; }, true, false); } @@ -2466,23 +2591,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean final long truncatedAt; final CommitLogPosition replayAfter; - if (!noSnapshot && (keyspace.getMetadata().params.durableWrites || DatabaseDescriptor.isAutoSnapshot())) + if (!noSnapshot && + ((keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable()) // need to clear dirty regions + || DatabaseDescriptor.isAutoSnapshot())) // need sstable for snapshot { - replayAfter = forceBlockingFlush(); - viewManager.forceBlockingFlush(); + replayAfter = forceBlockingFlush(FlushReason.TRUNCATE); + viewManager.forceBlockingFlush(FlushReason.TRUNCATE); } else { // just nuke the memtable data w/o writing to disk first + // note: this does not wait for the switch to complete, but because the post-flush processing is serial, + // the call below does. viewManager.dumpMemtables(); - try - { - replayAfter = dumpMemtable().get(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } + replayAfter = FBUtilities.waitOnFuture(dumpMemtable()); } long now = currentTimeMillis(); @@ -2527,7 +2649,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } /** - * Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable. + * Drops current memtable without flushing to disk. This should only be called when truncating a column family + * that cannot have dirty intervals in the commit log (i.e. one which is not durable, or where the memtable itself + * performs durable writes). */ public Future dumpMemtable() { @@ -2540,6 +2664,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean } } + public void unloadCf() + { + if (keyspace.getMetadata().params.durableWrites && !memtableWritesAreDurable()) // need to clear dirty regions + forceBlockingFlush(ColumnFamilyStore.FlushReason.DROP); + else + FBUtilities.waitOnFuture(dumpMemtable()); + } + public V runWithCompactionsDisabled(Callable callable, boolean interruptValidation, boolean interruptViews) { return runWithCompactionsDisabled(callable, (sstable) -> true, interruptValidation, interruptViews, true); @@ -3026,9 +3158,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return diskBoundaryManager.getDiskBoundaries(this); } - public void invalidateDiskBoundaries() + public void invalidateLocalRanges() { diskBoundaryManager.invalidate(); + + switchMemtableOrNotify(FlushReason.OWNED_RANGES_CHANGE, Memtable::localRangesUpdated); } @Override diff --git a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java index 48a40dddf5..0de745d3cf 100644 --- a/src/java/org/apache/cassandra/db/DiskBoundaryManager.java +++ b/src/java/org/apache/cassandra/db/DiskBoundaryManager.java @@ -67,7 +67,19 @@ public class DiskBoundaryManager diskBoundaries.invalidate(); } - private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs) + static class VersionedRangesAtEndpoint + { + public final RangesAtEndpoint rangesAtEndpoint; + public final long ringVersion; + + VersionedRangesAtEndpoint(RangesAtEndpoint rangesAtEndpoint, long ringVersion) + { + this.rangesAtEndpoint = rangesAtEndpoint; + this.ringVersion = ringVersion; + } + } + + public static VersionedRangesAtEndpoint getVersionedLocalRanges(ColumnFamilyStore cfs) { RangesAtEndpoint localRanges; @@ -77,23 +89,20 @@ public class DiskBoundaryManager { tmd = StorageService.instance.getTokenMetadata(); ringVersion = tmd.getRingVersion(); - if (StorageService.instance.isBootstrapMode() - && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally - { - PendingRangeCalculatorService.instance.blockUntilFinished(); - localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort()); - } - else - { - // Reason we use use the future settled TMD is that if we decommission a node, we want to stream - // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places. - // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled - localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort()); - } + localRanges = getLocalRanges(cfs, tmd); logger.debug("Got local ranges {} (ringVersion = {})", localRanges, ringVersion); } while (ringVersion != tmd.getRingVersion()); // if ringVersion is different here it means that - // it might have changed before we calculated localRanges - recalculate + // it might have changed before we calculated localRanges - recalculate + + return new VersionedRangesAtEndpoint(localRanges, ringVersion); + } + + private static DiskBoundaries getDiskBoundaryValue(ColumnFamilyStore cfs) + { + VersionedRangesAtEndpoint rangesAtEndpoint = getVersionedLocalRanges(cfs); + RangesAtEndpoint localRanges = rangesAtEndpoint.rangesAtEndpoint; + long ringVersion = rangesAtEndpoint.ringVersion; int directoriesVersion; Directories.DataDirectory[] dirs; @@ -112,6 +121,25 @@ public class DiskBoundaryManager return new DiskBoundaries(cfs, dirs, positions, ringVersion, directoriesVersion); } + private static RangesAtEndpoint getLocalRanges(ColumnFamilyStore cfs, TokenMetadata tmd) + { + RangesAtEndpoint localRanges; + if (StorageService.instance.isBootstrapMode() + && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally + { + PendingRangeCalculatorService.instance.blockUntilFinished(); + localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort()); + } + else + { + // Reason we use use the future settled TMD is that if we decommission a node, we want to stream + // from that node to the correct location on disk, if we didn't, we would put new files in the wrong places. + // We do this to minimize the amount of data we need to move in rebalancedisks once everything settled + localRanges = cfs.keyspace.getReplicationStrategy().getAddressReplicas(tmd.cloneAfterAllSettled(), FBUtilities.getBroadcastAddressAndPort()); + } + return localRanges; + } + /** * Returns a list of disk boundaries, the result will differ depending on whether vnodes are enabled or not. * diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index 34b8298f82..95eb966c93 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -377,7 +377,7 @@ public class Keyspace if (!ksm.params.replication.equals(replicationParams)) { logger.debug("New replication settings for keyspace {} - invalidating disk boundary caches", ksm.name); - columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateDiskBoundaries); + columnFamilyStores.values().forEach(ColumnFamilyStore::invalidateLocalRanges); } replicationParams = ksm.params.replication; } @@ -406,7 +406,7 @@ public class Keyspace // disassociate a cfs from this keyspace instance. private void unloadCf(ColumnFamilyStore cfs, boolean dropData) { - cfs.forceBlockingFlush(); + cfs.unloadCf(); cfs.invalidate(true, dropData); } @@ -677,11 +677,11 @@ public class Keyspace return replicationStrategy; } - public List> flush() + public List> flush(ColumnFamilyStore.FlushReason reason) { List> futures = new ArrayList<>(columnFamilyStores.size()); for (ColumnFamilyStore cfs : columnFamilyStores.values()) - futures.add(cfs.forceFlush()); + futures.add(cfs.forceFlush(reason)); return futures; } diff --git a/src/java/org/apache/cassandra/db/Memtable.java b/src/java/org/apache/cassandra/db/Memtable.java deleted file mode 100644 index 21bf31a182..0000000000 --- a/src/java/org/apache/cassandra/db/Memtable.java +++ /dev/null @@ -1,682 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.db.commitlog.CommitLogPosition; -import org.apache.cassandra.db.commitlog.IntervalSet; -import org.apache.cassandra.db.filter.ClusteringIndexFilter; -import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.partitions.AbstractBTreePartition; -import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator; -import org.apache.cassandra.db.partitions.AtomicBTreePartition; -import org.apache.cassandra.db.partitions.Partition; -import org.apache.cassandra.db.partitions.PartitionUpdate; -import org.apache.cassandra.db.rows.EncodingStats; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.Bounds; -import org.apache.cassandra.dht.IncludingExcludingBounds; -import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.index.transactions.UpdateTransaction; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTableMultiWriter; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.ObjectSizes; -import org.apache.cassandra.utils.concurrent.OpOrder; -import org.apache.cassandra.utils.memory.HeapPool; -import org.apache.cassandra.utils.memory.MemtableAllocator; -import org.apache.cassandra.utils.memory.MemtableCleaner; -import org.apache.cassandra.utils.memory.MemtablePool; -import org.apache.cassandra.utils.memory.NativePool; -import org.apache.cassandra.utils.memory.SlabPool; - -import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_COMPUTE_STEPS; -import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE; - -public class Memtable implements Comparable -{ - private static final Logger logger = LoggerFactory.getLogger(Memtable.class); - - public static final MemtablePool MEMORY_POOL = createMemtableAllocatorPool(); - - private static MemtablePool createMemtableAllocatorPool() - { - long heapLimit = DatabaseDescriptor.getMemtableHeapSpaceInMiB() << 20; - long offHeapLimit = DatabaseDescriptor.getMemtableOffheapSpaceInMiB() << 20; - final float cleaningThreshold = DatabaseDescriptor.getMemtableCleanupThreshold(); - final MemtableCleaner cleaner = ColumnFamilyStore::flushLargestMemtable; - switch (DatabaseDescriptor.getMemtableAllocationType()) - { - case unslabbed_heap_buffers_logged: - return new HeapPool.Logged(heapLimit, cleaningThreshold, cleaner); - case unslabbed_heap_buffers: - return new HeapPool(heapLimit, cleaningThreshold, cleaner); - case heap_buffers: - return new SlabPool(heapLimit, 0, cleaningThreshold, cleaner); - case offheap_buffers: - return new SlabPool(heapLimit, offHeapLimit, cleaningThreshold, cleaner); - case offheap_objects: - return new NativePool(heapLimit, offHeapLimit, cleaningThreshold, cleaner); - default: - throw new AssertionError(); - } - } - - private static final int ROW_OVERHEAD_HEAP_SIZE; - static - { - int userDefinedOverhead = MEMTABLE_OVERHEAD_SIZE.getInt(-1); - if (userDefinedOverhead > 0) ROW_OVERHEAD_HEAP_SIZE = userDefinedOverhead; - else ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(MEMTABLE_OVERHEAD_COMPUTE_STEPS.getInt()); - } - - private final MemtableAllocator allocator; - private final AtomicLong liveDataSize = new AtomicLong(0); - private final AtomicLong currentOperations = new AtomicLong(0); - - // the write barrier for directing writes to this memtable or the next during a switch - private volatile OpOrder.Barrier writeBarrier; - // the precise upper bound of CommitLogPosition owned by this memtable - private volatile AtomicReference commitLogUpperBound; - // the precise lower bound of CommitLogPosition owned by this memtable; equal to its predecessor's commitLogUpperBound - private AtomicReference commitLogLowerBound; - - // The approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor - // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound - private final CommitLogPosition approximateCommitLogLowerBound = CommitLog.instance.getCurrentPosition(); - - public int compareTo(Memtable that) - { - return this.approximateCommitLogLowerBound.compareTo(that.approximateCommitLogLowerBound); - } - - public static final class LastCommitLogPosition extends CommitLogPosition - { - public LastCommitLogPosition(CommitLogPosition copy) - { - super(copy.segmentId, copy.position); - } - } - - // We index the memtable by PartitionPosition only for the purpose of being able - // to select key range using Token.KeyBound. However put() ensures that we - // actually only store DecoratedKey. - private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); - public final ColumnFamilyStore cfs; - private final long creationNano = nanoTime(); - - // The smallest timestamp for all partitions stored in this memtable - private long minTimestamp = Long.MAX_VALUE; - - // Record the comparator of the CFS at the creation of the memtable. This - // is only used when a user update the CF comparator, to know if the - // memtable was created with the new or old comparator. - public final ClusteringComparator initialComparator; - - private final ColumnsCollector columnsCollector; - private final StatsCollector statsCollector = new StatsCollector(); - - // only to be used by init(), to setup the very first memtable for the cfs - public Memtable(AtomicReference commitLogLowerBound, ColumnFamilyStore cfs) - { - this.cfs = cfs; - this.commitLogLowerBound = commitLogLowerBound; - this.allocator = MEMORY_POOL.newAllocator(cfs); - this.initialComparator = cfs.metadata().comparator; - this.cfs.scheduleFlush(); - this.columnsCollector = new ColumnsCollector(cfs.metadata().regularAndStaticColumns()); - } - - // ONLY to be used for testing, to create a mock Memtable - @VisibleForTesting - public Memtable(TableMetadata metadata) - { - this.initialComparator = metadata.comparator; - this.cfs = null; - this.allocator = null; - this.columnsCollector = new ColumnsCollector(metadata.regularAndStaticColumns()); - } - - public MemtableAllocator getAllocator() - { - return allocator; - } - - public long getLiveDataSize() - { - return liveDataSize.get(); - } - - public long getOperations() - { - return currentOperations.get(); - } - - @VisibleForTesting - public void setDiscarding(OpOrder.Barrier writeBarrier, AtomicReference commitLogUpperBound) - { - assert this.writeBarrier == null; - this.commitLogUpperBound = commitLogUpperBound; - this.writeBarrier = writeBarrier; - allocator.setDiscarding(); - } - - void setDiscarded() - { - allocator.setDiscarded(); - } - - // decide if this memtable should take the write, or if it should go to the next memtable - public boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition) - { - // if the barrier hasn't been set yet, then this memtable is still taking ALL writes - OpOrder.Barrier barrier = this.writeBarrier; - if (barrier == null) - return true; - // if the barrier has been set, but is in the past, we are definitely destined for a future memtable - if (!barrier.isAfter(opGroup)) - return false; - // if we aren't durable we are directed only by the barrier - if (commitLogPosition == null) - return true; - while (true) - { - // otherwise we check if we are in the past/future wrt the CL boundary; - // if the boundary hasn't been finalised yet, we simply update it to the max of - // its current value and ours; if it HAS been finalised, we simply accept its judgement - // this permits us to coordinate a safe boundary, as the boundary choice is made - // atomically wrt our max() maintenance, so an operation cannot sneak into the past - CommitLogPosition currentLast = commitLogUpperBound.get(); - if (currentLast instanceof LastCommitLogPosition) - return currentLast.compareTo(commitLogPosition) >= 0; - if (currentLast != null && currentLast.compareTo(commitLogPosition) >= 0) - return true; - if (commitLogUpperBound.compareAndSet(currentLast, commitLogPosition)) - return true; - } - } - - public CommitLogPosition getCommitLogLowerBound() - { - return commitLogLowerBound.get(); - } - - public CommitLogPosition getCommitLogUpperBound() - { - return commitLogUpperBound.get(); - } - - public boolean isLive() - { - return allocator.isLive(); - } - - public boolean isClean() - { - return partitions.isEmpty(); - } - - public boolean mayContainDataBefore(CommitLogPosition position) - { - return approximateCommitLogLowerBound.compareTo(position) < 0; - } - - /** - * @return true if this memtable is expired. Expiration time is determined by CF's memtable_flush_period_in_ms. - */ - public boolean isExpired() - { - int period = cfs.metadata().params.memtableFlushPeriodInMs; - return period > 0 && (nanoTime() - creationNano >= TimeUnit.MILLISECONDS.toNanos(period)); - } - - /** - * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate - * OpOrdering. - * - * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null - */ - long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) - { - AtomicBTreePartition previous = partitions.get(update.partitionKey()); - - long initialSize = 0; - if (previous == null) - { - final DecoratedKey cloneKey = allocator.clone(update.partitionKey(), opGroup); - AtomicBTreePartition empty = new AtomicBTreePartition(cfs.metadata, cloneKey, allocator); - // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent - previous = partitions.putIfAbsent(cloneKey, empty); - if (previous == null) - { - previous = empty; - // allocate the row overhead after the fact; this saves over allocating and having to free after, but - // means we can overshoot our declared limit. - int overhead = (int) (cloneKey.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE); - allocator.onHeap().allocate(overhead, opGroup); - initialSize = 8; - } - } - - long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer); - minTimestamp = Math.min(minTimestamp, previous.stats().minTimestamp); - liveDataSize.addAndGet(initialSize + pair[0]); - columnsCollector.update(update.columns()); - statsCollector.update(update.stats()); - currentOperations.addAndGet(update.operationCount()); - return pair[1]; - } - - public int partitionCount() - { - return partitions.size(); - } - - public List flushRunnables(LifecycleTransaction txn) - { - return createFlushRunnables(txn); - } - - private List createFlushRunnables(LifecycleTransaction txn) - { - DiskBoundaries diskBoundaries = cfs.getDiskBoundaries(); - List boundaries = diskBoundaries.positions; - List locations = diskBoundaries.directories; - if (boundaries == null) - return Collections.singletonList(new FlushRunnable(txn)); - - List runnables = new ArrayList<>(boundaries.size()); - PartitionPosition rangeStart = cfs.getPartitioner().getMinimumToken().minKeyBound(); - try - { - for (int i = 0; i < boundaries.size(); i++) - { - PartitionPosition t = boundaries.get(i); - runnables.add(new FlushRunnable(rangeStart, t, locations.get(i), txn)); - rangeStart = t; - } - return runnables; - } - catch (Throwable e) - { - throw Throwables.propagate(abortRunnables(runnables, e)); - } - } - - public Throwable abortRunnables(List runnables, Throwable t) - { - if (runnables != null) - for (FlushRunnable runnable : runnables) - t = runnable.writer.abort(t); - return t; - } - - public String toString() - { - return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %.0f%%/%.0f%% of on/off-heap limit)", - cfs.name, hashCode(), FBUtilities.prettyPrintMemory(liveDataSize.get()), currentOperations, - 100 * allocator.onHeap().ownershipRatio(), 100 * allocator.offHeap().ownershipRatio()); - } - - public MemtableUnfilteredPartitionIterator makePartitionIterator(final ColumnFilter columnFilter, final DataRange dataRange) - { - AbstractBounds keyRange = dataRange.keyRange(); - - boolean startIsMin = keyRange.left.isMinimum(); - boolean stopIsMin = keyRange.right.isMinimum(); - - boolean isBound = keyRange instanceof Bounds; - boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds; - boolean includeStop = isBound || keyRange instanceof Range; - Map subMap; - if (startIsMin) - subMap = stopIsMin ? partitions : partitions.headMap(keyRange.right, includeStop); - else - subMap = stopIsMin - ? partitions.tailMap(keyRange.left, includeStart) - : partitions.subMap(keyRange.left, includeStart, keyRange.right, includeStop); - - int minLocalDeletionTime = Integer.MAX_VALUE; - - // avoid iterating over the memtable if we purge all tombstones - if (cfs.getCompactionStrategyManager().onlyPurgeRepairedTombstones()) - minLocalDeletionTime = findMinLocalDeletionTime(subMap.entrySet().iterator()); - - final Iterator> iter = subMap.entrySet().iterator(); - - return new MemtableUnfilteredPartitionIterator(cfs, iter, minLocalDeletionTime, columnFilter, dataRange); - } - - private int findMinLocalDeletionTime(Iterator> iterator) - { - int minLocalDeletionTime = Integer.MAX_VALUE; - while (iterator.hasNext()) - { - Map.Entry entry = iterator.next(); - minLocalDeletionTime = Math.min(minLocalDeletionTime, entry.getValue().stats().minLocalDeletionTime); - } - return minLocalDeletionTime; - } - - public Partition getPartition(DecoratedKey key) - { - return partitions.get(key); - } - - public long getMinTimestamp() - { - return minTimestamp; - } - - /** - * For testing only. Give this memtable too big a size to make it always fail flushing. - */ - @VisibleForTesting - public void makeUnflushable() - { - liveDataSize.addAndGet((long) 1024 * 1024 * 1024 * 1024 * 1024); - } - - class FlushRunnable implements Callable - { - private final long estimatedSize; - private final ConcurrentNavigableMap toFlush; - - private final boolean isBatchLogTable; - private final SSTableMultiWriter writer; - - // keeping these to be able to log what we are actually flushing - private final PartitionPosition from; - private final PartitionPosition to; - - FlushRunnable(PartitionPosition from, PartitionPosition to, Directories.DataDirectory flushLocation, LifecycleTransaction txn) - { - this(partitions.subMap(from, to), flushLocation, from, to, txn); - } - - FlushRunnable(LifecycleTransaction txn) - { - this(partitions, null, null, null, txn); - } - - FlushRunnable(ConcurrentNavigableMap toFlush, Directories.DataDirectory flushLocation, PartitionPosition from, PartitionPosition to, LifecycleTransaction txn) - { - this.toFlush = toFlush; - this.from = from; - this.to = to; - long keySize = 0; - for (PartitionPosition key : toFlush.keySet()) - { - // make sure we don't write non-sensical keys - assert key instanceof DecoratedKey; - keySize += ((DecoratedKey) key).getKey().remaining(); - } - estimatedSize = (long) ((keySize // index entries - + keySize // keys in data file - + liveDataSize.get()) // data - * 1.2); // bloom filter and row index overhead - - this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME); - - if (flushLocation == null) - writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get()); - else - writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get()); - - } - - protected Directories getDirectories() - { - return cfs.getDirectories(); - } - - private void writeSortedContents() - { - logger.info("Writing {}, flushed range = ({}, {}]", Memtable.this.toString(), from, to); - - boolean trackContention = logger.isTraceEnabled(); - int heavilyContendedRowCount = 0; - // (we can't clear out the map as-we-go to free up memory, - // since the memtable is being used for queries in the "pending flush" category) - for (AtomicBTreePartition partition : toFlush.values()) - { - // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2 - // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local, - // we don't need to preserve tombstones for repair. So if both operation are in this - // memtable (which will almost always be the case if there is no ongoing failure), we can - // just skip the entry (CASSANDRA-4667). - if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows()) - continue; - - if (trackContention && partition.useLock()) - heavilyContendedRowCount++; - - if (!partition.isEmpty()) - { - try (UnfilteredRowIterator iter = partition.unfilteredIterator()) - { - writer.append(iter); - } - } - } - - long bytesFlushed = writer.getFilePointer(); - logger.info("Completed flushing {} ({}) for commitlog position {}", - writer.getFilename(), - FBUtilities.prettyPrintMemory(bytesFlushed), - commitLogUpperBound); - // Update the metrics - cfs.metric.bytesFlushed.inc(bytesFlushed); - - if (heavilyContendedRowCount > 0) - logger.trace("High update contention in {}/{} partitions of {} ", heavilyContendedRowCount, toFlush.size(), Memtable.this); - } - - public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn, - Descriptor descriptor, - RegularAndStaticColumns columns, - EncodingStats stats) - { - MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata().comparator) - .commitLogIntervals(new IntervalSet<>(commitLogLowerBound.get(), commitLogUpperBound.get())); - - return cfs.createSSTableMultiWriter(descriptor, - toFlush.size(), - ActiveRepairService.UNREPAIRED_SSTABLE, - ActiveRepairService.NO_PENDING_REPAIR, - false, - sstableMetadataCollector, - new SerializationHeader(true, cfs.metadata(), columns, stats), txn); - } - - @Override - public SSTableMultiWriter call() - { - writeSortedContents(); - return writer; - } - - @Override - public String toString() - { - return "Flush " + cfs.keyspace + '.' + cfs.name; - } - } - - private static int estimateRowOverhead(final int count) - { - // calculate row overhead - try (final OpOrder.Group group = new OpOrder().start()) - { - int rowOverhead; - MemtableAllocator allocator = MEMORY_POOL.newAllocator(null); - ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); - final Object val = new Object(); - for (int i = 0 ; i < count ; i++) - partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val); - double avgSize = ObjectSizes.measureDeep(partitions) / (double) count; - rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize)); - rowOverhead -= ObjectSizes.measureDeep(new LongToken(0)); - rowOverhead += AtomicBTreePartition.EMPTY_SIZE; - rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE; - allocator.setDiscarding(); - allocator.setDiscarded(); - return rowOverhead; - } - } - - public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator - { - private final ColumnFamilyStore cfs; - private final Iterator> iter; - private final int minLocalDeletionTime; - private final ColumnFilter columnFilter; - private final DataRange dataRange; - - public MemtableUnfilteredPartitionIterator(ColumnFamilyStore cfs, Iterator> iter, int minLocalDeletionTime, ColumnFilter columnFilter, DataRange dataRange) - { - this.cfs = cfs; - this.iter = iter; - this.minLocalDeletionTime = minLocalDeletionTime; - this.columnFilter = columnFilter; - this.dataRange = dataRange; - } - - public int getMinLocalDeletionTime() - { - return minLocalDeletionTime; - } - - public TableMetadata metadata() - { - return cfs.metadata(); - } - - public boolean hasNext() - { - return iter.hasNext(); - } - - public UnfilteredRowIterator next() - { - Map.Entry entry = iter.next(); - // Actual stored key should be true DecoratedKey - assert entry.getKey() instanceof DecoratedKey; - DecoratedKey key = (DecoratedKey)entry.getKey(); - ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key); - - return filter.getUnfilteredRowIterator(columnFilter, entry.getValue()); - } - } - - private static class ColumnsCollector - { - private final HashMap predefined = new HashMap<>(); - private final ConcurrentSkipListSet extra = new ConcurrentSkipListSet<>(); - ColumnsCollector(RegularAndStaticColumns columns) - { - for (ColumnMetadata def : columns.statics) - predefined.put(def, new AtomicBoolean()); - for (ColumnMetadata def : columns.regulars) - predefined.put(def, new AtomicBoolean()); - } - - public void update(RegularAndStaticColumns columns) - { - for (ColumnMetadata s : columns.statics) - update(s); - for (ColumnMetadata r : columns.regulars) - update(r); - } - - private void update(ColumnMetadata definition) - { - AtomicBoolean present = predefined.get(definition); - if (present != null) - { - if (!present.get()) - present.set(true); - } - else - { - extra.add(definition); - } - } - - public RegularAndStaticColumns get() - { - RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder(); - for (Map.Entry e : predefined.entrySet()) - if (e.getValue().get()) - builder.add(e.getKey()); - return builder.addAll(extra).build(); - } - } - - private static class StatsCollector - { - private final AtomicReference stats = new AtomicReference<>(EncodingStats.NO_STATS); - - public void update(EncodingStats newStats) - { - while (true) - { - EncodingStats current = stats.get(); - EncodingStats updated = current.mergeWith(newStats); - if (stats.compareAndSet(current, updated)) - return; - } - } - - public EncodingStats get() - { - return stats.get(); - } - } -} diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index 48bacf6aeb..f000d63f99 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -24,12 +24,17 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.db.virtual.VirtualTable; -import org.apache.cassandra.net.Verb; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.lifecycle.View; -import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.partitions.CachedPartition; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.db.rows.BaseRowIterator; import org.apache.cassandra.db.transform.RTBoundValidator; import org.apache.cassandra.db.transform.Transformation; @@ -42,7 +47,9 @@ import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.tracing.Tracing; @@ -313,19 +320,19 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR InputCollector inputCollector = iteratorsForRange(view, controller); try { + SSTableReadsListener readCountUpdater = newReadCountUpdater(); for (Memtable memtable : view.memtables) { @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method - Memtable.MemtableUnfilteredPartitionIterator iter = memtable.makePartitionIterator(columnFilter(), dataRange()); - controller.updateMinOldestUnrepairedTombstone(iter.getMinLocalDeletionTime()); + UnfilteredPartitionIterator iter = memtable.partitionIterator(columnFilter(), dataRange(), readCountUpdater); + controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime()); inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false)); } - SSTableReadsListener readCountUpdater = newReadCountUpdater(); for (SSTableReader sstable : view.sstables) { @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method - UnfilteredPartitionIterator iter = sstable.getScanner(columnFilter(), dataRange(), readCountUpdater); + UnfilteredPartitionIterator iter = sstable.partitionIterator(columnFilter(), dataRange(), readCountUpdater); inputCollector.addSSTableIterator(sstable, RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false)); if (!sstable.isRepaired()) diff --git a/src/java/org/apache/cassandra/db/SimpleBuilders.java b/src/java/org/apache/cassandra/db/SimpleBuilders.java index 3167f8d17e..def33090d7 100644 --- a/src/java/org/apache/cassandra/db/SimpleBuilders.java +++ b/src/java/org/apache/cassandra/db/SimpleBuilders.java @@ -421,6 +421,12 @@ public abstract class SimpleBuilders return this; } + public Row.SimpleBuilder deletePrevious() + { + builder.addRowDeletion(Row.Deletion.regular(new DeletionTime(timestamp - 1, nowInSec))); + return this; + } + public Row.SimpleBuilder delete(String columnName) { return add(columnName, null); diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index ccd97681e6..64136b6099 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -32,6 +32,7 @@ import org.apache.cassandra.cache.RowCacheSentinel; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.lifecycle.*; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.transform.RTBoundValidator; @@ -660,19 +661,19 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar InputCollector inputCollector = iteratorsForPartition(view, controller); try { + SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector(); + for (Memtable memtable : view.memtables) { - Partition partition = memtable.getPartition(partitionKey()); - if (partition == null) + @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator + UnfilteredRowIterator iter = memtable.rowIterator(partitionKey(), filter.getSlices(metadata()), columnFilter(), filter.isReversed(), metricsCollector); + if (iter == null) continue; minTimestamp = Math.min(minTimestamp, memtable.getMinTimestamp()); - @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator - UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition); - // Memtable data is always considered unrepaired - controller.updateMinOldestUnrepairedTombstone(partition.stats().minLocalDeletionTime); + controller.updateMinOldestUnrepairedTombstone(memtable.getMinLocalDeletionTime()); inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false)); mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, @@ -695,8 +696,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar int nonIntersectingSSTables = 0; int includedDueToTombstones = 0; - SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector(); - if (controller.isTrackingRepairedStatus()) Tracing.trace("Collecting data from sstables and tracking repaired status"); @@ -859,17 +858,14 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, partitionKey())); ImmutableBTreePartition result = null; + SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector(); Tracing.trace("Merging memtable contents"); for (Memtable memtable : view.memtables) { - Partition partition = memtable.getPartition(partitionKey()); - if (partition == null) - continue; - - try (UnfilteredRowIterator iter = filter.getUnfilteredRowIterator(columnFilter(), partition)) + try (UnfilteredRowIterator iter = memtable.rowIterator(partitionKey, filter.getSlices(metadata()), columnFilter(), isReversed(), metricsCollector)) { - if (iter.isEmpty()) + if (iter == null) continue; result = add(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false), @@ -883,7 +879,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar /* add the SSTables on disk */ view.sstables.sort(SSTableReader.maxTimestampDescending); // read sorted sstables - SSTableReadMetricsCollector metricsCollector = new SSTableReadMetricsCollector(); for (SSTableReader sstable : view.sstables) { // if we've already seen a partition tombstone with a timestamp greater diff --git a/src/java/org/apache/cassandra/db/StorageHook.java b/src/java/org/apache/cassandra/db/StorageHook.java index be1d0bf754..c998338abe 100644 --- a/src/java/org/apache/cassandra/db/StorageHook.java +++ b/src/java/org/apache/cassandra/db/StorageHook.java @@ -84,7 +84,7 @@ public interface StorageHook boolean reversed, SSTableReadsListener listener) { - return sstable.iterator(key, slices, selectedColumns, reversed, listener); + return sstable.rowIterator(key, slices, selectedColumns, reversed, listener); } }; } diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index 789bc15a93..6fbbc3e621 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -923,7 +923,9 @@ public final class SystemKeyspace for (String cfname : cfnames) { - futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(cfname).forceFlush()); + futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME) + .getColumnFamilyStore(cfname) + .forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED)); } FBUtilities.waitOnFutures(futures); } @@ -1433,7 +1435,8 @@ public final class SystemKeyspace public static void flushPaxosRepairHistory() { - Schema.instance.getColumnFamilyStoreInstance(PaxosRepairHistoryTable.id).forceBlockingFlush(); + Schema.instance.getColumnFamilyStoreInstance(PaxosRepairHistoryTable.id) + .forceBlockingFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED); } public static PaxosRepairHistory loadPaxosRepairHistory(String keyspace, String table) diff --git a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java index a1eb7bc777..627c885cd0 100644 --- a/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java +++ b/src/java/org/apache/cassandra/db/commitlog/AbstractCommitLogSegmentManager.java @@ -420,9 +420,20 @@ public abstract class AbstractCommitLogSegmentManager else if (!flushes.containsKey(dirtyTableId)) { final ColumnFamilyStore cfs = Keyspace.open(metadata.keyspace).getColumnFamilyStore(dirtyTableId); - // can safely call forceFlush here as we will only ever block (briefly) for other attempts to flush, - // no deadlock possibility since switchLock removal - flushes.put(dirtyTableId, force ? cfs.forceFlush() : cfs.forceFlush(maxCommitLogPosition)); + + if (cfs.memtableWritesAreDurable()) + { + // The memtable does not need this data to be preserved (we only wrote it for PITR and CDC) + segment.markClean(dirtyTableId, CommitLogPosition.NONE, segment.getCurrentCommitLogPosition()); + } + else + { + // can safely call forceFlush here as we will only ever block (briefly) for other attempts to flush, + // no deadlock possibility since switchLock removal + flushes.put(dirtyTableId, force + ? cfs.forceFlush(ColumnFamilyStore.FlushReason.COMMITLOG_DIRTY) + : cfs.forceFlush(maxCommitLogPosition)); + } } } } diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java index 5ee79ff4d4..2e1b580716 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java @@ -66,15 +66,17 @@ public class CommitLogArchiver final String restoreCommand; final String restoreDirectories; public long restorePointInTime; + public CommitLogPosition snapshotCommitLogPosition; public final TimeUnit precision; public CommitLogArchiver(String archiveCommand, String restoreCommand, String restoreDirectories, - long restorePointInTime, TimeUnit precision) + long restorePointInTime, CommitLogPosition snapshotCommitLogPosition, TimeUnit precision) { this.archiveCommand = archiveCommand; this.restoreCommand = restoreCommand; this.restoreDirectories = restoreDirectories; this.restorePointInTime = restorePointInTime; + this.snapshotCommitLogPosition = snapshotCommitLogPosition; this.precision = precision; executor = !Strings.isNullOrEmpty(archiveCommand) ? executorFactory() @@ -85,7 +87,7 @@ public class CommitLogArchiver public static CommitLogArchiver disabled() { - return new CommitLogArchiver(null, null, null, Long.MAX_VALUE, TimeUnit.MICROSECONDS); + return new CommitLogArchiver(null, null, null, Long.MAX_VALUE, CommitLogPosition.NONE, TimeUnit.MICROSECONDS); } public static CommitLogArchiver construct() @@ -129,7 +131,27 @@ public class CommitLogArchiver { throw new RuntimeException("Unable to parse restore target time", e); } - return new CommitLogArchiver(archiveCommand, restoreCommand, restoreDirectories, restorePointInTime, precision); + + String snapshotPosition = commitlog_commands.getProperty("snapshot_commitlog_position"); + CommitLogPosition snapshotCommitLogPosition; + try + { + + snapshotCommitLogPosition = Strings.isNullOrEmpty(snapshotPosition) + ? CommitLogPosition.NONE + : CommitLogPosition.serializer.fromString(snapshotPosition); + } + catch (ParseException | NumberFormatException e) + { + throw new RuntimeException("Unable to parse snapshot commit log position", e); + } + + return new CommitLogArchiver(archiveCommand, + restoreCommand, + restoreDirectories, + restorePointInTime, + snapshotCommitLogPosition, + precision); } } catch (IOException e) diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java index 3ffb04ceae..3b3a21af3c 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogPosition.java @@ -18,8 +18,11 @@ package org.apache.cassandra.db.commitlog; import java.io.IOException; +import java.text.ParseException; import java.util.Comparator; +import com.google.common.base.Strings; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.util.DataInputPlus; @@ -118,5 +121,20 @@ public class CommitLogPosition implements Comparable { return TypeSizes.sizeof(clsp.segmentId) + TypeSizes.sizeof(clsp.position); } + + public CommitLogPosition fromString(String position) throws ParseException + { + if (Strings.isNullOrEmpty(position)) + return NONE; + String[] parts = position.split(","); + if (parts.length != 2) + throw new ParseException("Commit log position must be given as ,", 0); + return new CommitLogPosition(Long.parseLong(parts[0].trim()), Integer.parseInt(parts[1].trim())); + } + + public String toString(CommitLogPosition position) + { + return position == NONE ? "" : String.format("%d, %d", position.segmentId, position.position); + } } } diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index cbed01885b..74aa67dd01 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -130,7 +130,41 @@ public class CommitLogReplayer implements CommitLogReadHandler } } - IntervalSet filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt, localHostId); + IntervalSet filter; + final CommitLogPosition snapshotPosition = commitLog.archiver.snapshotCommitLogPosition; + if (snapshotPosition == CommitLogPosition.NONE) + { + // normal path: snapshot position is not explicitly specified, find it from sstables + if (!cfs.memtableWritesAreDurable()) + { + filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt, localHostId); + } + else + { + if (commitLog.archiver.restorePointInTime == Long.MAX_VALUE) + { + // Normal restart, everything is persisted and restored by the memtable itself. + filter = new IntervalSet<>(CommitLogPosition.NONE, CommitLog.instance.getCurrentPosition()); + } + else + { + // Point-in-time restore with a persistent memtable. In this case user should have restored + // the memtable from a snapshot and specified that snapshot's commit log position, reaching + // the "else" path below. + // If they haven't, do not filter any commit log data -- this supports a mode of operation where + // the user deletes old archived commit log segments when a snapshot completes -- but issue a + // message as this may be inefficient / not what the user wants. + logger.info("Point-in-time restore on a persistent memtable started without a snapshot time. " + + "All commit log data will be replayed."); + filter = IntervalSet.empty(); + } + } + } + else + { + // If the positions is specified, it must override whatever we calculate. + filter = new IntervalSet<>(CommitLogPosition.NONE, snapshotPosition); + } cfPersisted.put(cfs.metadata.id, filter); } CommitLogPosition globalPosition = firstNotCovered(cfPersisted.values()); @@ -212,12 +246,14 @@ public class CommitLogReplayer implements CommitLogReadHandler if (keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME)) flushingSystem = true; - futures.addAll(keyspace.flush()); + futures.addAll(keyspace.flush(ColumnFamilyStore.FlushReason.STARTUP)); } // also flush batchlog incase of any MV updates if (!flushingSystem) - futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceFlush()); + futures.add(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME) + .getColumnFamilyStore(SystemKeyspace.BATCHES) + .forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED)); FBUtilities.waitOnFutures(futures); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java index 814292f207..26dcdd39a6 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java @@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.Config; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; @@ -263,10 +263,9 @@ public class CompactionController extends AbstractCompactionController for (Memtable memtable : memtables) { - Partition partition = memtable.getPartition(key); - if (partition != null) + if (memtable.rowIterator(key) != null) { - minTimestampSeen = Math.min(minTimestampSeen, partition.stats().minTimestamp); + minTimestampSeen = Math.min(minTimestampSeen, memtable.getMinTimestamp()); hasTimestamp = true; } } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index caa249b9e3..dc08f5ae01 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -122,7 +122,7 @@ public class CompactionTask extends AbstractCompactionTask if (DatabaseDescriptor.isSnapshotBeforeCompaction()) { Instant creationTime = now(); - cfs.snapshotWithoutFlush(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime); + cfs.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime); } try (CompactionController controller = getCompactionController(transaction.originals())) diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java index 4c16ef7c86..d3f3a1e9c9 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java @@ -143,7 +143,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im public static LifecycleTransaction offline(OperationType operationType, Iterable readers) { // if offline, for simplicity we just use a dummy tracker - Tracker dummy = new Tracker(null, false); + Tracker dummy = Tracker.newDummyTracker(); dummy.addInitialSSTables(readers); dummy.apply(updateCompacting(emptySet(), readers)); return new LifecycleTransaction(dummy, operationType, readers); @@ -155,7 +155,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im @SuppressWarnings("resource") // log closed during postCleanup public static LifecycleTransaction offline(OperationType operationType) { - Tracker dummy = new Tracker(null, false); + Tracker dummy = Tracker.newDummyTracker(); return new LifecycleTransaction(dummy, new LogTransaction(operationType, dummy), Collections.emptyList()); } diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java index e15347b831..ab8a74bd1a 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java +++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java @@ -29,7 +29,7 @@ import com.google.common.collect.*; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.io.util.File; import org.slf4j.Logger; @@ -74,17 +74,23 @@ public class Tracker public final boolean loadsstables; /** + * @param columnFamilyStore * @param memtable Initial Memtable. Can be null. * @param loadsstables true to indicate to load SSTables (TODO: remove as this is only accessed from 2i) */ - public Tracker(Memtable memtable, boolean loadsstables) + public Tracker(ColumnFamilyStore columnFamilyStore, Memtable memtable, boolean loadsstables) { - this.cfstore = memtable != null ? memtable.cfs : null; + this.cfstore = columnFamilyStore; this.view = new AtomicReference<>(); this.loadsstables = loadsstables; this.reset(memtable); } + public static Tracker newDummyTracker() + { + return new Tracker(null, null, false); + } + public LifecycleTransaction tryModify(SSTableReader sstable, OperationType operationType) { return tryModify(singleton(sstable), operationType); diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java index 203f2faa19..958bc0d235 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/View.java +++ b/src/java/org/apache/cassandra/db/lifecycle/View.java @@ -26,7 +26,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.*; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -169,7 +169,7 @@ public class View return sstables.isEmpty() && liveMemtables.size() <= 1 && flushingMemtables.size() == 0 - && (liveMemtables.size() == 0 || liveMemtables.get(0).getOperations() == 0); + && (liveMemtables.size() == 0 || liveMemtables.get(0).operationCount() == 0); } @Override diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java new file mode 100644 index 0000000000..4688ef088c --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.ImmediateExecutor; +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.WrappedRunnable; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.concurrent.Promise; +import org.apache.cassandra.utils.memory.HeapPool; +import org.apache.cassandra.utils.memory.MemtableAllocator; +import org.apache.cassandra.utils.memory.MemtableCleaner; +import org.apache.cassandra.utils.memory.MemtablePool; +import org.apache.cassandra.utils.memory.NativePool; +import org.apache.cassandra.utils.memory.SlabPool; + +/** + * A memtable that uses memory tracked and maybe allocated via a MemtableAllocator from a MemtablePool. + * Provides methods of memory tracking and triggering flushes when the relevant limits are reached. + */ +public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithCommitlog +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractAllocatorMemtable.class); + + public static final MemtablePool MEMORY_POOL = AbstractAllocatorMemtable.createMemtableAllocatorPool(); + + protected final Owner owner; + protected final MemtableAllocator allocator; + + // Record the comparator of the CFS at the creation of the memtable. This + // is only used when a user update the CF comparator, to know if the + // memtable was created with the new or old comparator. + protected final ClusteringComparator initialComparator; + + private final long creationNano = Clock.Global.nanoTime(); + + private static MemtablePool createMemtableAllocatorPool() + { + long heapLimit = DatabaseDescriptor.getMemtableHeapSpaceInMiB() << 20; + long offHeapLimit = DatabaseDescriptor.getMemtableOffheapSpaceInMiB() << 20; + float memtableCleanupThreshold = DatabaseDescriptor.getMemtableCleanupThreshold(); + MemtableCleaner cleaner = AbstractAllocatorMemtable::flushLargestMemtable; + switch (DatabaseDescriptor.getMemtableAllocationType()) + { + case unslabbed_heap_buffers_logged: + return new HeapPool.Logged(heapLimit, memtableCleanupThreshold, cleaner); + case unslabbed_heap_buffers: + logger.debug("Memtables allocating with on-heap buffers"); + return new HeapPool(heapLimit, memtableCleanupThreshold, cleaner); + case heap_buffers: + logger.debug("Memtables allocating with on-heap slabs"); + return new SlabPool(heapLimit, 0, memtableCleanupThreshold, cleaner); + case offheap_buffers: + logger.debug("Memtables allocating with off-heap buffers"); + return new SlabPool(heapLimit, offHeapLimit, memtableCleanupThreshold, cleaner); + case offheap_objects: + logger.debug("Memtables allocating with off-heap objects"); + return new NativePool(heapLimit, offHeapLimit, memtableCleanupThreshold, cleaner); + default: + throw new AssertionError(); + } + } + + // only to be used by init(), to setup the very first memtable for the cfs + public AbstractAllocatorMemtable(AtomicReference commitLogLowerBound, TableMetadataRef metadataRef, Owner owner) + { + super(metadataRef, commitLogLowerBound); + this.allocator = MEMORY_POOL.newAllocator(metadataRef.toString()); + this.initialComparator = metadata.get().comparator; + this.owner = owner; + scheduleFlush(); + } + + public MemtableAllocator getAllocator() + { + return allocator; + } + + @Override + public boolean shouldSwitch(ColumnFamilyStore.FlushReason reason) + { + switch (reason) + { + case SCHEMA_CHANGE: + return initialComparator != metadata().comparator // If the CF comparator has changed, because our partitions reference the old one + || metadata().params.memtable.factory() != factory(); // If a different type of memtable is requested + case OWNED_RANGES_CHANGE: + return false; // by default we don't use the local ranges, thus this has no effect + default: + return true; + } + } + + public void metadataUpdated() + { + // We decided not to swap out this memtable, but if the flush period has changed we must schedule it for the + // new expiration time. + scheduleFlush(); + } + + public void localRangesUpdated() + { + // nothing to be done by default + } + + public void performSnapshot(String snapshotName) + { + throw new AssertionError("performSnapshot must be implemented if shouldSwitch(SNAPSHOT) can return false."); + } + + protected abstract Factory factory(); + + public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference commitLogUpperBound) + { + super.switchOut(writeBarrier, commitLogUpperBound); + allocator.setDiscarding(); + } + + public void discard() + { + super.discard(); + allocator.setDiscarded(); + } + + public String toString() + { + MemoryUsage usage = Memtable.getMemoryUsage(this); + return String.format("Memtable-%s@%s(%s serialized bytes, %s ops, %s)", + metadata.get().name, + hashCode(), + FBUtilities.prettyPrintMemory(getLiveDataSize()), + operationCount(), + usage); + } + + @Override + public void addMemoryUsageTo(MemoryUsage stats) + { + stats.ownershipRatioOnHeap += getAllocator().onHeap().ownershipRatio(); + stats.ownershipRatioOffHeap += getAllocator().offHeap().ownershipRatio(); + stats.ownsOnHeap += getAllocator().onHeap().owns(); + stats.ownsOffHeap += getAllocator().offHeap().owns(); + } + + public void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup) + { + getAllocator().onHeap().allocate(additionalSpace, opGroup); + } + + public void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup) + { + getAllocator().offHeap().allocate(additionalSpace, opGroup); + } + + void scheduleFlush() + { + int period = metadata().params.memtableFlushPeriodInMs; + if (period > 0) + scheduleFlush(owner, period); + } + + private static void scheduleFlush(Owner owner, int period) + { + logger.trace("scheduling flush in {} ms", period); + WrappedRunnable runnable = new WrappedRunnable() + { + protected void runMayThrow() + { + Memtable current = owner.getCurrentMemtable(); + if (current instanceof AbstractAllocatorMemtable) + ((AbstractAllocatorMemtable) current).flushIfPeriodExpired(); + } + }; + ScheduledExecutors.scheduledTasks.schedule(runnable, period, TimeUnit.MILLISECONDS); + } + + private void flushIfPeriodExpired() + { + int period = metadata().params.memtableFlushPeriodInMs; + if (period > 0 && (Clock.Global.nanoTime() - creationNano >= TimeUnit.MILLISECONDS.toNanos(period))) + { + if (isClean()) + { + // if we're still clean, instead of swapping just reschedule a flush for later + scheduleFlush(owner, period); + } + else + { + // we'll be rescheduled by the constructor of the Memtable. + owner.signalFlushRequired(AbstractAllocatorMemtable.this, + ColumnFamilyStore.FlushReason.MEMTABLE_PERIOD_EXPIRED); + } + } + } + + /** + * Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately + * queues it for flushing. If the memtable selected is flushed before this completes, no work is done. + */ + public static Future flushLargestMemtable() + { + float largestRatio = 0f; + AbstractAllocatorMemtable largestMemtable = null; + Memtable.MemoryUsage largestUsage = null; + float liveOnHeap = 0, liveOffHeap = 0; + // we take a reference to the current main memtable for the CF prior to snapping its ownership ratios + // to ensure we have some ordering guarantee for performing the switchMemtableIf(), i.e. we will only + // swap if the memtables we are measuring here haven't already been swapped by the time we try to swap them + for (Memtable currentMemtable : ColumnFamilyStore.activeMemtables()) + { + if (!(currentMemtable instanceof AbstractAllocatorMemtable)) + continue; + AbstractAllocatorMemtable current = (AbstractAllocatorMemtable) currentMemtable; + + // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF, + // both on- and off-heap, and select the largest of the two ratios to weight this CF + MemoryUsage usage = Memtable.newMemoryUsage(); + current.addMemoryUsageTo(usage); + + for (Memtable indexMemtable : current.owner.getIndexMemtables()) + if (indexMemtable instanceof AbstractAllocatorMemtable) + indexMemtable.addMemoryUsageTo(usage); + + float ratio = Math.max(usage.ownershipRatioOnHeap, usage.ownershipRatioOffHeap); + if (ratio > largestRatio) + { + largestMemtable = current; + largestUsage = usage; + largestRatio = ratio; + } + + liveOnHeap += usage.ownershipRatioOnHeap; + liveOffHeap += usage.ownershipRatioOffHeap; + } + + Promise returnFuture = new AsyncPromise<>(); + + if (largestMemtable != null) + { + float usedOnHeap = MEMORY_POOL.onHeap.usedRatio(); + float usedOffHeap = MEMORY_POOL.offHeap.usedRatio(); + float flushingOnHeap = MEMORY_POOL.onHeap.reclaimingRatio(); + float flushingOffHeap = MEMORY_POOL.offHeap.reclaimingRatio(); + logger.info("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}", + largestMemtable.owner, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap), + ratio(flushingOnHeap, flushingOffHeap), ratio(largestUsage.ownershipRatioOnHeap, largestUsage.ownershipRatioOffHeap)); + + Future flushFuture = largestMemtable.owner.signalFlushRequired(largestMemtable, ColumnFamilyStore.FlushReason.MEMTABLE_LIMIT); + flushFuture.addListener(() -> { + try + { + flushFuture.get(); + returnFuture.trySuccess(true); + } + catch (Throwable t) + { + returnFuture.tryFailure(t); + } + }, ImmediateExecutor.INSTANCE); + } + else + { + logger.debug("Flushing of largest memtable, not done, no memtable found"); + + returnFuture.trySuccess(false); + } + + return returnFuture; + } + + private static String ratio(float onHeap, float offHeap) + { + return String.format("%.2f/%.2f", onHeap, offHeap); + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java new file mode 100644 index 0000000000..1d683db910 --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; + +public abstract class AbstractMemtable implements Memtable +{ + protected final AtomicLong currentOperations = new AtomicLong(0); + protected final ColumnsCollector columnsCollector; + protected final StatsCollector statsCollector = new StatsCollector(); + // The smallest timestamp for all partitions stored in this memtable + protected AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE); + // The smallest local deletion time for all partitions in this memtable + protected AtomicInteger minLocalDeletionTime = new AtomicInteger(Integer.MAX_VALUE); + // Note: statsCollector has corresponding statistics to the two above, but starts with an epoch value which is not + // correct for their usage. + + protected TableMetadataRef metadata; + + public AbstractMemtable(TableMetadataRef metadataRef) + { + this.metadata = metadataRef; + this.columnsCollector = new ColumnsCollector(metadata.get().regularAndStaticColumns()); + } + + public TableMetadata metadata() + { + return metadata.get(); + } + + public long operationCount() + { + return currentOperations.get(); + } + + public long getMinTimestamp() + { + return minTimestamp.get(); + } + + public int getMinLocalDeletionTime() + { + return minLocalDeletionTime.get(); + } + + protected static void updateMin(AtomicLong minTracker, long newValue) + { + while (true) + { + long existing = minTracker.get(); + if (existing <= newValue) + break; + if (minTracker.compareAndSet(existing, newValue)) + break; + } + } + + protected static void updateMin(AtomicInteger minTracker, int newValue) + { + while (true) + { + int existing = minTracker.get(); + if (existing <= newValue) + break; + if (minTracker.compareAndSet(existing, newValue)) + break; + } + } + + RegularAndStaticColumns columns() + { + return columnsCollector.get(); + } + + EncodingStats encodingStats() + { + return statsCollector.get(); + } + + protected static class ColumnsCollector + { + private final HashMap predefined = new HashMap<>(); + private final ConcurrentSkipListSet extra = new ConcurrentSkipListSet<>(); + + ColumnsCollector(RegularAndStaticColumns columns) + { + for (ColumnMetadata def : columns.statics) + predefined.put(def, new AtomicBoolean()); + for (ColumnMetadata def : columns.regulars) + predefined.put(def, new AtomicBoolean()); + } + + public void update(RegularAndStaticColumns columns) + { + for (ColumnMetadata s : columns.statics) + update(s); + for (ColumnMetadata r : columns.regulars) + update(r); + } + + public void update(ColumnsCollector other) + { + for (Map.Entry v : other.predefined.entrySet()) + if (v.getValue().get()) + update(v.getKey()); + + extra.addAll(other.extra); + } + + private void update(ColumnMetadata definition) + { + AtomicBoolean present = predefined.get(definition); + if (present != null) + { + if (!present.get()) + present.set(true); + } + else + { + extra.add(definition); + } + } + + /** + * Get the current state of the columns set. + * + * Note: If this is executed while mutations are still being performed on the table (e.g. to prepare + * an sstable for streaming when Memtable.Factory.streamFromMemtable() is true), the resulting view may be + * in a somewhat inconsistent state (it may include partial updates, as well as miss updates older than + * ones it does include). + */ + public RegularAndStaticColumns get() + { + RegularAndStaticColumns.Builder builder = RegularAndStaticColumns.builder(); + for (Map.Entry e : predefined.entrySet()) + if (e.getValue().get()) + builder.add(e.getKey()); + return builder.addAll(extra).build(); + } + } + + protected static class StatsCollector + { + private final AtomicReference stats = new AtomicReference<>(EncodingStats.NO_STATS); + + public void update(EncodingStats newStats) + { + while (true) + { + EncodingStats current = stats.get(); + EncodingStats updated = current.mergeWith(newStats); + if (stats.compareAndSet(current, updated)) + return; + } + } + + public EncodingStats get() + { + return stats.get(); + } + } + + protected abstract class AbstractFlushablePartitionSet

implements FlushablePartitionSet

+ { + public long dataSize() + { + return getLiveDataSize(); + } + + public CommitLogPosition commitLogLowerBound() + { + return AbstractMemtable.this.getCommitLogLowerBound(); + } + + public CommitLogPosition commitLogUpperBound() + { + return AbstractMemtable.this.getCommitLogUpperBound(); + } + + public EncodingStats encodingStats() + { + return AbstractMemtable.this.encodingStats(); + } + + public RegularAndStaticColumns columns() + { + return AbstractMemtable.this.columns(); + } + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtableWithCommitlog.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtableWithCommitlog.java new file mode 100644 index 0000000000..d60fe866ab --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtableWithCommitlog.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.concurrent.OpOrder; + +/** + * Memtable that uses a commit log for persistence. Provides methods of tracking the commit log positions covered by + * it and safely switching between memtables. + */ +public abstract class AbstractMemtableWithCommitlog extends AbstractMemtable +{ + // The approximate lower bound by this memtable; must be <= commitLogLowerBound once our predecessor + // has been finalised, and this is enforced in the ColumnFamilyStore.setCommitLogUpperBound + private final CommitLogPosition approximateCommitLogLowerBound = CommitLog.instance.getCurrentPosition(); + // the precise lower bound of CommitLogPosition owned by this memtable; equal to its predecessor's commitLogUpperBound + private final AtomicReference commitLogLowerBound; + // the write barrier for directing writes to this memtable or the next during a switch + private volatile OpOrder.Barrier writeBarrier; + // the precise upper bound of CommitLogPosition owned by this memtable + private volatile AtomicReference commitLogUpperBound; + + public AbstractMemtableWithCommitlog(TableMetadataRef metadataRef, AtomicReference commitLogLowerBound) + { + super(metadataRef); + this.commitLogLowerBound = commitLogLowerBound; + } + + public CommitLogPosition getApproximateCommitLogLowerBound() + { + return approximateCommitLogLowerBound; + } + + public void switchOut(OpOrder.Barrier writeBarrier, AtomicReference commitLogUpperBound) + { + // This can prepare the memtable data for deletion; it will still be used while the flush is proceeding. + // A setDiscarded call will follow. + assert this.writeBarrier == null; + this.writeBarrier = writeBarrier; + this.commitLogUpperBound = commitLogUpperBound; + } + + public void discard() + { + assert writeBarrier != null : "Memtable must be switched out before being discarded."; + } + + // decide if this memtable should take the write, or if it should go to the next memtable + @Override + public boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition) + { + // if the barrier hasn't been set yet, then this memtable is still the newest and is taking ALL writes. + OpOrder.Barrier barrier = this.writeBarrier; + if (barrier == null) + return true; + // Note that if this races with taking the barrier the opGroup and commit log position we were given must + // necessarily be before the barrier and any LastCommitLogPosition is set, thus this function will return true + // and no update to commitLogUpperBound is necessary. + + // If the barrier has been set and issued, but is in the past, we are definitely destined for a future memtable. + // Because we issue the barrier after taking LastCommitLogPosition and mutations take their position after + // taking the opGroup, this condition also ensures the given commit log position is greater than the chosen + // upper bound. + if (!barrier.isAfter(opGroup)) + return false; + // We are in the segment of time between the barrier is constructed (and the memtable is switched out) + // and the barrier is issued. + // if we aren't durable we are directed only by the barrier + if (commitLogPosition == null) + return true; + while (true) + { + // If the CL boundary has been set, the mutation can be accepted depending on whether it falls before it. + // However, if it has not been set, the old sstable must still accept writes but we must also ensure that + // their positions are accounted for in the boundary (as there may be a delay between taking the log + // position for the boundary and setting it where a mutation sneaks in). + // Thus, if the boundary hasn't been finalised yet, we simply update it to the max of its current value and + // ours; this permits us to coordinate a safe boundary, as the boundary choice is made atomically wrt our + // max() maintenance, so an operation cannot sneak into the past. + CommitLogPosition currentLast = commitLogUpperBound.get(); + if (currentLast instanceof LastCommitLogPosition) + return currentLast.compareTo(commitLogPosition) >= 0; + if (currentLast != null && currentLast.compareTo(commitLogPosition) >= 0) + return true; + if (commitLogUpperBound.compareAndSet(currentLast, commitLogPosition)) + return true; + } + } + + public CommitLogPosition getCommitLogLowerBound() + { + return commitLogLowerBound.get(); + } + + public CommitLogPosition getCommitLogUpperBound() + { + return commitLogUpperBound.get(); + } + + public boolean mayContainDataBefore(CommitLogPosition position) + { + return approximateCommitLogLowerBound.compareTo(position) < 0; + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java new file mode 100644 index 0000000000..6717e64bed --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; + +import com.google.common.base.Throwables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.DiskBoundaries; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.commitlog.IntervalSet; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableMultiWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.FBUtilities; + +public class Flushing +{ + private static final Logger logger = LoggerFactory.getLogger(Flushing.class); + + private Flushing() // prevent instantiation + { + } + + public static List flushRunnables(ColumnFamilyStore cfs, + Memtable memtable, + LifecycleTransaction txn) + { + DiskBoundaries diskBoundaries = cfs.getDiskBoundaries(); + List boundaries = diskBoundaries.positions; + List locations = diskBoundaries.directories; + if (boundaries == null) + { + FlushRunnable runnable = flushRunnable(cfs, memtable, null, null, txn, null); + return Collections.singletonList(runnable); + } + + List runnables = new ArrayList<>(boundaries.size()); + PartitionPosition rangeStart = boundaries.get(0).getPartitioner().getMinimumToken().minKeyBound(); + try + { + for (int i = 0; i < boundaries.size(); i++) + { + PartitionPosition t = boundaries.get(i); + FlushRunnable runnable = flushRunnable(cfs, memtable, rangeStart, t, txn, locations.get(i)); + + runnables.add(runnable); + rangeStart = t; + } + return runnables; + } + catch (Throwable e) + { + throw Throwables.propagate(abortRunnables(runnables, e)); + } + } + + @SuppressWarnings("resource") // writer owned by runnable, to be closed or aborted by its caller + static FlushRunnable flushRunnable(ColumnFamilyStore cfs, + Memtable memtable, + PartitionPosition from, + PartitionPosition to, + LifecycleTransaction txn, + Directories.DataDirectory flushLocation) + { + Memtable.FlushablePartitionSet flushSet = memtable.getFlushSet(from, to); + SSTableFormat.Type formatType = SSTableFormat.Type.current(); + long estimatedSize = formatType.info.getWriterFactory().estimateSize(flushSet); + + Descriptor descriptor = flushLocation == null + ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), formatType) + : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), formatType); + + SSTableMultiWriter writer = createFlushWriter(cfs, + flushSet, + txn, + descriptor, + flushSet.partitionCount()); + + return new FlushRunnable(flushSet, writer, cfs.metric, true); + } + + public static Throwable abortRunnables(List runnables, Throwable t) + { + if (runnables != null) + for (FlushRunnable runnable : runnables) + t = runnable.writer.abort(t); + return t; + } + + public static class FlushRunnable implements Callable + { + private final Memtable.FlushablePartitionSet toFlush; + + private final SSTableMultiWriter writer; + private final TableMetrics metrics; + private final boolean isBatchLogTable; + private final boolean logCompletion; + + public FlushRunnable(Memtable.FlushablePartitionSet flushSet, + SSTableMultiWriter writer, + TableMetrics metrics, + boolean logCompletion) + { + this.toFlush = flushSet; + this.writer = writer; + this.metrics = metrics; + this.isBatchLogTable = toFlush.metadata() == SystemKeyspace.Batches; + this.logCompletion = logCompletion; + } + + private void writeSortedContents() + { + logger.info("Writing {}, flushed range = [{}, {})", toFlush.memtable(), toFlush.from(), toFlush.to()); + + // (we can't clear out the map as-we-go to free up memory, + // since the memtable is being used for queries in the "pending flush" category) + for (Partition partition : toFlush) + { + // Each batchlog partition is a separate entry in the log. And for an entry, we only do 2 + // operations: 1) we insert the entry and 2) we delete it. Further, BL data is strictly local, + // we don't need to preserve tombstones for repair. So if both operation are in this + // memtable (which will almost always be the case if there is no ongoing failure), we can + // just skip the entry (CASSANDRA-4667). + if (isBatchLogTable && !partition.partitionLevelDeletion().isLive() && partition.hasRows()) + continue; + + if (!partition.isEmpty()) + { + try (UnfilteredRowIterator iter = partition.unfilteredIterator()) + { + writer.append(iter); + } + } + } + + if (logCompletion) + { + long bytesFlushed = writer.getFilePointer(); + logger.info("Completed flushing {} ({}) for commitlog position {}", + writer.getFilename(), + FBUtilities.prettyPrintMemory(bytesFlushed), + toFlush.memtable().getCommitLogUpperBound()); + // Update the metrics + metrics.bytesFlushed.inc(bytesFlushed); + } + } + + @Override + public SSTableMultiWriter call() + { + writeSortedContents(); + return writer; + // We don't close the writer on error as the caller aborts all runnables if one happens. + } + + @Override + public String toString() + { + return "Flush " + toFlush.metadata().keyspace + '.' + toFlush.metadata().name; + } + } + + public static SSTableMultiWriter createFlushWriter(ColumnFamilyStore cfs, + Memtable.FlushablePartitionSet flushSet, + LifecycleTransaction txn, + Descriptor descriptor, + long partitionCount) + { + MetadataCollector sstableMetadataCollector = new MetadataCollector(flushSet.metadata().comparator) + .commitLogIntervals(new IntervalSet<>(flushSet.commitLogLowerBound(), + flushSet.commitLogUpperBound())); + + return cfs.createSSTableMultiWriter(descriptor, + partitionCount, + ActiveRepairService.UNREPAIRED_SSTABLE, + ActiveRepairService.NO_PENDING_REPAIR, + false, + sstableMetadataCollector, + new SerializationHeader(true, + flushSet.metadata(), + flushSet.columns(), + flushSet.encodingStats()), + txn); + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java new file mode 100644 index 0000000000..a4fffafb32 --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.UnfilteredSource; +import org.apache.cassandra.index.transactions.UpdateTransaction; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.OpOrder; + +/** + * Memtable interface. This defines the operations the ColumnFamilyStore can perform with memtables. + * They are of several types: + * - construction factory interface + * - write and read operations: put, rowIterator and partitionIterator + * - statistics and features, including partition counts, data size, encoding stats, written columns + * - memory usage tracking, including methods of retrieval and of adding extra allocated space (used non-CFS secondary + * indexes) + * - flush functionality, preparing the set of partitions to flush for given ranges + * - lifecycle management, i.e. operations that prepare and execute switch to a different memtable, together + * with ways of tracking the affected commit log spans + * + * See Memtable_API.md for details on implementing and using alternative memtable implementations. + */ +public interface Memtable extends Comparable, UnfilteredSource +{ + // Construction + + /** + * Factory interface for constructing memtables, and querying write durability features. + * + * The factory is chosen using the MemtableParams class (passed as argument to + * {@code CREATE TABLE ... WITH memtable = ''} where the configuration definition is a map given + * under {@code memtable_configurations} in cassandra.yaml). To make that possible, implementations must provide + * either a static {@code FACTORY} field (if they accept no further option) or a static + * {@code factory(Map)} method. In the latter case, the method should avoid creating + * multiple instances of the factory for the same parameters, or factories should at least implement hashCode and + * equals. + */ + interface Factory + { + /** + * Create a memtable. + * + * @param commitLogLowerBound A commit log lower bound for the new memtable. This will be equal to the previous + * memtable's upper bound and defines the span of positions that any flushed sstable + * will cover. + * @param metadaRef Pointer to the up-to-date table metadata. + * @param owner Owning objects that will receive flush requests triggered by the memtable (e.g. on expiration). + */ + Memtable create(AtomicReference commitLogLowerBound, TableMetadataRef metadaRef, Owner owner); + + /** + * If the memtable can achieve write durability directly (i.e. using some feature other than the commitlog, e.g. + * persistent memory), it can return true here, in which case the commit log will not store mutations in this + * table. + * Note that doing so will prevent point-in-time restores and changed data capture, thus a durable memtable must + * allow the option of turning commit log writing on even if it does not need it. + */ + default boolean writesShouldSkipCommitLog() + { + return false; + } + + /** + * This should be true if the memtable can achieve write durability for crash recovery directly (i.e. using some + * feature other than the commitlog, e.g. persistent memory). + * Setting this flag to true means that the commitlog should not replay mutations for this table on restart, + * and that it should not try to preserve segments that contain relevant data. + * Unless writesShouldSkipCommitLog() is also true, writes will be recorded in the commit log as they may be + * needed for changed data capture or point-in-time restore. + */ + default boolean writesAreDurable() + { + return false; + } + + /** + * Normally we can receive streamed sstables directly, skipping the memtable stage (zero-copy-streaming). When + * the memtable is the primary data store (e.g. persistent memtables), it will usually prefer to receive the + * data instead. + * + * If this returns true, all streamed sstables's content will be read and replayed as mutations, disabling + * zero-copy streaming. + */ + default boolean streamToMemtable() + { + return false; + } + + /** + * When we need to stream data, we usually flush and stream the resulting sstables. This will not work correctly + * if the memtable does not want to flush for streaming (e.g. persistent memtables acting as primary data + * store), because data (not just recent) will be missing from the streamed view. Such memtables must present + * their data separately for streaming. + * In other words if the memtable returns false on shouldSwitch(STREAMING/REPAIR), its factory must return true + * here. + * + * If this flag returns true, streaming will write the relevant content that resides in the memtable to + * temporary sstables, stream these sstables and then delete them. + */ + default boolean streamFromMemtable() + { + return false; + } + + /** + * Override this method to include implementation-specific memtable metrics in the table metrics. + * + * Memtable metrics lifecycle matches table lifecycle. It is the table that owns the metrics and + * decides when to release them. + */ + default TableMetrics.ReleasableMetric createMemtableMetrics(TableMetadataRef metadataRef) + { + return null; + } + } + + /** + * Interface for providing signals back and requesting information from the owner, i.e. the object that controls the + * memtable. This is usually the ColumnFamilyStore; the interface is used to limit the dependency of memtables on + * the details of its implementation. + */ + interface Owner + { + /** Signal to the owner that a flush is required (e.g. in response to hitting space limits) */ + Future signalFlushRequired(Memtable memtable, ColumnFamilyStore.FlushReason reason); + + /** Get the current memtable for this owner. Used to avoid capturing memtable in scheduled flush tasks. */ + Memtable getCurrentMemtable(); + + /** + * Collect the index memtables flushed together with this. Used to accurately calculate memory that would be + * freed by a flush. + */ + Iterable getIndexMemtables(); + + /** + * Construct a list of boundaries that split the locally-owned ranges into the given number of shards, + * splitting the owned space evenly. It is up to the memtable to use this information. + * Any changes in the ring structure (e.g. added or removed nodes) will invalidate the splits; in such a case + * the memtable will be sent a {@link #shouldSwitch}(OWNED_RANGES_CHANGE) and, should that return false, a + * {@link #localRangesUpdated()} call. + */ + ShardBoundaries localRangeSplits(int shardCount); + } + + // Main write and read operations + + /** + * Put new data in the memtable. This operation may block until enough memory is available in the memory pool. + * + * @param update the partition update, may be a new partition or an update to an existing one + * @param indexer receives information about the update's effect + * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a + * flush to free space. + * + * @return the smallest timestamp delta between corresponding rows from existing and update. A + * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition + * and those in {@code update}. See CASSANDRA-7979. + */ + long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup); + + // Read operations are provided by the UnfilteredSource interface. + + // Statistics + + /** Number of partitions stored in the memtable */ + long partitionCount(); + + /** Size of the data not accounting for any metadata / mapping overheads */ + long getLiveDataSize(); + + /** + * Number of "operations" (in the sense defined in {@link PartitionUpdate#operationCount()}) the memtable has + * executed. + */ + long operationCount(); + + /** + * The table's definition metadata. + * + * Note that this tracks the current state of the table and is not necessarily the same as what was used to create + * the memtable. + */ + TableMetadata metadata(); + + + // Memory usage tracking + + /** + * Add this memtable's used memory to the given usage object. This can be used to retrieve a single memtable's usage + * as well as to combine the ones of related sstables (e.g. a table and its table-based secondary indexes). + */ + void addMemoryUsageTo(MemoryUsage usage); + + + /** + * Creates a holder for memory usage collection. + * + * This is used to track on- and off-heap memory, as well as the ratio to the total permitted memtable memory. + */ + static MemoryUsage newMemoryUsage() + { + return new MemoryUsage(); + } + + /** + * Shorthand for the getting a given table's memory usage. + * Implemented as a static to prevent implementations altering expectations by e.g. returning a cached object. + */ + static MemoryUsage getMemoryUsage(Memtable memtable) + { + MemoryUsage usage = newMemoryUsage(); + memtable.addMemoryUsageTo(usage); + return usage; + } + + @NotThreadSafe + class MemoryUsage + { + /** On-heap memory used in bytes */ + public long ownsOnHeap = 0; + /** Off-heap memory used in bytes */ + public long ownsOffHeap = 0; + /** On-heap memory as ratio to permitted memtable space */ + public float ownershipRatioOnHeap = 0.0f; + /** Off-heap memory as ratio to permitted memtable space */ + public float ownershipRatioOffHeap = 0.0f; + + @Override + public String toString() + { + return String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap", + FBUtilities.prettyPrintMemory(ownsOnHeap), + ownershipRatioOnHeap * 100, + FBUtilities.prettyPrintMemory(ownsOffHeap), + ownershipRatioOffHeap * 100); + } + } + + /** + * Adjust the used on-heap space by the given size (e.g. to reflect memory used by a non-table-based index). + * This operation may block until enough memory is available in the memory pool. + * + * @param additionalSpace the number of allocated bytes + * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a + * flush to free space. + */ + void markExtraOnHeapUsed(long additionalSpace, OpOrder.Group opGroup); + + /** + * Adjust the used off-heap space by the given size (e.g. to reflect memory used by a non-table-based index). + * This operation may block until enough memory is available in the memory pool. + * + * @param additionalSpace the number of allocated bytes + * @param opGroup write operation group, used to permit the operation to complete if it is needed to complete a + * flush to free space. + */ + void markExtraOffHeapUsed(long additionalSpace, OpOrder.Group opGroup); + + + // Flushing + + /** + * Get the collection of data between the given partition boundaries in a form suitable for flushing. + */ + FlushablePartitionSet getFlushSet(PartitionPosition from, PartitionPosition to); + + /** + * A collection of partitions for flushing plus some information required for writing an sstable. + * + * Note that the listed entries must conform with the specified metadata. In particular, if the memtable is still + * being written to, care must be taken to not list newer items as they may violate the bounds collected by the + * encoding stats or refer to columns that don't exist in the collected columns set. + */ + interface FlushablePartitionSet

extends Iterable

, SSTableWriter.SSTableSizeParameters + { + Memtable memtable(); + + PartitionPosition from(); + PartitionPosition to(); + + /** The commit log position at the time that this memtable was created */ + CommitLogPosition commitLogLowerBound(); + /** The commit log position at the time that this memtable was switched out */ + CommitLogPosition commitLogUpperBound(); + + /** The set of all columns that have been written */ + RegularAndStaticColumns columns(); + /** Statistics required for writing an sstable efficiently */ + EncodingStats encodingStats(); + + default TableMetadata metadata() + { + return memtable().metadata(); + } + + default boolean isEmpty() + { + return partitionCount() > 0; + } + } + + + // Lifecycle management + + /** + * Called to tell the memtable that it is being switched out and will be flushed (or dropped) and discarded. + * Will be followed by a {@link #getFlushSet} call (if the table is not truncated or dropped), and a + * {@link #discard}. + * + * @param writeBarrier The barrier that will signal that all writes to this memtable have completed. That is, the + * point after which writes cannot be accepted by this memtable (it is permitted for writes + * before this barrier to go into the next; see {@link #accepts}). + * @param commitLogUpperBound The upper commit log position for this memtable. The value may be modified after this + * call and will match the next memtable's lower commit log bound. + */ + void switchOut(OpOrder.Barrier writeBarrier, AtomicReference commitLogUpperBound); + + /** + * This memtable is no longer in use or required for outstanding flushes or operations. + * All held memory must be released. + */ + void discard(); + + /** + * Decide if this memtable should take a write with the given parameters, or if the write should go to the next + * memtable. This enforces that no writes after the barrier set by {@link #switchOut} can be accepted, and + * is also used to define a shared commit log bound as the upper for this memtable and lower for the next. + */ + boolean accepts(OpOrder.Group opGroup, CommitLogPosition commitLogPosition); + + /** Approximate commit log lower bound, <= getCommitLogLowerBound, used as a time stamp for ordering */ + CommitLogPosition getApproximateCommitLogLowerBound(); + + /** The commit log position at the time that this memtable was created */ + CommitLogPosition getCommitLogLowerBound(); + + /** The commit log position at the time that this memtable was switched out */ + CommitLogPosition getCommitLogUpperBound(); + + /** True if the memtable can contain any data that was written before the given commit log position */ + boolean mayContainDataBefore(CommitLogPosition position); + + /** True if the memtable contains no data */ + boolean isClean(); + + /** Order memtables by time as reflected in the commit log position at time of construction */ + default int compareTo(Memtable that) + { + return this.getApproximateCommitLogLowerBound().compareTo(that.getApproximateCommitLogLowerBound()); + } + + /** + * Decides whether the memtable should be switched/flushed for the passed reason. + * Normally this will return true, but e.g. persistent memtables may choose not to flush. Returning false will + * trigger further action for some reasons: + * - SCHEMA_CHANGE will be followed by metadataUpdated(). + * - OWNED_RANGES_CHANGE will be followed by localRangesUpdated(). + * - SNAPSHOT will be followed by performSnapshot(). + * - STREAMING/REPAIR will be followed by creating a FlushSet for the streamed/repaired ranges. This data will be + * used to create sstables, which will be streamed and then deleted. + * This will not be called to perform truncation or drop (in that case the memtable is unconditionally dropped), + * but a flush may nevertheless be requested in that case to prepare a snapshot. + */ + boolean shouldSwitch(ColumnFamilyStore.FlushReason reason); + + /** + * Called when the table's metadata is updated. The memtable's metadata reference now points to the new version. + * This will not be called if {@link #shouldSwitch)(SCHEMA_CHANGE) returns true, as the memtable will be swapped out + * instead. + */ + void metadataUpdated(); + + /** + * Called when the known ranges have been updated and owner.localRangeSplits() may return different values. + * This will not be called if {@link #shouldSwitch)(OWNED_RANGES_CHANGE) returns true, as the memtable will be + * swapped out instead. + */ + void localRangesUpdated(); + + /** + * If the memtable needs to do some special action for snapshots (e.g. because it is persistent and does not want + * to flush), it should return false on the above with reason SNAPSHOT and implement this method. + */ + void performSnapshot(String snapshotName); + + /** + * Special commit log position marker used in the upper bound marker setting process + * (see {@link org.apache.cassandra.db.ColumnFamilyStore#setCommitLogUpperBound} and {@link AbstractMemtable#accepts}) + */ + public static final class LastCommitLogPosition extends CommitLogPosition + { + public LastCommitLogPosition(CommitLogPosition copy) + { + super(copy.segmentId, copy.position); + } + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable_API.md b/src/java/org/apache/cassandra/db/memtable/Memtable_API.md new file mode 100644 index 0000000000..39f9b201af --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/Memtable_API.md @@ -0,0 +1,177 @@ +# Memtable API + +[CEP-11](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-11%3A+Pluggable+memtable+implementations) +/ [CASSANDRA-17034](https://issues.apache.org/jira/browse/CASSANDRA-17034) + +## Configuration specification + +Memtable types and options are specified using memtable "configurations", which specify an implementation class +and its parameters. + +The memtable configurations are defined in `cassandra.yaml`, using the following format: + +```yaml +memtable: + configurations: + 〈configuration name〉: + class_name: 〈class〉 + inherits: 〈configuration name〉 + parameters: + 〈parameters〉 +``` + +Configurations can copy the properties from others, including being full copies of another, which can be useful for +easily remapping one name to another configuration. + +The default memtable configuration is named `default`. It can be overridden if the yaml specifies it (including +using inheritance to copy another configuration), and it can be inherited, even if it is not explicitly defined in +the yaml (e.g. to change some parameter but not the memtable class). + +Examples: + +```yaml +memtable: + configurations: + more_shards: + inherits: default + parameters: + shards: 32 +``` + +```yaml +memtable: + configurations: + skiplist: + class_name: SkipListMemtable + sharded: + class_name: ShardedSkipListMemtable + default: + inherits: sharded +``` + +Note that the database will only validate the memtable class and its parameters when a configuration needs to be +instantiated for a table. + +## Memtable selection + +Once a configuration has been defined, it can be used by specifying it in the `memtable` parameter of a `CREATE TABLE` +or `ALTER TABLE` statement, for example: + +``` +CREATE TABLE ... WITH ... AND memtable = 'trie'; +``` +or +``` +ALTER TABLE ... WITH memtable = 'skiplist'; +``` + +If a memtable is not specified, the configuration `default` will be used. To reset a table to the default memtable, +use +``` +ALTER TABLE ... WITH memtable = 'default'; +``` + +The memtable configuration selection is per table, i.e. it will be propagated to all nodes in the cluster. If some nodes +do not have a definition for that configuration, cannot instantiate the class, or are still on a version of Cassandra +before 4.1, they will reject the schema change. We therefore recommend using a separate `ALTER` statement to change a +table's memtable implementation; upgrading all nodes to 4.1 or later is required to use the API. + +As additional safety when first deploying an alternative implementation to a production cluster, one may consider +first deploying a remapped `default` configuration to all nodes in the cluster, switching the schema to reference +it, and then changing the implementation by modifying the configuration one node at a time. + +For example, a remapped default can be specified with this: +```yaml +memtable: + configurations: + better_memtable: + inherits: default +``` +selected via +``` +ALTER TABLE production_table WITH memtable = 'better_memtable'; +``` +and later switched one node at a time to +```yaml +memtable: + configurations: + our_memtable: + class_name: ... + better_memtable: + inherits: our_memtable +``` + +## Memtable implementation + +A memtable implementation is an implementation of the `Memtable` interface. The main work of the class will be +performed by the `put`, `rowIterator` and `partitionIterator` methods, used to write and read information to/from the +memtable. In addition to this, the implementation must support retrieval of the content in a form suitable for flushing +(via `getFlushSet`), memory use and statistics tracking, mechanics for triggering a flush for reasons +controlled by the memtable (e.g. exhaustion of the given memory allocation), and finally mechanisms for tracking the +commit log spans covered by a memtable. + +Abstract classes that provide the latter parts of the functionality (expected to be shared by most +implementations) are provided as the `AbstractMemtable` (statistics tracking), `AbstractMemtableWithCommitlog` (adds +commit log span tracking) and `AbstractAllocatorMemtable` (adds memory management via the `Allocator` class, together +with flush triggering on memory use and time interval expiration). + +The memtable API also gives the memtable some control over flushing and the functioning of the commit log. The former +is there to permit memtables that operate long-term and/or can handle some events internally, without a need to flush. +The latter enables memtables that have an internal durability mechanism, such as ones using persistent memory or a +tightly integrated commit log (e.g. using the commit log buffers for memtable data storage). + +The memtable implementation must also provide a mechanism for memtable construction called a memtable "factory" +(the `Memtable.Factory` interface). Some features of the implementation may be needed before an instance is created or +where the memtable instance is not accessible. To make working with them more straightforward, the following +memtable-controlled options are implemented on the factory: + +- `boolean writesAreDurable()` should return true if the memtable has its own persistence mechanism and does not want + the commitlog to provide persistence. In this case the commit log can still store the writes for changed-data-capture (CDC) + or point-in-time restore (PITR), but it need not keep them for replay until the memtable is flushed. +- `boolean writesShouldSkipCommitLog()` should return true if the memtable does not want the commit log to store any of + its data. The expectation for this flag is that a persistent memtable will take a configuration parameter to turn this + option on to improve performance. Enabling this flag is not compatible with CDC or PITR. +- `boolean streamToMemtable()` and `boolean streamFromMemtable()` should return true if the memtable is long-lived and + cannot flush to facilitate streaming. In this case the streaming code will implement the process in a way that + retrieves data in the memtable before sending, and applies received data in the memtable instead of directly creating + an sstable. + +### Instantiation and configuration + +The memtables are instantiated by the factory, which is constructed via reflection on creating a `ColumnFamilyStore` or +altering the table's configuration. + +Memtable classes must either contain a static `FACTORY` field (if they take no arguments other than class), or implement +a `factory(Map)` method, which is called using the configuration `parameters`. For validation, the +latter should consume any further options (using `map.remove`). + +The `MemtableParams` class will look for the specified class name (prefixed with `org.apache.cassandra.db.memtable.` +if only a short name was given), then look for a `factory` method. If it finds one, it will call it with a copy of the +supplied parameters; if it does not, it will look for the `FACTORY` field and use its value if found. It will error out +if the class was not found, if neither the method or field was found, or if the user supplied parameters that did not +get consumed. + +Because multiple configurations and tables may use the same parameters, it is expected that the factory method will +store and reuse constructed factories to avoid wasting space for duplicate objects (this is typical for configuration +objects in Cassandra). + +At this time many of the configuration parameters for memtables are still configured using top-level parameters like +`memtable_allocation_type` in `cassandra.yaml` and `memtable_flush_period_in_ms` in the table schema. + + +### Sample implementation + +The API comes with a proof-of-concept implementation, a sharded skip-list memtable implemented by the +`ShardedSkipListMemtable` class. The difference between this and the default memtable is that the sharded version breaks +the token space served by the node into roughly equal regions and uses a separate skip-list for each shard. Sharding +spreads the write concurrency among these independent skip lists, reducing congestion and can lead to significantly +improved write throughput. + +This implementation takes two parameters, `shards` which specifies the number of shards to split into (by default, the +number of CPU threads available to the process) and `serialize_writes`, which, if set to `true` causes writes to the +memtable to be synchronized. The latter can be useful to minimize space and time wasted for unsuccesful lockless +partition modification where a new copy of the partition would be prepared but not used due to concurrent modification. +Regardless of the setting, reads can always execute in parallel, including concurrently with writes. + +Please note that sharding cannot be used with non-hashing partitioners (i.e. `ByteOrderPartitioner` or +`OrderPreservingPartitioner`). \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java new file mode 100644 index 0000000000..fb9cc98426 --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/ShardBoundaries.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.memtable; + +import java.util.Arrays; +import java.util.List; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.Token; + +/** + * Holds boundaries (tokens) used to map a particular token (so partition key) to a shard id. + * In practice, each keyspace has its associated boundaries, see {@link Keyspace}. + *

+ * Technically, if we use {@code n} shards, this is a list of {@code n-1} tokens and each token {@code tk} gets assigned + * to the shard ID corresponding to the slot of the smallest token in the list that is greater to {@code tk}, or {@code n} + * if {@code tk} is bigger than any token in the list. + */ +public class ShardBoundaries +{ + private static final Token[] EMPTY_TOKEN_ARRAY = new Token[0]; + + // Special boundaries that map all tokens to one shard. + // These boundaries will be used in either of these cases: + // - there is only 1 shard configured + // - the default partitioner doesn't support splitting + // - the keyspace is local system keyspace + public static final ShardBoundaries NONE = new ShardBoundaries(EMPTY_TOKEN_ARRAY, -1); + + private final Token[] boundaries; + public final long ringVersion; + + @VisibleForTesting + public ShardBoundaries(Token[] boundaries, long ringVersion) + { + this.boundaries = boundaries; + this.ringVersion = ringVersion; + } + + public ShardBoundaries(List boundaries, long ringVersion) + { + this(boundaries.toArray(EMPTY_TOKEN_ARRAY), ringVersion); + } + + /** + * Computes the shard to use for the provided token. + */ + public int getShardForToken(Token tk) + { + for (int i = 0; i < boundaries.length; i++) + { + if (tk.compareTo(boundaries[i]) < 0) + return i; + } + return boundaries.length; + } + + /** + * Computes the shard to use for the provided key. + */ + public int getShardForKey(PartitionPosition key) + { + // Boundaries are missing if the node is not sufficiently initialized yet + if (boundaries.length == 0) + return 0; + + assert (key.getPartitioner() == DatabaseDescriptor.getPartitioner()); + return getShardForToken(key.getToken()); + } + + /** + * The number of shards that this boundaries support, that is how many different shard ids {@link #getShardForToken} might + * possibly return. + * + * @return the number of shards supported by theses boundaries. + */ + public int shardCount() + { + return boundaries.length + 1; + } + + @Override + public String toString() + { + if (boundaries.length == 0) + return "shard 0: (min, max)"; + + StringBuilder sb = new StringBuilder(); + sb.append("shard 0: (min, ").append(boundaries[0]).append(") "); + for (int i = 0; i < boundaries.length - 1; i++) + sb.append("shard ").append(i+1).append(": (").append(boundaries[i]).append(", ").append(boundaries[i+1]).append("] "); + sb.append("shard ").append(boundaries.length).append(": (").append(boundaries[boundaries.length-1]).append(", max)"); + return sb.toString(); + } + + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ShardBoundaries that = (ShardBoundaries) o; + + return Arrays.equals(boundaries, that.boundaries); + } + + public int hashCode() + { + return Arrays.hashCode(boundaries); + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java new file mode 100644 index 0000000000..6bbbc061fb --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.memtable; + +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.AtomicBTreePartition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.IncludingExcludingBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.index.transactions.UpdateTransaction; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.memory.MemtableAllocator; +import org.github.jamm.Unmetered; + +/** + * A proof-of-concept sharded memtable implementation. This implementation splits the partition skip-list into several + * independent skip-lists each covering a roughly equal part of the token space served by this node. This reduces + * congestion of the skip-list from concurrent writes and can lead to improved write throughput. + * + * The implementation takes two parameters: + * - shards: the number of shards to split into. + * - serialize_writes: if false, each shard may serve multiple writes in parallel; if true, writes to each shard are + * synchronized. + * + * Also see Memtable_API.md. + */ +public class ShardedSkipListMemtable extends AbstractAllocatorMemtable +{ + private static final Logger logger = LoggerFactory.getLogger(ShardedSkipListMemtable.class); + + public static final String SHARDS_OPTION = "shards"; + public static final String LOCKING_OPTION = "serialize_writes"; + + // The boundaries for the keyspace as they were calculated when the memtable is created. + // The boundaries will be NONE for system keyspaces or if StorageService is not yet initialized. + // The fact this is fixed for the duration of the memtable lifetime, guarantees we'll always pick the same shard + // for a given key, even if we race with the StorageService initialization or with topology changes. + @Unmetered + final ShardBoundaries boundaries; + + /** + * Core-specific memtable regions. All writes must go through the specific core. The data structures used + * are concurrent-read safe, thus reads can be carried out from any thread. + */ + final MemtableShard[] shards; + + @VisibleForTesting + public static final String SHARD_COUNT_PROPERTY = "cassandra.memtable.shard.count"; + + // default shard count, used when a specific number of shards is not specified in the parameters + private static final int SHARD_COUNT = Integer.getInteger(SHARD_COUNT_PROPERTY, FBUtilities.getAvailableProcessors()); + + private final Factory factory; + + // only to be used by init(), to setup the very first memtable for the cfs + ShardedSkipListMemtable(AtomicReference commitLogLowerBound, + TableMetadataRef metadataRef, + Owner owner, + Integer shardCountOption, + Factory factory) + { + super(commitLogLowerBound, metadataRef, owner); + int shardCount = shardCountOption != null ? shardCountOption : SHARD_COUNT; + this.boundaries = owner.localRangeSplits(shardCount); + this.shards = generatePartitionShards(boundaries.shardCount(), allocator, metadataRef); + this.factory = factory; + } + + private static MemtableShard[] generatePartitionShards(int splits, + MemtableAllocator allocator, + TableMetadataRef metadata) + { + MemtableShard[] partitionMapContainer = new MemtableShard[splits]; + for (int i = 0; i < splits; i++) + partitionMapContainer[i] = new MemtableShard(metadata, allocator); + + return partitionMapContainer; + } + + public boolean isClean() + { + for (MemtableShard shard : shards) + if (!shard.isEmpty()) + return false; + return true; + } + + @Override + protected Memtable.Factory factory() + { + return factory; + } + + /** + * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate + * OpOrdering. + * + * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null + */ + public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + { + DecoratedKey key = update.partitionKey(); + MemtableShard shard = shards[boundaries.getShardForKey(key)]; + return shard.put(key, update, indexer, opGroup); + } + + /** + * Technically we should scatter gather on all the core threads because the size in following calls are not + * using volatile variables, but for metrics purpose this should be good enough. + */ + @Override + public long getLiveDataSize() + { + long total = 0L; + for (MemtableShard shard : shards) + total += shard.liveDataSize(); + return total; + } + + @Override + public long operationCount() + { + long total = 0L; + for (MemtableShard shard : shards) + total += shard.currentOperations(); + return total; + } + + @Override + public long partitionCount() + { + int total = 0; + for (MemtableShard shard : shards) + total += shard.size(); + return total; + } + + @Override + public long getMinTimestamp() + { + long min = Long.MAX_VALUE; + for (MemtableShard shard : shards) + min = Long.min(min, shard.minTimestamp()); + return min; + } + + @Override + public int getMinLocalDeletionTime() + { + int min = Integer.MAX_VALUE; + for (MemtableShard shard : shards) + min = Integer.min(min, shard.minLocalDeletionTime()); + return min; + } + + @Override + RegularAndStaticColumns columns() + { + for (MemtableShard shard : shards) + columnsCollector.update(shard.columnsCollector); + return columnsCollector.get(); + } + + @Override + EncodingStats encodingStats() + { + for (MemtableShard shard : shards) + statsCollector.update(shard.statsCollector.get()); + return statsCollector.get(); + } + + @Override + public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter, + final DataRange dataRange, + SSTableReadsListener readsListener) + { + AbstractBounds keyRange = dataRange.keyRange(); + + PartitionPosition left = keyRange.left; + PartitionPosition right = keyRange.right; + + boolean isBound = keyRange instanceof Bounds; + boolean includeStart = isBound || keyRange instanceof IncludingExcludingBounds; + boolean includeStop = isBound || keyRange instanceof Range; + + Iterator iterator = getPartitionIterator(left, includeStart, right, includeStop); + + return new MemtableUnfilteredPartitionIterator(metadata(), iterator, columnFilter, dataRange); + // readsListener is ignored as it only accepts sstable signals + } + + private Iterator getPartitionIterator(PartitionPosition left, boolean includeStart, PartitionPosition right, boolean includeStop) + { + int leftShard = left != null && !left.isMinimum() ? boundaries.getShardForKey(left) : 0; + int rightShard = right != null && !right.isMinimum() ? boundaries.getShardForKey(right) : boundaries.shardCount() - 1; + Iterator iterator; + if (leftShard == rightShard) + iterator = shards[leftShard].getPartitionsSubMap(left, includeStart, right, includeStop).values().iterator(); + else + { + Iterator[] iters = new Iterator[rightShard - leftShard + 1]; + int i = leftShard; + iters[0] = shards[leftShard].getPartitionsSubMap(left, includeStart, null, true).values().iterator(); + for (++i; i < rightShard; ++i) + iters[i - leftShard] = shards[i].partitions.values().iterator(); + iters[i - leftShard] = shards[i].getPartitionsSubMap(null, true, right, includeStop).values().iterator(); + iterator = Iterators.concat(iters); + } + return iterator; + } + + private Partition getPartition(DecoratedKey key) + { + int shardIndex = boundaries.getShardForKey(key); + return shards[shardIndex].partitions.get(key); + } + + @Override + public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) + { + Partition p = getPartition(key); + if (p == null) + return null; + else + return p.unfilteredIterator(selectedColumns, slices, reversed); + } + + @Override + public UnfilteredRowIterator rowIterator(DecoratedKey key) + { + Partition p = getPartition(key); + return p != null ? p.unfilteredIterator() : null; + } + + public FlushablePartitionSet getFlushSet(PartitionPosition from, PartitionPosition to) + { + long keySize = 0; + int keyCount = 0; + + for (Iterator it = getPartitionIterator(from, true, to,false); it.hasNext();) + { + AtomicBTreePartition en = it.next(); + keySize += en.partitionKey().getKey().remaining(); + keyCount++; + } + long partitionKeySize = keySize; + int partitionCount = keyCount; + Iterator toFlush = getPartitionIterator(from, true, to,false); + + return new AbstractFlushablePartitionSet() + { + public Memtable memtable() + { + return ShardedSkipListMemtable.this; + } + + public PartitionPosition from() + { + return from; + } + + public PartitionPosition to() + { + return to; + } + + public long partitionCount() + { + return partitionCount; + } + + public Iterator iterator() + { + return toFlush; + } + + public long partitionKeysSize() + { + return partitionKeySize; + } + }; + } + + static class MemtableShard + { + // The following fields are volatile as we have to make sure that when we + // collect results from all sub-ranges, the thread accessing the value + // is guaranteed to see the changes to the values. + + // The smallest timestamp for all partitions stored in this shard + private final AtomicLong minTimestamp = new AtomicLong(Long.MAX_VALUE); + private final AtomicInteger minLocalDeletionTime = new AtomicInteger(Integer.MAX_VALUE); + + private final AtomicLong liveDataSize = new AtomicLong(0); + + private final AtomicLong currentOperations = new AtomicLong(0); + + // We index the memtable by PartitionPosition only for the purpose of being able + // to select key range using Token.KeyBound. However put() ensures that we + // actually only store DecoratedKey. + private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); + + private final ColumnsCollector columnsCollector; + + private final StatsCollector statsCollector; + + @Unmetered // total pool size should not be included in memtable's deep size + private final MemtableAllocator allocator; + + private final TableMetadataRef metadata; + + @VisibleForTesting + MemtableShard(TableMetadataRef metadata, MemtableAllocator allocator) + { + this.columnsCollector = new ColumnsCollector(metadata.get().regularAndStaticColumns()); + this.statsCollector = new StatsCollector(); + this.allocator = allocator; + this.metadata = metadata; + } + + public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + { + AtomicBTreePartition previous = partitions.get(key); + + long initialSize = 0; + if (previous == null) + { + final DecoratedKey cloneKey = allocator.clone(key, opGroup); + AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator); + // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent + previous = partitions.putIfAbsent(cloneKey, empty); + if (previous == null) + { + previous = empty; + // allocate the row overhead after the fact; this saves over allocating and having to free after, but + // means we can overshoot our declared limit. + int overhead = (int) (cloneKey.getToken().getHeapSize() + SkipListMemtable.ROW_OVERHEAD_HEAP_SIZE); + allocator.onHeap().allocate(overhead, opGroup); + initialSize = 8; + } + } + + long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer); + updateMin(minTimestamp, update.stats().minTimestamp); + updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime); + liveDataSize.addAndGet(initialSize + pair[0]); + columnsCollector.update(update.columns()); + statsCollector.update(update.stats()); + currentOperations.addAndGet(update.operationCount()); + return pair[1]; + } + + private Map getPartitionsSubMap(PartitionPosition left, + boolean includeLeft, + PartitionPosition right, + boolean includeRight) + { + if (left != null && left.isMinimum()) + left = null; + if (right != null && right.isMinimum()) + right = null; + + try + { + if (left == null) + return right == null ? partitions : partitions.headMap(right, includeRight); + else + return right == null + ? partitions.tailMap(left, includeLeft) + : partitions.subMap(left, includeLeft, right, includeRight); + } + catch (IllegalArgumentException e) + { + logger.error("Invalid range requested {} - {}", left, right); + throw e; + } + } + + public boolean isEmpty() + { + return partitions.isEmpty(); + } + + public int size() + { + return partitions.size(); + } + + long minTimestamp() + { + return minTimestamp.get(); + } + + long liveDataSize() + { + return liveDataSize.get(); + } + + long currentOperations() + { + return currentOperations.get(); + } + + public int minLocalDeletionTime() + { + return minLocalDeletionTime.get(); + } + } + + public static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator + { + private final TableMetadata metadata; + private final Iterator iter; + private final ColumnFilter columnFilter; + private final DataRange dataRange; + + public MemtableUnfilteredPartitionIterator(TableMetadata metadata, Iterator iterator, ColumnFilter columnFilter, DataRange dataRange) + { + this.metadata = metadata; + this.iter = iterator; + this.columnFilter = columnFilter; + this.dataRange = dataRange; + } + + public TableMetadata metadata() + { + return metadata; + } + + public boolean hasNext() + { + return iter.hasNext(); + } + + public UnfilteredRowIterator next() + { + AtomicBTreePartition entry = iter.next(); + DecoratedKey key = entry.partitionKey(); + ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key); + + return filter.getUnfilteredRowIterator(columnFilter, entry); + } + } + + static class Locking extends ShardedSkipListMemtable + { + Locking(AtomicReference commitLogLowerBound, TableMetadataRef metadataRef, Owner owner, Integer shardCountOption, Factory factory) + { + super(commitLogLowerBound, metadataRef, owner, shardCountOption, factory); + } + + /** + * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate + * OpOrdering. + * + * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null + */ + public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + { + DecoratedKey key = update.partitionKey(); + MemtableShard shard = shards[boundaries.getShardForKey(key)]; + synchronized (shard) + { + return shard.put(key, update, indexer, opGroup); + } + } + + } + + public static Factory factory(Map optionsCopy) + { + String shardsString = optionsCopy.remove(SHARDS_OPTION); + Integer shardCount = shardsString != null ? Integer.parseInt(shardsString) : null; + boolean isLocking = Boolean.parseBoolean(optionsCopy.remove(LOCKING_OPTION)); + return new Factory(shardCount, isLocking); + } + + static class Factory implements Memtable.Factory + { + final Integer shardCount; + final boolean isLocking; + + Factory(Integer shardCount, boolean isLocking) + { + this.shardCount = shardCount; + this.isLocking = isLocking; + } + + public Memtable create(AtomicReference commitLogLowerBound, + TableMetadataRef metadataRef, + Owner owner) + { + return isLocking + ? new Locking(commitLogLowerBound, metadataRef, owner, shardCount, this) + : new ShardedSkipListMemtable(commitLogLowerBound, metadataRef, owner, shardCount, this); + } + + public boolean equals(Object o) + { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Factory factory = (Factory) o; + return Objects.equals(shardCount, factory.shardCount); + } + + public int hashCode() + { + return Objects.hash(shardCount); + } + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java new file mode 100644 index 0000000000..22b7747213 --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.memtable; + +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.AbstractBTreePartition; +import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.AtomicBTreePartition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.IncludingExcludingBounds; +import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.index.transactions.UpdateTransaction; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.memory.MemtableAllocator; + +import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_COMPUTE_STEPS; +import static org.apache.cassandra.config.CassandraRelevantProperties.MEMTABLE_OVERHEAD_SIZE; + +public class SkipListMemtable extends AbstractAllocatorMemtable +{ + private static final Logger logger = LoggerFactory.getLogger(SkipListMemtable.class); + + public static final Factory FACTORY = SkipListMemtableFactory.INSTANCE; + + protected static final int ROW_OVERHEAD_HEAP_SIZE; + static + { + int userDefinedOverhead = MEMTABLE_OVERHEAD_SIZE.getInt(-1); + if (userDefinedOverhead > 0) + ROW_OVERHEAD_HEAP_SIZE = userDefinedOverhead; + else + ROW_OVERHEAD_HEAP_SIZE = estimateRowOverhead(MEMTABLE_OVERHEAD_COMPUTE_STEPS.getInt()); + } + + // We index the memtable by PartitionPosition only for the purpose of being able + // to select key range using Token.KeyBound. However put() ensures that we + // actually only store DecoratedKey. + private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); + + private final AtomicLong liveDataSize = new AtomicLong(0); + + protected SkipListMemtable(AtomicReference commitLogLowerBound, TableMetadataRef metadataRef, Owner owner) + { + super(commitLogLowerBound, metadataRef, owner); + } + + @Override + protected Factory factory() + { + return FACTORY; + } + + @Override + public boolean isClean() + { + return partitions.isEmpty(); + } + + /** + * Should only be called by ColumnFamilyStore.apply via Keyspace.apply, which supplies the appropriate + * OpOrdering. + * + * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null + */ + @Override + public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + { + AtomicBTreePartition previous = partitions.get(update.partitionKey()); + + long initialSize = 0; + if (previous == null) + { + final DecoratedKey cloneKey = allocator.clone(update.partitionKey(), opGroup); + AtomicBTreePartition empty = new AtomicBTreePartition(metadata, cloneKey, allocator); + // We'll add the columns later. This avoids wasting works if we get beaten in the putIfAbsent + previous = partitions.putIfAbsent(cloneKey, empty); + if (previous == null) + { + previous = empty; + // allocate the row overhead after the fact; this saves over allocating and having to free after, but + // means we can overshoot our declared limit. + int overhead = (int) (cloneKey.getToken().getHeapSize() + ROW_OVERHEAD_HEAP_SIZE); + allocator.onHeap().allocate(overhead, opGroup); + initialSize = 8; + } + } + + long[] pair = previous.addAllWithSizeDelta(update, opGroup, indexer); + updateMin(minTimestamp, update.stats().minTimestamp); + updateMin(minLocalDeletionTime, update.stats().minLocalDeletionTime); + liveDataSize.addAndGet(initialSize + pair[0]); + columnsCollector.update(update.columns()); + statsCollector.update(update.stats()); + currentOperations.addAndGet(update.operationCount()); + return pair[1]; + } + + @Override + public long partitionCount() + { + return partitions.size(); + } + + @Override + public MemtableUnfilteredPartitionIterator partitionIterator(final ColumnFilter columnFilter, + final DataRange dataRange, + SSTableReadsListener readsListener) + { + AbstractBounds keyRange = dataRange.keyRange(); + + PartitionPosition left = keyRange.left; + PartitionPosition right = keyRange.right; + + boolean isBound = keyRange instanceof Bounds; + boolean includeLeft = isBound || keyRange instanceof IncludingExcludingBounds; + boolean includeRight = isBound || keyRange instanceof Range; + Map subMap = getPartitionsSubMap(left, + includeLeft, + right, + includeRight); + + return new MemtableUnfilteredPartitionIterator(metadata.get(), subMap, columnFilter, dataRange); + // readsListener is ignored as it only accepts sstable signals + } + + private Map getPartitionsSubMap(PartitionPosition left, + boolean includeLeft, + PartitionPosition right, + boolean includeRight) + { + if (left != null && left.isMinimum()) + left = null; + if (right != null && right.isMinimum()) + right = null; + + try + { + if (left == null) + return right == null ? partitions : partitions.headMap(right, includeRight); + else + return right == null + ? partitions.tailMap(left, includeLeft) + : partitions.subMap(left, includeLeft, right, includeRight); + } + catch (IllegalArgumentException e) + { + logger.error("Invalid range requested {} - {}", left, right); + throw e; + } + } + + Partition getPartition(DecoratedKey key) + { + return partitions.get(key); + } + + @Override + public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) + { + Partition p = getPartition(key); + if (p == null) + return null; + else + return p.unfilteredIterator(selectedColumns, slices, reversed); + } + + @Override + public UnfilteredRowIterator rowIterator(DecoratedKey key) + { + Partition p = getPartition(key); + return p != null ? p.unfilteredIterator() : null; + } + + private static int estimateRowOverhead(final int count) + { + // calculate row overhead + try (final OpOrder.Group group = new OpOrder().start()) + { + int rowOverhead; + MemtableAllocator allocator = MEMORY_POOL.newAllocator(""); + ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); + final Object val = new Object(); + for (int i = 0 ; i < count ; i++) + partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val); + double avgSize = ObjectSizes.measureDeep(partitions) / (double) count; + rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize)); + rowOverhead -= ObjectSizes.measureDeep(new LongToken(0)); + rowOverhead += AtomicBTreePartition.EMPTY_SIZE; + rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE; + allocator.setDiscarding(); + allocator.setDiscarded(); + return rowOverhead; + } + } + + @Override + public FlushablePartitionSet getFlushSet(PartitionPosition from, PartitionPosition to) + { + Map toFlush = getPartitionsSubMap(from, true, to, false); + long keysSize = 0; + long keyCount = 0; + + boolean trackContention = logger.isTraceEnabled(); + if (trackContention) + { + int heavilyContendedRowCount = 0; + + for (AtomicBTreePartition partition : toFlush.values()) + { + keysSize += partition.partitionKey().getKey().remaining(); + ++keyCount; + if (partition.useLock()) + heavilyContendedRowCount++; + } + + if (heavilyContendedRowCount > 0) + logger.trace("High update contention in {}/{} partitions of {} ", heavilyContendedRowCount, toFlush.size(), SkipListMemtable.this); + } + else + { + for (PartitionPosition key : toFlush.keySet()) + { + // make sure we don't write non-sensical keys + assert key instanceof DecoratedKey; + keysSize += ((DecoratedKey) key).getKey().remaining(); + ++keyCount; + } + } + final long partitionKeysSize = keysSize; + final long partitionCount = keyCount; + + return new AbstractFlushablePartitionSet() + { + @Override + public Memtable memtable() + { + return SkipListMemtable.this; + } + + @Override + public PartitionPosition from() + { + return from; + } + + @Override + public PartitionPosition to() + { + return to; + } + + @Override + public long partitionCount() + { + return partitionCount; + } + + @Override + public Iterator iterator() + { + return toFlush.values().iterator(); + } + + @Override + public long partitionKeysSize() + { + return partitionKeysSize; + } + }; + } + + + private static class MemtableUnfilteredPartitionIterator extends AbstractUnfilteredPartitionIterator implements UnfilteredPartitionIterator + { + private final TableMetadata metadata; + private final Iterator> iter; + private final ColumnFilter columnFilter; + private final DataRange dataRange; + + MemtableUnfilteredPartitionIterator(TableMetadata metadata, Map map, ColumnFilter columnFilter, DataRange dataRange) + { + this.metadata = metadata; + this.iter = map.entrySet().iterator(); + this.columnFilter = columnFilter; + this.dataRange = dataRange; + } + + @Override + public TableMetadata metadata() + { + return metadata; + } + + @Override + public boolean hasNext() + { + return iter.hasNext(); + } + + @Override + public UnfilteredRowIterator next() + { + Map.Entry entry = iter.next(); + // Actual stored key should be true DecoratedKey + assert entry.getKey() instanceof DecoratedKey; + DecoratedKey key = (DecoratedKey)entry.getKey(); + ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(key); + + return filter.getUnfilteredRowIterator(columnFilter, entry.getValue()); + } + } + + @Override + public long getLiveDataSize() + { + return liveDataSize.get(); + } + + /** + * For testing only. Give this memtable too big a size to make it always fail flushing. + */ + @VisibleForTesting + public void makeUnflushable() + { + liveDataSize.addAndGet(1024L * 1024 * 1024 * 1024 * 1024); + } +} diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtableFactory.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtableFactory.java new file mode 100644 index 0000000000..ce8c321f84 --- /dev/null +++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtableFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.config.InheritingClass; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.schema.TableMetadataRef; + +/** + * This class makes better sense as an inner class to SkipListMemtable (which could be as simple as + * FACTORY = SkipListMemtable::new), but having it there causes the SkipListMemtable class to be initialized the first + * time it is referenced (e.g. during default memtable factory construction). + * + * Some tests want to setup table parameters before initializing DatabaseDescriptor -- this allows them to do so, and + * also makes sure the memtable memory pools are not created for offline tools. + */ +public class SkipListMemtableFactory implements Memtable.Factory +{ + @Override + public Memtable create(AtomicReference commitLogLowerBound, TableMetadataRef metadaRef, Memtable.Owner owner) + { + return new SkipListMemtable(commitLogLowerBound, metadaRef, owner); + } + + public static final SkipListMemtableFactory INSTANCE = new SkipListMemtableFactory(); + public static InheritingClass CONFIGURATION = new InheritingClass(null, SkipListMemtable.class.getName(), ImmutableMap.of()); +} diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java index b51a6787a0..b85efce977 100644 --- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java @@ -66,6 +66,11 @@ public abstract class AbstractBTreePartition implements Partition, Iterable this.staticRow = staticRow == null ? Rows.EMPTY_STATIC_ROW : staticRow; this.stats = stats; } + + protected Holder withColumns(RegularAndStaticColumns columns) + { + return new Holder(columns, this.tree, this.deletionInfo, this.staticRow, this.stats); + } } public DeletionInfo deletionInfo() diff --git a/src/java/org/apache/cassandra/db/partitions/Partition.java b/src/java/org/apache/cassandra/db/partitions/Partition.java index a9a96539fc..8888104d95 100644 --- a/src/java/org/apache/cassandra/db/partitions/Partition.java +++ b/src/java/org/apache/cassandra/db/partitions/Partition.java @@ -49,6 +49,11 @@ public interface Partition */ public boolean isEmpty(); + /** + * Whether the partition object has rows. This may be false but partition still be non-empty if it has a deletion. + */ + boolean hasRows(); + /** * Returns the row corresponding to the provided clustering, or null if there is not such row. * diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java index f6fd259e88..d705880cb2 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -21,10 +21,11 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import org.slf4j.Logger; @@ -210,6 +211,20 @@ public class PartitionUpdate extends AbstractBTreePartition return new PartitionUpdate(iterator.metadata(), iterator.partitionKey(), holder, deletionInfo, false); } + + public PartitionUpdate withOnlyPresentColumns() + { + Set columnSet = new HashSet<>(); + + for (Row row : this) + for (ColumnData column : row) + columnSet.add(column.column()); + + RegularAndStaticColumns columns = RegularAndStaticColumns.builder().addAll(columnSet).build(); + return new PartitionUpdate(this.metadata, this.partitionKey, this.holder.withColumns(columns), this.deletionInfo.mutableCopy(), false); + } + + protected boolean canHaveShadowedData() { return canHaveShadowedData; diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java index 023d2344b8..98e683e264 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java @@ -29,6 +29,7 @@ import java.util.function.LongPredicate; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; import com.google.common.collect.Maps; import org.slf4j.Logger; @@ -56,7 +57,6 @@ import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.repair.NoSuchRepairSessionException; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Refs; @@ -196,12 +196,19 @@ public class CassandraValidationIterator extends ValidationPartitionIterator if (!isIncremental) { // flush first so everyone is validating data that is as similar as possible - StorageService.instance.forceKeyspaceFlush(cfs.keyspace.getName(), cfs.name); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VALIDATION); + // Note: we also flush for incremental repair during the anti-compaction process. } sstables = getSSTablesToValidate(cfs, ranges, parentId, isIncremental); } + // Persistent memtables will not flush or snapshot to sstables, make an sstable with their data. + cfs.writeAndAddMemtableRanges(parentId, + () -> Collections2.transform(Range.normalize(ranges), Range::makeRowRange), + sstables); + Preconditions.checkArgument(sstables != null); + ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance.getParentRepairSession(parentId); logger.info("{}, parentSessionId={}: Performing validation compaction on {} sstables in {}.{}", prs.previewKind.logPrefix(sessionID), diff --git a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java index b96532d9cb..af9888a3f1 100644 --- a/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java +++ b/src/java/org/apache/cassandra/db/repair/PendingAntiCompaction.java @@ -365,7 +365,7 @@ public class PendingAntiCompaction List> tasks = new ArrayList<>(tables.size()); for (ColumnFamilyStore cfs : tables) { - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.ANTICOMPACTION); FutureTask task = new FutureTask<>(getAcquisitionCallable(cfs, tokenRanges.ranges(), prsId, acquireRetrySeconds, acquireSleepMillis)); executor.submit(task); tasks.add(task); diff --git a/src/java/org/apache/cassandra/db/rows/Row.java b/src/java/org/apache/cassandra/db/rows/Row.java index 7575f06552..8cc2d22d3b 100644 --- a/src/java/org/apache/cassandra/db/rows/Row.java +++ b/src/java/org/apache/cassandra/db/rows/Row.java @@ -626,6 +626,14 @@ public interface Row extends Unfiltered, Iterable, IMeasurableMemory */ public SimpleBuilder delete(); + /** + * Deletes the whole row with a timestamp that is just before the new data's timestamp, to make sure no expired + * data remains on the row. + * + * @return this builder. + */ + public SimpleBuilder deletePrevious(); + /** * Removes the value for a given column (creating a tombstone). * diff --git a/src/java/org/apache/cassandra/db/rows/Rows.java b/src/java/org/apache/cassandra/db/rows/Rows.java index 82abb03d7e..71307d1e0b 100644 --- a/src/java/org/apache/cassandra/db/rows/Rows.java +++ b/src/java/org/apache/cassandra/db/rows/Rows.java @@ -273,7 +273,7 @@ public abstract class Rows * * @return the smallest timestamp delta between corresponding rows from existing and update. A * timestamp delta being computed as the difference between the cells and DeletionTimes from {@code existing} - * and those in {@code existing}. + * and those in {@code update}. */ public static long merge(Row existing, Row update, diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java index 2842e662c5..4d1d71bf12 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java @@ -97,7 +97,7 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt { @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator UnfilteredRowIterator iter = RTBoundValidator.validate( - sstable.iterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener), + sstable.rowIterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener), RTBoundValidator.Stage.SSTABLE, false ); diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java new file mode 100644 index 0000000000..b984522f62 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.rows; + +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; + +/** + * Common data access interface for sstables and memtables. + */ +public interface UnfilteredSource +{ + /** + * Returns a row iterator for the given partition, applying the specified row and column filters. + * + * @param key the partition key + * @param slices the row ranges to return + * @param columnFilter filter to apply to all returned partitions + * @param reversed true if the content should be returned in reverse order + * @param listener a listener used to handle internal read events + */ + UnfilteredRowIterator rowIterator(DecoratedKey key, + Slices slices, + ColumnFilter columnFilter, + boolean reversed, + SSTableReadsListener listener); + + default UnfilteredRowIterator rowIterator(DecoratedKey key) + { + return rowIterator(key, Slices.ALL, ColumnFilter.NONE, false, SSTableReadsListener.NOOP_LISTENER); + } + + /** + * Returns a partition iterator for the given data range. + * + * @param columnFilter filter to apply to all returned partitions + * @param dataRange the partition and clustering range queried + * @param listener a listener used to handle internal read events + */ + UnfilteredPartitionIterator partitionIterator(ColumnFilter columnFilter, + DataRange dataRange, + SSTableReadsListener listener); + + /** Minimum timestamp of all stored data */ + long getMinTimestamp(); + + /** Minimum local deletion time in the memtable */ + int getMinLocalDeletionTime(); +} diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java index 11c859dc76..46cf253d4d 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java @@ -81,6 +81,7 @@ public class CassandraStreamManager implements TableStreamManager return new CassandraStreamReceiver(cfs, session, totalStreams); } + @SuppressWarnings("resource") // references placed onto returned collection or closed on error @Override public Collection createOutgoingStreams(StreamSession session, RangesAtEndpoint replicas, TimeUUID pendingRepair, PreviewKind previewKind) { @@ -126,6 +127,9 @@ public class CassandraStreamManager implements TableStreamManager return sstables; }).refs); + // This call is normally preceded by a memtable flush in StreamSession.addTransferRanges. + // Persistent memtables will not flush, make an sstable with their data. + cfs.writeAndAddMemtableRanges(session.getPendingRepair(), () -> Range.normalize(keyRanges), refs); List> normalizedFullRanges = Range.normalize(replicas.onlyFull().ranges()); List> normalizedAllRanges = Range.normalize(replicas.ranges()); diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 3113778a1d..48de8b54fc 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -181,11 +181,13 @@ public class CassandraStreamReceiver implements StreamReceiver * For CDC-enabled tables, we want to ensure that the mutations are run through the CommitLog so they * can be archived by the CDC process on discard. */ - private boolean requiresWritePath(ColumnFamilyStore cfs) { - return hasCDC(cfs) || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); + private boolean requiresWritePath(ColumnFamilyStore cfs) + { + return hasCDC(cfs) || cfs.streamToMemtable() || (session.streamOperation().requiresViewBuild() && hasViews(cfs)); } - private void sendThroughWritePath(ColumnFamilyStore cfs, Collection readers) { + private void sendThroughWritePath(ColumnFamilyStore cfs, Collection readers) + { boolean hasCdc = hasCDC(cfs); ColumnFilter filter = ColumnFilter.all(cfs.metadata()); for (SSTableReader reader : readers) @@ -273,7 +275,7 @@ public class CassandraStreamReceiver implements StreamReceiver // the streamed sstables. if (requiresWritePath) { - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.STREAMS_RECEIVED); abort(); } } diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java index 3afd128299..4c175e2d42 100644 --- a/src/java/org/apache/cassandra/db/view/TableViews.java +++ b/src/java/org/apache/cassandra/db/view/TableViews.java @@ -104,10 +104,10 @@ public class TableViews extends AbstractCollection views.forEach(View::stopBuild); } - public void forceBlockingFlush() + public void forceBlockingFlush(ColumnFamilyStore.FlushReason reason) { for (ColumnFamilyStore viewCfs : allViewsCfs()) - viewCfs.forceBlockingFlush(); + viewCfs.forceBlockingFlush(reason); } public void dumpMemtables() diff --git a/src/java/org/apache/cassandra/db/view/ViewBuilder.java b/src/java/org/apache/cassandra/db/view/ViewBuilder.java index 8c840e9506..daedf48f29 100644 --- a/src/java/org/apache/cassandra/db/view/ViewBuilder.java +++ b/src/java/org/apache/cassandra/db/view/ViewBuilder.java @@ -95,7 +95,7 @@ class ViewBuilder logger.debug("Starting build of view({}.{}). Flushing base table {}.{}", ksName, view.name, ksName, baseCfs.name); - baseCfs.forceBlockingFlush(); + baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.VIEW_BUILD_STARTED); loadStatusAndBuild(); } diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java index 63f619848e..9f51b16806 100644 --- a/src/java/org/apache/cassandra/index/Index.java +++ b/src/java/org/apache/cassandra/index/Index.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.function.BiFunction; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.*; diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java index 623b264b0e..93ecd595b5 100644 --- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.FutureTask; import org.apache.cassandra.concurrent.ImmediateExecutor; -import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.*; @@ -51,6 +50,7 @@ import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.*; @@ -65,8 +65,6 @@ import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Indexes; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.pager.SinglePartitionPager; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.ProtocolVersion; @@ -261,7 +259,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum * Adds and builds a index * * @param indexDef the IndexMetadata describing the index - * @param isNewCF true if the index is added as part of a new table/columnfamily (i.e. loading a CF at startup), + * @param isNewCF true if the index is added as part of a new table/columnfamily (i.e. loading a CF at startup), * false for all other cases (i.e. newly added index) */ public synchronized Future addIndex(IndexMetadata indexDef, boolean isNewCF) @@ -378,7 +376,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum // Once we are tracking new writes, flush any memtable contents to not miss them from the sstable-based rebuild if (needsFlush) - baseCfs.forceBlockingFlush(); + baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED); // Now that we are tracking new writes and we haven't left untracked contents on the memtables, we are ready to // index the sstables @@ -618,7 +616,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum * * @param indexes the index to be marked as building * @param isFullRebuild {@code true} if this method is invoked as a full index rebuild, {@code false} otherwise - * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup), + * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup), * {@code false} for all other cases (i.e. newly added index) */ private synchronized void markIndexesBuilding(Set indexes, boolean isFullRebuild, boolean isNewCF) @@ -669,7 +667,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum if (writableIndexes.put(indexName, index) == null) logger.info("Index [{}] became writable after successful build.", indexName); } - + AtomicInteger counter = inProgressBuilds.get(indexName); if (counter != null) { @@ -835,7 +833,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum { indexes.forEach(index -> index.getBackingTable() - .map(cfs -> wait.add(cfs.forceFlush())) + .map(cfs -> wait.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_COMPLETED))) .orElseGet(() -> nonCfsIndexes.add(index))); } @@ -892,7 +890,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum /** * When building an index against existing data in sstables, add the given partition to the index - * + * * @param key the key for the partition being indexed * @param indexes the indexes that must be updated * @param pageSize the number of {@link Unfiltered} objects to process in a single page @@ -905,12 +903,12 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum if (!indexes.isEmpty()) { - SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(), - FBUtilities.nowInSeconds(), - ColumnFilter.selection(columns), - RowFilter.NONE, - DataLimits.NONE, - key, + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(), + FBUtilities.nowInSeconds(), + ColumnFilter.selection(columns), + RowFilter.NONE, + DataLimits.NONE, + key, new ClusteringIndexSliceFilter(Slices.ALL, false)); int nowInSec = cmd.nowInSec(); @@ -1201,7 +1199,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum { if (!hasIndexes()) return UpdateTransaction.NO_OP; - + ArrayList idxrs = new ArrayList<>(); for (Index i : writableIndexes.values()) { @@ -1209,7 +1207,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum if (idxr != null) idxrs.add(idxr); } - + if (idxrs.size() == 0) return UpdateTransaction.NO_OP; else diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java index 2561040577..0aac15d994 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java @@ -184,7 +184,7 @@ public abstract class CassandraIndex implements Index public Callable getBlockingFlushTask() { return () -> { - indexCfs.forceBlockingFlush(); + indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_TABLE_FLUSH); return null; }; } @@ -659,7 +659,7 @@ public abstract class CassandraIndex implements Index CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true); CompactionManager.instance.waitForCessation(cfss, (sstable) -> true); Keyspace.writeOrder.awaitNewBarrier(); - indexCfs.forceBlockingFlush(); + indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_REMOVED); indexCfs.readOrdering.awaitNewBarrier(); indexCfs.invalidate(); } @@ -685,7 +685,7 @@ public abstract class CassandraIndex implements Index @SuppressWarnings("resource") private void buildBlocking() { - baseCfs.forceBlockingFlush(); + baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED); try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)); Refs sstables = viewFragment.refs) @@ -709,7 +709,7 @@ public abstract class CassandraIndex implements Index ImmutableSet.copyOf(sstables)); Future future = CompactionManager.instance.submitIndexBuild(builder); FBUtilities.waitOnFuture(future); - indexCfs.forceBlockingFlush(); + indexCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_COMPLETED); } logger.info("Index build of {} complete", metadata.name); } diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java index b187c8c3f7..1e86bc6d8d 100644 --- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java @@ -291,7 +291,7 @@ public class SASIIndex implements Index, INotificationConsumer public void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup) { - baseCfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().allocate(additionalSpace, opGroup); + baseCfs.getTracker().getView().getCurrentMemtable().markExtraOnHeapUsed(additionalSpace, opGroup); } }; } diff --git a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java index 5ae1cc9c27..81b776de62 100644 --- a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java @@ -31,7 +31,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.UTF8Type; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index e6214891f1..f26cf65c93 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -34,6 +34,7 @@ import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.rows.UnfilteredSource; import org.apache.cassandra.concurrent.ExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -138,7 +139,7 @@ import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQu * * TODO: fill in details about Tracker and lifecycle interactions for tools, and for compaction strategies */ -public abstract class SSTableReader extends SSTable implements SelfRefCounted +public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted { private static final Logger logger = LoggerFactory.getLogger(SSTableReader.class); @@ -1419,13 +1420,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted> rangeIterator); - /** - * @param columns the columns to return. - * @param dataRange filter to use when reading the columns - * @param listener a listener used to handle internal read events - * @return A Scanner for seeking over the rows of the SSTable. - */ - public abstract ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener); - public FileDataInput getFileDataInput(long position) { return dfile.createReader(position); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 186ee5abc4..f82a7c2fc6 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -382,8 +382,21 @@ public abstract class SSTableWriter extends SSTable implements Transactional FileUtils.createHardLinkWithoutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY)); } + /** + * Parameters for calculating the expected size of an sstable. Exposed on memtable flush sets (i.e. collected + * subsets of a memtable that will be written to sstables). + */ + public interface SSTableSizeParameters + { + long partitionCount(); + long partitionKeysSize(); + long dataSize(); + } + public static abstract class Factory { + public abstract long estimateSize(SSTableSizeParameters parameters); + public abstract SSTableWriter open(Descriptor descriptor, long keyCount, long repairedAt, diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index cff1f1c2d5..c84782f5f6 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -78,6 +78,15 @@ public class BigFormat implements SSTableFormat static class WriterFactory extends SSTableWriter.Factory { + @Override + public long estimateSize(SSTableWriter.SSTableSizeParameters parameters) + { + return (long) ((parameters.partitionKeysSize() // index entries + + parameters.partitionKeysSize() // keys in data file + + parameters.dataSize()) // data + * 1.2); // bloom filter and row index overhead + } + @Override public SSTableWriter open(Descriptor descriptor, long keyCount, diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java index 69bc1a414e..e0edd7af06 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java @@ -57,18 +57,18 @@ public class BigTableReader extends SSTableReader super(builder); } - public UnfilteredRowIterator iterator(DecoratedKey key, - Slices slices, - ColumnFilter selectedColumns, - boolean reversed, - SSTableReadsListener listener) + public UnfilteredRowIterator rowIterator(DecoratedKey key, + Slices slices, + ColumnFilter selectedColumns, + boolean reversed, + SSTableReadsListener listener) { RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ, listener); - return iterator(null, key, rie, slices, selectedColumns, reversed); + return rowIterator(null, key, rie, slices, selectedColumns, reversed); } @SuppressWarnings("resource") - public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) + public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) { if (indexEntry == null) return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed); @@ -78,7 +78,7 @@ public class BigTableReader extends SSTableReader } @Override - public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener) + public ISSTableScanner partitionIterator(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener) { return BigTableScanner.getScanner(this, columns, dataRange, listener); } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java index 20105cd1e1..235b9b1a71 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java @@ -365,7 +365,7 @@ public class BigTableScanner implements ISSTableScanner } ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey()); - return sstable.iterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed()); + return sstable.rowIterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed()); } catch (CorruptSSTableException | IOException e) { @@ -441,5 +441,10 @@ public class BigTableScanner implements ISSTableScanner { return null; } + + public int getMinLocalDeletionTime() + { + return DeletionTime.LIVE.localDeletionTime(); + } } } diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 3e12c5e6f2..89ac036804 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -38,7 +38,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.index.SecondaryIndexManager; @@ -392,11 +392,16 @@ public class TableMetrics * * @param cfs ColumnFamilyStore to measure metrics */ - public TableMetrics(final ColumnFamilyStore cfs) + public TableMetrics(final ColumnFamilyStore cfs, ReleasableMetric memtableMetrics) { factory = new TableMetricNameFactory(cfs, "Table"); aliasFactory = new TableMetricNameFactory(cfs, "ColumnFamily"); + if (memtableMetrics != null) + { + all.add(memtableMetrics); + } + samplers = new EnumMap<>(SamplerType.class); topReadPartitionFrequency = new FrequencySampler() { @@ -441,16 +446,16 @@ public class TableMetrics samplers.put(SamplerType.LOCAL_READ_TIME, topLocalReadQueryTime); memtableColumnsCount = createTableGauge("MemtableColumnsCount", - () -> cfs.getTracker().getView().getCurrentMemtable().getOperations()); + () -> cfs.getTracker().getView().getCurrentMemtable().operationCount()); // MemtableOnHeapSize naming deprecated in 4.0 memtableOnHeapDataSize = createTableGaugeWithDeprecation("MemtableOnHeapDataSize", "MemtableOnHeapSize", - () -> cfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns(), + () -> Memtable.getMemoryUsage(cfs.getTracker().getView().getCurrentMemtable()).ownsOnHeap, new GlobalTableGauge("MemtableOnHeapDataSize")); // MemtableOffHeapSize naming deprecated in 4.0 memtableOffHeapDataSize = createTableGaugeWithDeprecation("MemtableOffHeapDataSize", "MemtableOffHeapSize", - () -> cfs.getTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns(), + () -> Memtable.getMemoryUsage(cfs.getTracker().getView().getCurrentMemtable()).ownsOffHeap, new GlobalTableGauge("MemtableOnHeapDataSize")); memtableLiveDataSize = createTableGauge("MemtableLiveDataSize", @@ -461,10 +466,7 @@ public class TableMetrics { public Long getValue() { - long size = 0; - for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes()) - size += cfs2.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().owns(); - return size; + return getMemoryUsageWithIndexes(cfs).ownsOnHeap; } }, new GlobalTableGauge("AllMemtablesOnHeapDataSize")); @@ -473,10 +475,7 @@ public class TableMetrics { public Long getValue() { - long size = 0; - for (ColumnFamilyStore cfs2 : cfs.concatWithIndexes()) - size += cfs2.getTracker().getView().getCurrentMemtable().getAllocator().offHeap().owns(); - return size; + return getMemoryUsageWithIndexes(cfs).ownsOffHeap; } }, new GlobalTableGauge("AllMemtablesOffHeapDataSize")); allMemtablesLiveDataSize = createTableGauge("AllMemtablesLiveDataSize", new Gauge() @@ -946,6 +945,15 @@ public class TableMetrics rowIndexSize = createTableHistogram("RowIndexSize", cfs.keyspace.metric.rowIndexSize, false); } + private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs) + { + Memtable.MemoryUsage usage = Memtable.newMemoryUsage(); + cfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage); + for (ColumnFamilyStore indexCfs : cfs.indexManager.getAllIndexColumnFamilyStores()) + indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage); + return usage; + } + public void updateSSTableIterated(int count) { sstablesPerReadHistogram.update(count); diff --git a/src/java/org/apache/cassandra/notifications/MemtableDiscardedNotification.java b/src/java/org/apache/cassandra/notifications/MemtableDiscardedNotification.java index 778cad06c0..849b2f698a 100644 --- a/src/java/org/apache/cassandra/notifications/MemtableDiscardedNotification.java +++ b/src/java/org/apache/cassandra/notifications/MemtableDiscardedNotification.java @@ -17,7 +17,7 @@ */ package org.apache.cassandra.notifications; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; public class MemtableDiscardedNotification implements INotification { diff --git a/src/java/org/apache/cassandra/notifications/MemtableRenewedNotification.java b/src/java/org/apache/cassandra/notifications/MemtableRenewedNotification.java index 4c7e6c5b05..776c9da161 100644 --- a/src/java/org/apache/cassandra/notifications/MemtableRenewedNotification.java +++ b/src/java/org/apache/cassandra/notifications/MemtableRenewedNotification.java @@ -17,7 +17,7 @@ */ package org.apache.cassandra.notifications; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; public class MemtableRenewedNotification implements INotification { diff --git a/src/java/org/apache/cassandra/notifications/MemtableSwitchedNotification.java b/src/java/org/apache/cassandra/notifications/MemtableSwitchedNotification.java index 946de4ee84..b1737bebfd 100644 --- a/src/java/org/apache/cassandra/notifications/MemtableSwitchedNotification.java +++ b/src/java/org/apache/cassandra/notifications/MemtableSwitchedNotification.java @@ -17,7 +17,7 @@ */ package org.apache.cassandra.notifications; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; public class MemtableSwitchedNotification implements INotification { diff --git a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java index 9c95a182de..857af69847 100644 --- a/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java +++ b/src/java/org/apache/cassandra/notifications/SSTableAddedNotification.java @@ -21,7 +21,7 @@ import java.util.Optional; import javax.annotation.Nullable; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.io.sstable.format.SSTableReader; /** diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java index c9c751ad9d..862b22466e 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java @@ -604,7 +604,7 @@ public class LocalSessions { TableId tid = Schema.instance.getTableMetadata(keyspace, table).id; ColumnFamilyStore cfm = Schema.instance.getColumnFamilyStoreInstance(tid); - cfm.forceBlockingFlush(); + cfm.forceBlockingFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED); } /** diff --git a/src/java/org/apache/cassandra/schema/MemtableParams.java b/src/java/org/apache/cassandra/schema/MemtableParams.java new file mode 100644 index 0000000000..a3f1bb2323 --- /dev/null +++ b/src/java/org/apache/cassandra/schema/MemtableParams.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.schema; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.InheritingClass; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.SkipListMemtableFactory; +import org.apache.cassandra.exceptions.ConfigurationException; + +/** + * Memtable types and options are specified with these parameters. Memtable classes must either contain a static + * {@code FACTORY} field (if they take no arguments other than class), or implement a + * {@code factory(Map)} method. + * + * The latter should consume any further options (using {@code map.remove}). + * + * See Memtable_API.md for further details on the configuration and usage of memtable implementations. + */ +public final class MemtableParams +{ + private final Memtable.Factory factory; + private final String configurationKey; + + private MemtableParams(Memtable.Factory factory, String configurationKey) + { + this.configurationKey = configurationKey; + this.factory = factory; + } + + public String configurationKey() + { + return configurationKey; + } + + public Memtable.Factory factory() + { + return factory; + } + + @Override + public String toString() + { + return configurationKey; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + + if (!(o instanceof MemtableParams)) + return false; + + MemtableParams c = (MemtableParams) o; + + return Objects.equal(configurationKey, c.configurationKey); + } + + @Override + public int hashCode() + { + return configurationKey.hashCode(); + } + + private static final String DEFAULT_CONFIGURATION_KEY = "default"; + private static final Memtable.Factory DEFAULT_MEMTABLE_FACTORY = SkipListMemtableFactory.INSTANCE; + private static final ParameterizedClass DEFAULT_CONFIGURATION = SkipListMemtableFactory.CONFIGURATION; + private static final Map + CONFIGURATION_DEFINITIONS = expandDefinitions(DatabaseDescriptor.getMemtableConfigurations()); + private static final Map CONFIGURATIONS = new HashMap<>(); + public static final MemtableParams DEFAULT = get(null); + + public static MemtableParams get(String key) + { + if (key == null) + key = DEFAULT_CONFIGURATION_KEY; + + synchronized (CONFIGURATIONS) + { + return CONFIGURATIONS.computeIfAbsent(key, MemtableParams::parseConfiguration); + } + } + + @VisibleForTesting + static Map expandDefinitions(Map memtableConfigurations) + { + if (memtableConfigurations == null) + return ImmutableMap.of(DEFAULT_CONFIGURATION_KEY, DEFAULT_CONFIGURATION); + + LinkedHashMap configs = new LinkedHashMap<>(memtableConfigurations.size() + 1); + + // If default is not overridden, add an entry first so that other configurations can inherit from it. + // If it is, process it in its point of definition, so that the default can inherit from another configuration. + if (!memtableConfigurations.containsKey(DEFAULT_CONFIGURATION_KEY)) + configs.put(DEFAULT_CONFIGURATION_KEY, DEFAULT_CONFIGURATION); + + for (Map.Entry en : memtableConfigurations.entrySet()) + configs.put(en.getKey(), en.getValue().resolve(configs)); + + return ImmutableMap.copyOf(configs); + } + + private static MemtableParams parseConfiguration(String configurationKey) + { + ParameterizedClass definition = CONFIGURATION_DEFINITIONS.get(configurationKey); + + if (definition == null) + throw new ConfigurationException("Memtable configuration \"" + configurationKey + "\" not found."); + return new MemtableParams(getMemtableFactory(definition), configurationKey); + } + + + private static Memtable.Factory getMemtableFactory(ParameterizedClass options) + { + // Special-case this so that we don't initialize memtable class for tests that need to delay that. + if (options == DEFAULT_CONFIGURATION) + return DEFAULT_MEMTABLE_FACTORY; + + String className = options.class_name; + if (className == null || className.isEmpty()) + throw new ConfigurationException("The 'class_name' option must be specified."); + + className = className.contains(".") ? className : "org.apache.cassandra.db.memtable." + className; + try + { + Memtable.Factory factory; + Class clazz = Class.forName(className); + final Map parametersCopy = options.parameters != null + ? new HashMap<>(options.parameters) + : new HashMap<>(); + try + { + Method factoryMethod = clazz.getDeclaredMethod("factory", Map.class); + factory = (Memtable.Factory) factoryMethod.invoke(null, parametersCopy); + } + catch (NoSuchMethodException e) + { + // continue with FACTORY field + Field factoryField = clazz.getDeclaredField("FACTORY"); + factory = (Memtable.Factory) factoryField.get(null); + } + if (!parametersCopy.isEmpty()) + throw new ConfigurationException("Memtable class " + className + " does not accept any futher parameters, but " + + parametersCopy + " were given."); + return factory; + } + catch (NoSuchFieldException | ClassNotFoundException | IllegalAccessException | InvocationTargetException | ClassCastException e) + { + if (e.getCause() instanceof ConfigurationException) + throw (ConfigurationException) e.getCause(); + throw new ConfigurationException("Could not create memtable factory for class " + options, e); + } + } +} diff --git a/src/java/org/apache/cassandra/schema/SchemaEvent.java b/src/java/org/apache/cassandra/schema/SchemaEvent.java index c4085007e4..5703fe29b5 100644 --- a/src/java/org/apache/cassandra/schema/SchemaEvent.java +++ b/src/java/org/apache/cassandra/schema/SchemaEvent.java @@ -219,6 +219,7 @@ public final class SchemaEvent extends DiagnosticEvent ret.put("caching", repr(params.caching)); ret.put("compaction", repr(params.compaction)); ret.put("compression", repr(params.compression)); + ret.put("memtable", repr(params.memtable)); if (params.speculativeRetry != null) ret.put("speculativeRetry", params.speculativeRetry.kind().name()); return ret; } @@ -247,6 +248,11 @@ public final class SchemaEvent extends DiagnosticEvent return ret; } + private String repr(MemtableParams params) + { + return params.configurationKey(); + } + private HashMap repr(IndexMetadata index) { HashMap ret = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index dd134f02ee..33d2b7d6c8 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -105,6 +105,7 @@ public final class SchemaKeyspace + "comment text," + "compaction frozen>," + "compression frozen>," + + "memtable text," + "crc_check_chance double," + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake + "default_time_to_live int," @@ -172,6 +173,7 @@ public final class SchemaKeyspace + "comment text," + "compaction frozen>," + "compression frozen>," + + "memtable text," + "crc_check_chance double," + "dclocal_read_repair_chance double," // no longer used, left for drivers' sake + "default_time_to_live int," @@ -327,7 +329,7 @@ public final class SchemaKeyspace private static void flush() { if (!DatabaseDescriptor.isUnsafeSystem()) - ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush())); + ALL.forEach(table -> FBUtilities.waitOnFuture(getSchemaCFS(table).forceFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED))); } /** @@ -400,7 +402,7 @@ public final class SchemaKeyspace DecoratedKey key = partition.partitionKey(); Mutation.PartitionUpdateCollector puCollector = mutationMap.computeIfAbsent(key, k -> new Mutation.PartitionUpdateCollector(SchemaConstants.SCHEMA_KEYSPACE_NAME, key)); - puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter())); + puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter()).withOnlyPresentColumns()); } } } @@ -510,6 +512,7 @@ public final class SchemaKeyspace { Row.SimpleBuilder rowBuilder = builder.update(Tables) .row(table.name) + .deletePrevious() .add("id", table.id.asUUID()) .add("flags", TableMetadata.Flag.toStringSet(table.flags)); @@ -555,6 +558,11 @@ public final class SchemaKeyspace // node sends table schema to a < 3.8 versioned node with an unknown column. if (DatabaseDescriptor.isCDCEnabled()) builder.add("cdc", params.cdc); + + // As above, only add the memtable column if the table uses a non-default memtable configuration to avoid RTE + // in mixed operation with pre-4.1 versioned node during upgrades. + if (params.memtable != MemtableParams.DEFAULT) + builder.add("memtable", params.memtable.configurationKey()); } private static void addAlterTableToSchemaMutation(TableMetadata oldTable, TableMetadata newTable, Mutation.SimpleBuilder builder) @@ -714,6 +722,7 @@ public final class SchemaKeyspace TableMetadata table = view.metadata; Row.SimpleBuilder rowBuilder = builder.update(Views) .row(view.name()) + .deletePrevious() .add("include_all_columns", view.includeAllColumns) .add("base_table_id", view.baseTableId.asUUID()) .add("base_table_name", view.baseTableName) @@ -951,6 +960,7 @@ public final class SchemaKeyspace .comment(row.getString("comment")) .compaction(CompactionParams.fromMap(row.getFrozenTextMap("compaction"))) .compression(CompressionParams.fromMap(row.getFrozenTextMap("compression"))) + .memtable(MemtableParams.get(row.has("memtable") ? row.getString("memtable") : null)) // memtable column was introduced in 4.1 .defaultTimeToLive(row.getInt("default_time_to_live")) .extensions(row.getFrozenMap("extensions", UTF8Type.instance, BytesType.instance)) .gcGraceSeconds(row.getInt("gc_grace_seconds")) diff --git a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java index 5f5ccb95be..dc40093d4d 100644 --- a/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SystemDistributedKeyspace.java @@ -43,6 +43,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.Range; @@ -368,7 +369,7 @@ public final class SystemDistributedKeyspace { String buildReq = "DELETE FROM %s.%s WHERE keyspace_name = ? AND view_name = ?"; QueryProcessor.executeInternal(format(buildReq, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, VIEW_BUILD_STATUS), keyspaceName, viewName); - forceBlockingFlush(VIEW_BUILD_STATUS); + forceBlockingFlush(VIEW_BUILD_STATUS, ColumnFamilyStore.FlushReason.INTERNALLY_FORCED); } private static void processSilent(String fmtQry, String... values) @@ -388,10 +389,12 @@ public final class SystemDistributedKeyspace } } - public static void forceBlockingFlush(String table) + public static void forceBlockingFlush(String table, ColumnFamilyStore.FlushReason reason) { if (!DatabaseDescriptor.isUnsafeSystem()) - FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME).getColumnFamilyStore(table).forceFlush()); + FBUtilities.waitOnFuture(Keyspace.open(SchemaConstants.DISTRIBUTED_KEYSPACE_NAME) + .getColumnFamilyStore(table) + .forceFlush(reason)); } private enum RepairState diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 9266b0d754..2e9d507ba9 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -867,6 +867,13 @@ public class TableMetadata implements SchemaElement return this; } + public Builder memtable(MemtableParams val) + { + params.memtable(val); + return this; + } + + public Builder isCounter(boolean val) { return flag(Flag.COUNTER, val); diff --git a/src/java/org/apache/cassandra/schema/TableParams.java b/src/java/org/apache/cassandra/schema/TableParams.java index 62db8f78bd..440729c4ef 100644 --- a/src/java/org/apache/cassandra/schema/TableParams.java +++ b/src/java/org/apache/cassandra/schema/TableParams.java @@ -46,6 +46,7 @@ public final class TableParams COMMENT, COMPACTION, COMPRESSION, + MEMTABLE, DEFAULT_TIME_TO_LIVE, EXTENSIONS, GC_GRACE_SECONDS, @@ -78,6 +79,7 @@ public final class TableParams public final CachingParams caching; public final CompactionParams compaction; public final CompressionParams compression; + public final MemtableParams memtable; public final ImmutableMap extensions; public final boolean cdc; public final ReadRepairStrategy readRepair; @@ -99,6 +101,7 @@ public final class TableParams caching = builder.caching; compaction = builder.compaction; compression = builder.compression; + memtable = builder.memtable; extensions = builder.extensions; cdc = builder.cdc; readRepair = builder.readRepair; @@ -116,6 +119,7 @@ public final class TableParams .comment(params.comment) .compaction(params.compaction) .compression(params.compression) + .memtable(params.memtable) .crcCheckChance(params.crcCheckChance) .defaultTimeToLive(params.defaultTimeToLive) .gcGraceSeconds(params.gcGraceSeconds) @@ -178,6 +182,9 @@ public final class TableParams if (memtableFlushPeriodInMs < 0) fail("%s must be greater than or equal to 0 (got %s)", Option.MEMTABLE_FLUSH_PERIOD_IN_MS, memtableFlushPeriodInMs); + + if (cdc && memtable.factory().writesShouldSkipCommitLog()) + fail("CDC cannot work if writes skip the commit log. Check your memtable configuration."); } private static void fail(String format, Object... args) @@ -208,6 +215,7 @@ public final class TableParams && caching.equals(p.caching) && compaction.equals(p.compaction) && compression.equals(p.compression) + && memtable.equals(p.memtable) && extensions.equals(p.extensions) && cdc == p.cdc && readRepair == p.readRepair; @@ -228,6 +236,7 @@ public final class TableParams caching, compaction, compression, + memtable, extensions, cdc, readRepair); @@ -249,6 +258,7 @@ public final class TableParams .add(Option.CACHING.toString(), caching) .add(Option.COMPACTION.toString(), compaction) .add(Option.COMPRESSION.toString(), compression) + .add(Option.MEMTABLE.toString(), memtable) .add(Option.EXTENSIONS.toString(), extensions) .add(Option.CDC.toString(), cdc) .add(Option.READ_REPAIR.toString(), readRepair) @@ -272,6 +282,8 @@ public final class TableParams .newLine() .append("AND compression = ").append(compression.asMap()) .newLine() + .append("AND memtable = ").appendWithSingleQuotes(memtable.configurationKey()) + .newLine() .append("AND crc_check_chance = ").append(crcCheckChance) .newLine(); @@ -315,6 +327,7 @@ public final class TableParams private CachingParams caching = CachingParams.DEFAULT; private CompactionParams compaction = CompactionParams.DEFAULT; private CompressionParams compression = CompressionParams.DEFAULT; + private MemtableParams memtable = MemtableParams.DEFAULT; private ImmutableMap extensions = ImmutableMap.of(); private boolean cdc; private ReadRepairStrategy readRepair = ReadRepairStrategy.BLOCKING; @@ -400,6 +413,12 @@ public final class TableParams return this; } + public Builder memtable(MemtableParams val) + { + memtable = val; + return this; + } + public Builder compression(CompressionParams val) { compression = val; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index d5cd898c35..97d5204722 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -355,6 +355,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE setGossipTokens(localTokens); tokenMetadata.updateNormalTokens(tokens, FBUtilities.getBroadcastAddressAndPort()); setMode(Mode.NORMAL, false); + invalidateLocalRanges(); } public void setGossipTokens(Collection tokens) @@ -1866,7 +1867,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } // Force disk boundary invalidation now that local tokens are set - invalidateDiskBoundaries(); + invalidateLocalRanges(); repairPaxosForTopologyChange("bootstrap"); Future bootstrapStream = startBootstrap(tokens); @@ -1900,7 +1901,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return bootstrapper.bootstrap(streamStateStore, useStrictConsistency && !replacing); // handles token update } - private void invalidateDiskBoundaries() + private void invalidateLocalRanges() { for (Keyspace keyspace : Keyspace.all()) { @@ -1908,7 +1909,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { for (final ColumnFamilyStore store : cfs.concatWithIndexes()) { - store.invalidateDiskBoundaries(); + store.invalidateLocalRanges(); } } } @@ -2834,6 +2835,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } if (!tokensToUpdateInSystemKeyspace.isEmpty()) SystemKeyspace.updateTokens(endpoint, tokensToUpdateInSystemKeyspace); + + // Tokens changed, the local range ownership probably changed too. + invalidateLocalRanges(); } @VisibleForTesting @@ -2954,6 +2958,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (isMoving || operationMode == Mode.MOVING) { tokenMetadata.removeFromMoving(endpoint); + // The above may change the local ownership. + invalidateLocalRanges(); notifyMoved(endpoint); } else if (!isMember) // prior to this, the node was not a member @@ -4219,7 +4225,21 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tableNames)) { logger.debug("Forcing flush on keyspace {}, CF {}", keyspaceName, cfStore.name); - cfStore.forceBlockingFlush(); + cfStore.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); + } + } + + /** + * Flush all memtables for a keyspace and column families. + * @param keyspaceName + * @throws IOException + */ + public void forceKeyspaceFlush(String keyspaceName, ColumnFamilyStore.FlushReason reason) throws IOException + { + for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName)) + { + logger.debug("Forcing flush on keyspace {}, CF {}", keyspaceName, cfStore.name); + cfStore.forceBlockingFlush(reason); } } @@ -5282,7 +5302,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (Keyspace keyspace : Keyspace.nonSystem()) { for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores()) - flushes.add(cfs.forceFlush()); + flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.DRAIN)); } // wait for the flushes. // TODO this is a godawful way to track progress, since they flush in parallel. a long one could @@ -5314,7 +5334,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE for (Keyspace keyspace : Keyspace.system()) { for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores()) - flushes.add(cfs.forceFlush()); + flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.DRAIN)); } FBUtilities.waitOnFutures(flushes); diff --git a/src/java/org/apache/cassandra/service/disk/usage/DiskUsageMonitor.java b/src/java/org/apache/cassandra/service/disk/usage/DiskUsageMonitor.java index d7ed650f91..730e53e0f1 100644 --- a/src/java/org/apache/cassandra/service/disk/usage/DiskUsageMonitor.java +++ b/src/java/org/apache/cassandra/service/disk/usage/DiskUsageMonitor.java @@ -41,9 +41,9 @@ import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.db.guardrails.GuardrailsConfig; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.io.util.FileUtils; /** diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java index 904d2f55cd..320a16db24 100644 --- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.Range; @@ -42,6 +43,7 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.IndexRegistry; import org.apache.cassandra.index.transactions.IndexTransaction; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Indexes; @@ -144,7 +146,7 @@ public class PaxosUncommittedIndex implements Index, PaxosUncommittedTracker.Upd for (int j=0, jsize=dataRanges.size(); j flushIterator(Memtable flushing) { - List iters = singletonList(flushing.makePartitionIterator(memtableColumnFilter, FULL_RANGE)); + List iters = singletonList(flushing.partitionIterator(memtableColumnFilter, FULL_RANGE, SSTableReadsListener.NOOP_LISTENER)); return getPaxosUpdates(iters, null, true); } diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java index 50b0363314..353ad4422c 100644 --- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTracker.java @@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.File; diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index 9c1aa76041..595890def8 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -993,7 +993,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber { List> flushes = new ArrayList<>(); for (ColumnFamilyStore cfs : stores) - flushes.add(cfs.forceFlush()); + flushes.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.STREAMING)); FBUtilities.waitOnFutures(flushes); } diff --git a/src/java/org/apache/cassandra/utils/memory/HeapPool.java b/src/java/org/apache/cassandra/utils/memory/HeapPool.java index ebe92ac3aa..e7a7cef2aa 100644 --- a/src/java/org/apache/cassandra/utils/memory/HeapPool.java +++ b/src/java/org/apache/cassandra/utils/memory/HeapPool.java @@ -20,7 +20,6 @@ package org.apache.cassandra.utils.memory; import java.nio.ByteBuffer; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -35,7 +34,7 @@ public class HeapPool extends MemtablePool super(maxOnHeapMemory, 0, cleanupThreshold, cleaner); } - public MemtableAllocator newAllocator(ColumnFamilyStore table) + public MemtableAllocator newAllocator(String table) { return new Allocator(this); } @@ -94,9 +93,9 @@ public class HeapPool extends MemtablePool super(maxOnHeapMemory, 0, cleanupThreshold, cleaner); } - public MemtableAllocator newAllocator(ColumnFamilyStore cfs) + public MemtableAllocator newAllocator(String table) { - return new Allocator(this, cfs == null ? "" : cfs.keyspace.getName() + '.' + cfs.name); + return new Allocator(this, table); } private static class Allocator extends MemtableBufferAllocator diff --git a/src/java/org/apache/cassandra/utils/memory/MemtablePool.java b/src/java/org/apache/cassandra/utils/memory/MemtablePool.java index b43973d5fd..8a62f13f57 100644 --- a/src/java/org/apache/cassandra/utils/memory/MemtablePool.java +++ b/src/java/org/apache/cassandra/utils/memory/MemtablePool.java @@ -27,7 +27,6 @@ import com.google.common.base.Preconditions; import com.codahale.metrics.Gauge; import com.codahale.metrics.Timer; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.utils.concurrent.WaitQueue; @@ -82,7 +81,7 @@ public abstract class MemtablePool ExecutorUtils.shutdownNowAndWait(timeout, unit, cleaner); } - public abstract MemtableAllocator newAllocator(ColumnFamilyStore table); + public abstract MemtableAllocator newAllocator(String table); public boolean needsCleaning() { diff --git a/src/java/org/apache/cassandra/utils/memory/NativePool.java b/src/java/org/apache/cassandra/utils/memory/NativePool.java index 03c42e1e85..1391a376da 100644 --- a/src/java/org/apache/cassandra/utils/memory/NativePool.java +++ b/src/java/org/apache/cassandra/utils/memory/NativePool.java @@ -18,8 +18,6 @@ */ package org.apache.cassandra.utils.memory; -import org.apache.cassandra.db.ColumnFamilyStore; - public class NativePool extends MemtablePool { public NativePool(long maxOnHeapMemory, long maxOffHeapMemory, float cleanThreshold, MemtableCleaner cleaner) @@ -28,7 +26,7 @@ public class NativePool extends MemtablePool } @Override - public NativeAllocator newAllocator(ColumnFamilyStore table) + public NativeAllocator newAllocator(String table) { return new NativeAllocator(this); } diff --git a/src/java/org/apache/cassandra/utils/memory/SlabPool.java b/src/java/org/apache/cassandra/utils/memory/SlabPool.java index adeb14f517..edaddc9296 100644 --- a/src/java/org/apache/cassandra/utils/memory/SlabPool.java +++ b/src/java/org/apache/cassandra/utils/memory/SlabPool.java @@ -18,8 +18,6 @@ */ package org.apache.cassandra.utils.memory; -import org.apache.cassandra.db.ColumnFamilyStore; - public class SlabPool extends MemtablePool { final boolean allocateOnHeap; @@ -30,7 +28,7 @@ public class SlabPool extends MemtablePool this.allocateOnHeap = maxOffHeapMemory == 0; } - public MemtableAllocator newAllocator(ColumnFamilyStore table) + public MemtableAllocator newAllocator(String table) { return new SlabAllocator(onHeap.newAllocator(), offHeap.newAllocator(), allocateOnHeap); } diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 764379c7f9..80de239390 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -61,3 +61,45 @@ local_read_size_warn_threshold: 4096KiB local_read_size_fail_threshold: 8192KiB row_index_read_size_warn_threshold: 4096KiB row_index_read_size_fail_threshold: 8192KiB + +memtable: + configurations: + skiplist: + inherits: default + class_name: SkipListMemtable + skiplist_sharded: + class_name: ShardedSkipListMemtable + parameters: + serialize_writes: false + shards: 4 + skiplist_sharded_locking: + inherits: skiplist_sharded + parameters: + serialize_writes: true + skiplist_remapped: + inherits: skiplist + test_fullname: + inherits: default + class_name: org.apache.cassandra.db.memtable.TestMemtable + test_shortname: + class_name: TestMemtable + parameters: + skiplist: true # note: YAML must interpret this as string, not a boolean + test_empty_class: + class_name: "" + test_missing_class: + parameters: + test_unknown_class: + class_name: NotExisting + test_invalid_param: + class_name: SkipListMemtable + parameters: + invalid: throw + test_invalid_extra_param: + inherits: test_shortname + parameters: + invalid: throw + test_invalid_factory_method: + class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryMethod + test_invalid_factory_field: + class_name: org.apache.cassandra.cql3.validation.operations.CreateTest$InvalidMemtableFactoryField diff --git a/test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java b/test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java index f7327da10a..13e4f8ce64 100644 --- a/test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/fuzz/FuzzTestBase.java @@ -27,6 +27,7 @@ import harry.core.Configuration; import harry.core.Run; import harry.ddl.SchemaSpec; import harry.model.clock.OffsetClock; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.distributed.Cluster; @@ -124,7 +125,7 @@ public abstract class FuzzTestBase extends TestBaseImpl if (schema.staticColumns != null) writeStatic(current, 0, current, current, true).applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); return null; } } diff --git a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java index 6f2c92035e..6140ce6b07 100644 --- a/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java +++ b/test/distributed/org/apache/cassandra/distributed/fuzz/SSTableGenerator.java @@ -34,6 +34,7 @@ import harry.operations.Relation; import harry.operations.Query; import harry.operations.QueryGenerator; import harry.util.BitSet; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.AbstractMarker; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; @@ -113,7 +114,7 @@ public class SSTableGenerator public Collection flush() { - store.forceBlockingFlush(); + Util.flush(store); sstables.removeAll(store.getLiveSSTables()); Set ret = new HashSet<>(sstables); diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index aa178dad69..32d3bccf7d 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -46,6 +46,7 @@ import javax.management.NotificationListener; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.GlobalEventExecutor; +import org.apache.cassandra.Util; import org.apache.cassandra.auth.AuthCache; import org.apache.cassandra.batchlog.Batch; import org.apache.cassandra.batchlog.BatchlogManager; @@ -63,12 +64,12 @@ import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.SystemKeyspaceMigrator41; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.CompactionLogger; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.memtable.AbstractAllocatorMemtable; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.Constants; import org.apache.cassandra.distributed.action.GossipHelper; @@ -141,8 +142,8 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Ref; -import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor; import org.apache.cassandra.utils.memory.BufferPools; +import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor; import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -513,7 +514,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance public void flush(String keyspace) { - FBUtilities.waitOnFutures(Keyspace.open(keyspace).flush()); + Util.flushKeyspace(keyspace); } public void forceCompact(String keyspace, String table) @@ -768,7 +769,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance () -> PaxosRepair.shutdownAndWait(1L, MINUTES), () -> Ref.shutdownReferenceReaper(1L, MINUTES), () -> UncommittedTableData.shutdownAndWait(1L, MINUTES), - () -> Memtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES), + () -> AbstractAllocatorMemtable.MEMORY_POOL.shutdownAndWait(1L, MINUTES), () -> DiagnosticSnapshotService.instance.shutdownAndWait(1L, MINUTES), () -> SSTableReader.shutdownBlocking(1L, MINUTES), () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())), diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java index b95f854281..eb1cdffd0e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java @@ -45,6 +45,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.Keyspace; @@ -112,7 +113,7 @@ public class FailingRepairTest extends TestBaseImpl implements Serializable return () -> { String cfName = getCfName(type, parallelism, withTracing); ColumnFamilyStore cf = Keyspace.open(KEYSPACE).getColumnFamilyStore(cfName); - cf.forceBlockingFlush(); + Util.flush(cf); Set remove = cf.getLiveSSTables(); Set replace = new HashSet<>(); if (type == Verb.VALIDATION_REQ) diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java index 531493972e..10e62944ac 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java @@ -25,6 +25,7 @@ import java.util.Set; import org.junit.Assert; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.config.Config.DiskFailurePolicy; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -159,7 +160,7 @@ public class JVMStabilityInspectorCorruptSSTableExceptionTest extends TestBaseIm { node.runOnInstance(() -> { ColumnFamilyStore cf = Keyspace.open(keyspace).getColumnFamilyStore(table); - cf.forceBlockingFlush(); + Util.flush(cf); Set remove = cf.getLiveSSTables(); Set replace = new HashSet<>(); @@ -187,13 +188,13 @@ public class JVMStabilityInspectorCorruptSSTableExceptionTest extends TestBaseIm } @Override - public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) + public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) { throw throwCorrupted(); } @Override - public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) + public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) { throw throwCorrupted(); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java index 207cb3797b..d97337b94a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest.java @@ -317,7 +317,7 @@ public class PaxosRepairTest extends TestBaseImpl // exception expected } Assert.assertTrue(hasUncommitted(cluster, KEYSPACE, TABLE)); - cluster.forEach(i -> i.runOnInstance(() -> Keyspace.open("system").getColumnFamilyStore("paxos").forceBlockingFlush())); + cluster.forEach(i -> i.runOnInstance(() -> Keyspace.open("system").getColumnFamilyStore("paxos").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS))); CountDownLatch haveFetchedLowBound = new CountDownLatch(1); CountDownLatch haveReproposed = new CountDownLatch(1); @@ -580,7 +580,7 @@ public class PaxosRepairTest extends TestBaseImpl private static void compactPaxos() { ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS); - FBUtilities.waitOnFuture(paxos.forceFlush()); + FBUtilities.waitOnFuture(paxos.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)); FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false)); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java index e4ea6f2f7c..084562ff74 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PaxosRepairTest2.java @@ -48,11 +48,11 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.ReadExecutionController; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.ReadQuery; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.BTreeRow; @@ -298,7 +298,7 @@ public class PaxosRepairTest2 extends TestBaseImpl private static void compactPaxos() { ColumnFamilyStore paxos = Keyspace.open(SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PAXOS); - FBUtilities.waitOnFuture(paxos.forceFlush()); + FBUtilities.waitOnFuture(paxos.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)); FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(paxos, 0, false)); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java index cb831e6041..23ce43c12f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/RepairTest.java @@ -27,6 +27,7 @@ import java.util.function.Consumer; import com.google.common.collect.ImmutableMap; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.utils.concurrent.Condition; import org.junit.AfterClass; @@ -81,7 +82,8 @@ public class RepairTest extends TestBaseImpl private static void flush(ICluster cluster, String keyspace, int ... nodes) { for (int node : nodes) - cluster.get(node).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush(keyspace))); + cluster.get(node).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush(keyspace, + ColumnFamilyStore.FlushReason.UNIT_TESTS))); } private static ICluster create(Consumer configModifier) throws IOException diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java index 352da0225d..a727b51a29 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableLoaderEncryptionOptionsTest.java @@ -29,6 +29,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; @@ -133,7 +134,8 @@ public class SSTableLoaderEncryptionOptionsTest extends AbstractEncryptionOption CLUSTER.get(1).executeInternal(String.format("INSERT INTO ssl_upload_tables.test (pk, val) VALUES (%s, '%s')", i, Integer.toString(i))); } - CLUSTER.get(1).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush("ssl_upload_tables"))); + CLUSTER.get(1).runOnInstance(rethrow(() -> StorageService.instance.forceKeyspaceFlush("ssl_upload_tables", + ColumnFamilyStore.FlushReason.UNIT_TESTS))); } private static void truncateGeneratedTables() throws IOException diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java index 2f36f56786..9012dcca27 100644 --- a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java +++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java @@ -35,6 +35,7 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.AbstractBounds; @@ -283,15 +284,15 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) + public UnfilteredRowIterator rowIterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, SSTableReadsListener listener) { - return delegate.iterator(key, slices, selectedColumns, reversed, listener); + return delegate.rowIterator(key, slices, selectedColumns, reversed, listener); } @Override - public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) + public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) { - return delegate.iterator(file, key, indexEntry, slices, selectedColumns, reversed); + return delegate.rowIterator(file, key, indexEntry, slices, selectedColumns, reversed); } @Override @@ -385,9 +386,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public ISSTableScanner getScanner(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener) + public UnfilteredPartitionIterator partitionIterator(ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener) { - return delegate.getScanner(columns, dataRange, listener); + return delegate.partitionIterator(columns, dataRange, listener); } @Override diff --git a/test/long/org/apache/cassandra/cql3/ViewLongTest.java b/test/long/org/apache/cassandra/cql3/ViewLongTest.java index cec4d21899..2a9415fab6 100644 --- a/test/long/org/apache/cassandra/cql3/ViewLongTest.java +++ b/test/long/org/apache/cassandra/cql3/ViewLongTest.java @@ -29,10 +29,10 @@ import org.junit.Test; import com.datastax.driver.core.Row; import com.datastax.driver.core.exceptions.NoHostAvailableException; import com.datastax.driver.core.exceptions.WriteTimeoutException; +import org.apache.cassandra.Util; import org.apache.cassandra.batchlog.BatchlogManager; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.WrappedRunnable; public class ViewLongTest extends ViewAbstractParameterizedTest @@ -332,7 +332,7 @@ public class ViewLongTest extends ViewAbstractParameterizedTest updateViewWithFlush("UPDATE %s USING TTL 90 SET b = null WHERE k = 1 AND c = 2", flush); if (flush) - FBUtilities.waitOnFutures(Keyspace.open(keyspace()).flush()); + Util.flushKeyspace(keyspace()); assertRows(execute("select k,c,a,b from %s")); assertRows(executeView("select k,c,a from %s")); diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java index 58ad585fcb..a75ca5250b 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java @@ -168,7 +168,7 @@ public class LongCompactionsTest inserted.add(key); } - cfs.forceBlockingFlush(); + Util.flush(cfs); CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size()); diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java index c8a86e1099..1df7e7c95a 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyCQLTest.java @@ -32,6 +32,7 @@ import org.junit.Test; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Hex; @@ -72,7 +73,7 @@ public class LongLeveledCompactionStrategyCQLTest extends CQLTester throw new RuntimeException(throwable); } } - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); Uninterruptibles.sleepUninterruptibly(r.nextInt(200), TimeUnit.MILLISECONDS); } }); diff --git a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java index 4d692c6da4..733e46fd8d 100644 --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java @@ -162,7 +162,7 @@ public class LongLeveledCompactionStrategyTest } //Flush sstable - store.forceBlockingFlush(); + Util.flush(store); store.runWithCompactionsDisabled(new Callable() { @@ -261,7 +261,7 @@ public class LongLeveledCompactionStrategyTest Mutation rm = new Mutation(builder.build()); rm.apply(); - store.forceBlockingFlush(); + Util.flush(store); } } } diff --git a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java index 976872fc0a..4d652b70fa 100644 --- a/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java +++ b/test/memory/org/apache/cassandra/db/compaction/CompactionAllocationTest.java @@ -36,6 +36,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.Uninterruptibles; + +import org.apache.cassandra.Util; import org.apache.cassandra.index.internal.CollatedViewIndexBuilder; import org.junit.AfterClass; import org.junit.Assert; @@ -524,7 +526,7 @@ public class CompactionAllocationTest reads.add(() -> runQuery(query, cfs.metadata.get())); } } - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size()); @@ -639,7 +641,7 @@ public class CompactionAllocationTest reads.add(() -> runQuery(query, cfs.metadata.get())); } } - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size()); @@ -739,7 +741,7 @@ public class CompactionAllocationTest reads.add(() -> runQuery(query, cfs.metadata.get())); } } - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(numSSTable, cfs.getLiveSSTables().size()); @@ -833,7 +835,7 @@ public class CompactionAllocationTest makeRandomString(rowWidth>>2), makeRandomString(rowWidth>>2)); } } - cfs.forceBlockingFlush(); + Util.flush(cfs); } for (IndexDef index : indexes) diff --git a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java index d91db6e37f..de788b7aad 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java @@ -97,7 +97,7 @@ public class CacheLoaderBench extends CQLTester RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() + random.nextInt(), "key"); rowBuilder.add(colDef, "val1"); rowBuilder.build().apply(); - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); } Assert.assertEquals(numSSTables, cfs.getLiveSSTables().size()); diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java index a745054b42..8d7e800755 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java @@ -70,13 +70,13 @@ public class CompactionBench extends CQLTester execute(writeStatement, i, i, i ); - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); System.err.println("Writing 50k again..."); for (long i = 0; i < 50000; i++) execute(writeStatement, i, i, i ); - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); cfs.snapshot("originals"); diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java index fd201b0676..b5bb40c3c1 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java +++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java @@ -203,7 +203,7 @@ public class ZeroCopyStreamingBenchmark .build() .applyUnsafe(); } - store.forceBlockingFlush(); + store.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); CompactionManager.instance.performMaximal(store, false); } diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java index 789ca008a6..54f721bbb7 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java +++ b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTest.java @@ -19,108 +19,59 @@ package org.apache.cassandra.test.microbench.instance; -import java.util.Random; -import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; import java.util.function.Supplier; -import java.util.stream.IntStream; import com.google.common.base.Throwables; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Memtable; -import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.utils.FBUtilities; import org.openjdk.jmh.annotations.*; -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@Warmup(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 15, time = 2, timeUnit = TimeUnit.SECONDS) -@Fork(value = 1) -@Threads(1) @State(Scope.Benchmark) -public abstract class ReadTest extends CQLTester +public abstract class ReadTest extends SimpleTableWriter { - static String keyspace; - String table; - ColumnFamilyStore cfs; - Random rand; - - @Param({"1000"}) - int BATCH = 1_000; - public enum Flush { INMEM, NO, YES } - @Param({"1000000"}) - int count = 1_000_000; - @Param({"INMEM", "YES"}) Flush flush = Flush.INMEM; - public enum Execution - { - SERIAL, - SERIAL_NET, - PARALLEL, - PARALLEL_NET, - } - - @Param({"PARALLEL"}) - Execution async = Execution.PARALLEL; - @Setup(Level.Trial) public void setup() throws Throwable { - rand = new Random(1); - CQLTester.setUpClass(); - CQLTester.prepareServer(); - System.err.println("setupClass done."); - keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); - table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid)) with compression = {'enabled': false}"); - execute("use "+keyspace+";"); - switch (async) - { - case SERIAL_NET: - case PARALLEL_NET: - CQLTester.requireNetwork(); - executeNet(getDefaultVersion(), "use " + keyspace + ";"); - } - String writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)"; - System.err.println("Prepared, batch " + BATCH + " flush " + flush); - System.err.println("Disk access mode " + DatabaseDescriptor.getDiskAccessMode() + " index " + DatabaseDescriptor.getIndexAccessMode()); + super.commonSetup(); - cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); - cfs.disableAutoCompaction(); - cfs.forceBlockingFlush(); - - //Warm up + // Write the data we are going to read. + long writeStart = System.currentTimeMillis(); System.err.println("Writing " + count); long i; for (i = 0; i <= count - BATCH; i += BATCH) - performWrite(writeStatement, i, BATCH); + performWrite(i, BATCH); if (i < count) - performWrite(writeStatement, i, count - i); + performWrite(i, (int) (count - i)); + long writeLength = System.currentTimeMillis() - writeStart; + System.err.format("... done in %.3f s.\n", writeLength / 1000.0); Memtable memtable = cfs.getTracker().getView().getCurrentMemtable(); - System.err.format("Memtable in %s mode: %d ops, %s serialized bytes, %s (%.0f%%) on heap, %s (%.0f%%) off-heap\n", + Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable); + System.err.format("%s in %s mode: %d ops, %s serialized bytes, %s\n", + memtable.getClass().getSimpleName(), DatabaseDescriptor.getMemtableAllocationType(), - memtable.getOperations(), + memtable.operationCount(), FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()), - FBUtilities.prettyPrintMemory(memtable.getAllocator().onHeap().owns()), - 100 * memtable.getAllocator().onHeap().ownershipRatio(), - FBUtilities.prettyPrintMemory(memtable.getAllocator().offHeap().owns()), - 100 * memtable.getAllocator().offHeap().ownershipRatio()); + usage); switch (flush) { case YES: - cfs.forceBlockingFlush(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); break; case INMEM: if (!cfs.getLiveSSTables().isEmpty()) @@ -137,29 +88,6 @@ public abstract class ReadTest extends CQLTester } } - abstract Object[] writeArguments(long i); - - public void performWrite(String writeStatement, long ofs, long count) throws Throwable - { - for (long i = ofs; i < ofs + count; ++i) - execute(writeStatement, writeArguments(i)); - } - - - @TearDown(Level.Trial) - public void teardown() throws InterruptedException - { - if (flush == Flush.INMEM && !cfs.getLiveSSTables().isEmpty()) - throw new AssertionError("SSTables created for INMEM test."); - - // do a flush to print sizes - cfs.forceBlockingFlush(); - - CommitLog.instance.shutdownBlocking(); - CQLTester.tearDownClass(); - CQLTester.cleanup(); - } - public Object performReadSerial(String readStatement, Supplier supplier) throws Throwable { long sum = 0; @@ -170,20 +98,25 @@ public abstract class ReadTest extends CQLTester public Object performReadThreads(String readStatement, Supplier supplier) throws Throwable { - return IntStream.range(0, BATCH) - .parallel() - .mapToLong(i -> - { - try - { - return execute(readStatement, supplier.get()).size(); - } - catch (Throwable throwable) - { - throw Throwables.propagate(throwable); - } - }) - .sum(); + List> futures = new ArrayList<>(); + for (long i = 0; i < BATCH; ++i) + { + futures.add(executorService.submit(() -> + { + try + { + return execute(readStatement, supplier.get()).size(); + } + catch (Throwable throwable) + { + throw Throwables.propagate(throwable); + } + })); + } + long done = 0; + for (Future f : futures) + done += f.get(); + return done; } public Object performReadSerialNet(String readStatement, Supplier supplier) throws Throwable @@ -197,37 +130,55 @@ public abstract class ReadTest extends CQLTester public long performReadThreadsNet(String readStatement, Supplier supplier) throws Throwable { - return IntStream.range(0, BATCH) - .parallel() - .mapToLong(i -> - { - try - { - return executeNet(getDefaultVersion(), readStatement, supplier.get()) - .getAvailableWithoutFetching(); - } - catch (Throwable throwable) - { - throw Throwables.propagate(throwable); - } - }) - .sum(); + List> futures = new ArrayList<>(); + for (long i = 0; i < BATCH; ++i) + { + futures.add(executorService.submit(() -> + { + try + { + return executeNet(getDefaultVersion(), readStatement, supplier.get()) + .getAvailableWithoutFetching(); + } + catch (Throwable throwable) + { + throw Throwables.propagate(throwable); + } + })); + } + long done = 0; + for (Future f : futures) + done += f.get(); + return done; } public Object performRead(String readStatement, Supplier supplier) throws Throwable { - switch (async) + if (useNet) { - case SERIAL: - return performReadSerial(readStatement, supplier); - case SERIAL_NET: + if (threadCount == 1) return performReadSerialNet(readStatement, supplier); - case PARALLEL: - return performReadThreads(readStatement, supplier); - case PARALLEL_NET: + else return performReadThreadsNet(readStatement, supplier); } - return null; + else + { + if (threadCount == 1) + return performReadSerial(readStatement, supplier); + else + return performReadThreads(readStatement, supplier); + } + } + + void doExtraChecks() + { + if (flush == Flush.INMEM && !cfs.getLiveSSTables().isEmpty()) + throw new AssertionError("SSTables created for INMEM test."); + } + + String extraInfo() + { + return " flush " + flush; } } diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTestSmallPartitions.java b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTestSmallPartitions.java index b36cfd1238..c665c5947e 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTestSmallPartitions.java +++ b/test/microbench/org/apache/cassandra/test/microbench/instance/ReadTestSmallPartitions.java @@ -19,8 +19,26 @@ package org.apache.cassandra.test.microbench.instance; -import org.openjdk.jmh.annotations.Benchmark; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(value = 1) +@Threads(1) +@State(Scope.Benchmark) public class ReadTestSmallPartitions extends ReadTest { String readStatement() diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java new file mode 100644 index 0000000000..fba8d16b6b --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/instance/SimpleTableWriter.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.instance; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Throwables; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.utils.FBUtilities; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; + +@State(Scope.Benchmark) +public abstract class SimpleTableWriter extends CQLTester +{ + static String keyspace; + String table; + ColumnFamilyStore cfs; + Random rand; + String writeStatement; + ExecutorService executorService; + + @Param({"1000000"}) + int count = 1_000_000; + + @Param({ "1000" }) + int BATCH = 1_000; + + @Param({ "default" }) + String memtableClass = "default"; + + @Param({ "false" }) + boolean useNet = false; + + @Param({ "32" }) + int threadCount; + + public void commonSetup() throws Throwable + { + rand = new Random(1); + executorService = Executors.newFixedThreadPool(threadCount); + CQLTester.setUpClass(); + CQLTester.prepareServer(); + DatabaseDescriptor.setAutoSnapshot(false); + System.err.println("setupClass done."); + String memtableSetup = ""; + if (!memtableClass.isEmpty()) + memtableSetup = String.format(" AND memtable = '%s'", memtableClass); + keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); + table = createTable(keyspace, + "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid)) with compression = {'enabled': false}" + + memtableSetup); + execute("use " + keyspace + ";"); + if (useNet) + { + CQLTester.requireNetwork(); + executeNet(getDefaultVersion(), "use " + keyspace + ";"); + } + writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)"; + System.err.println("Prepared, batch " + BATCH + " threads " + threadCount + extraInfo()); + System.err.println("Disk access mode " + DatabaseDescriptor.getDiskAccessMode() + + " index " + DatabaseDescriptor.getIndexAccessMode()); + + cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); + cfs.disableAutoCompaction(); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); + } + + abstract Object[] writeArguments(long i); + + public void performWrite(long ofs, int count) throws Throwable + { + if (useNet) + { + if (threadCount == 1) + performWriteSerialNet(ofs, count); + else + performWriteThreadsNet(ofs, count); + } + else + { + if (threadCount == 1) + performWriteSerial(ofs, count); + else + performWriteThreads(ofs, count); + } + } + + public void performWriteSerial(long ofs, int count) throws Throwable + { + for (long i = ofs; i < ofs + count; ++i) + execute(writeStatement, writeArguments(i)); + } + + public void performWriteThreads(long ofs, int count) throws Throwable + { + List> futures = new ArrayList<>(); + for (int i = 0; i < count; ++i) + { + long pos = ofs + i; + futures.add(executorService.submit(() -> + { + try + { + execute(writeStatement, writeArguments(pos)); + return 1; + } + catch (Throwable throwable) + { + throw Throwables.propagate(throwable); + } + })); + } + int done = 0; + for (Future f : futures) + done += f.get(); + assert count == done; + } + + public void performWriteSerialNet(long ofs, int count) throws Throwable + { + for (long i = ofs; i < ofs + count; ++i) + sessionNet().execute(writeStatement, writeArguments(i)); + } + + public void performWriteThreadsNet(long ofs, int count) throws Throwable + { + List> futures = new ArrayList<>(); + for (long i = 0; i < count; ++i) + { + long pos = ofs + i; + futures.add(executorService.submit(() -> + { + try + { + sessionNet().execute(writeStatement, writeArguments(pos)); + return 1; + } + catch (Throwable throwable) + { + throw Throwables.propagate(throwable); + } + })); + } + long done = 0; + for (Future f : futures) + done += f.get(); + assert count == done; + } + + @TearDown(Level.Trial) + public void teardown() throws InterruptedException + { + executorService.shutdown(); + executorService.awaitTermination(15, TimeUnit.SECONDS); + + Memtable memtable = cfs.getTracker().getView().getCurrentMemtable(); + Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable); + System.err.format("\n%s in %s mode: %d ops, %s serialized bytes, %s\n", + memtable.getClass().getSimpleName(), + DatabaseDescriptor.getMemtableAllocationType(), + memtable.operationCount(), + FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()), + usage); + + doExtraChecks(); + + // do a flush to print sizes + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); + + CommitLog.instance.shutdownBlocking(); + CQLTester.tearDownClass(); + CQLTester.cleanup(); + } + + abstract void doExtraChecks(); + abstract String extraInfo(); +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/instance/WriteTest.java b/test/microbench/org/apache/cassandra/test/microbench/instance/WriteTest.java new file mode 100644 index 0000000000..eda93b755d --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/instance/WriteTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench.instance; + + +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.openjdk.jmh.annotations.*; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(value = 1) +@Threads(1) +@State(Scope.Benchmark) +public class WriteTest extends SimpleTableWriter +{ + + public enum EndOp + { + INMEM, TRUNCATE, FLUSH + } + + @Param({"INMEM", "TRUNCATE", "FLUSH"}) + EndOp flush = EndOp.INMEM; + + @Setup(Level.Trial) + public void setup() throws Throwable + { + super.commonSetup(); + } + + @Benchmark + public void writeTable() throws Throwable + { + long i; + for (i = 0; i <= count - BATCH; i += BATCH) + performWrite(i, BATCH); + if (i < count) + performWrite(i, Math.toIntExact(count - i)); + + switch (flush) + { + case FLUSH: + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); + // if we flush we also must truncate to avoid accummulating sstables + case TRUNCATE: + execute("TRUNCATE TABLE " + table); + // note: we turn snapshotting and durable writes (which would have caused a flush) off for this benchmark + break; + case INMEM: + if (!cfs.getLiveSSTables().isEmpty()) + throw new AssertionError("SSTables created for INMEM test."); + // leave unflushed, i.e. next iteration will overwrite data + default: + } + } + + public Object[] writeArguments(long i) + { + return new Object[] { i, i, i }; + } + + void doExtraChecks() + { + if (flush == WriteTest.EndOp.INMEM && !cfs.getLiveSSTables().isEmpty()) + throw new AssertionError("SSTables created for INMEM test."); + } + + String extraInfo() + { + return " flush " + flush; + } +} diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceFlushAndCleanup.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceFlushAndCleanup.java index 162049cf0b..c34887ad38 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceFlushAndCleanup.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnInstanceFlushAndCleanup.java @@ -20,6 +20,7 @@ package org.apache.cassandra.simulator.cluster; import java.util.function.Function; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; @@ -47,7 +48,7 @@ class OnInstanceFlushAndCleanup extends ClusterReliableAction { try { - cfs.forceBlockingFlush(); + Util.flush(cfs); if (cfs.forceCleanup(1) != CompactionManager.AllSSTableOpStatus.SUCCESSFUL) throw new IllegalStateException(); cfs.forceMajorCompaction(); diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java index 7862f60d60..45ff1f90bc 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java @@ -18,28 +18,31 @@ package org.apache.cassandra.simulator.paxos; +import java.util.Iterator; import java.util.List; import com.google.common.collect.ImmutableList; +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.TimeUUIDType; -import org.apache.cassandra.db.partitions.AbstractBTreePartition; -import org.apache.cassandra.db.partitions.ImmutableBTreePartition; -import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; @@ -108,8 +111,8 @@ public class Ballots long baseTable = latestBallotFromBaseTable(key, metadata); return new LatestBallots( promised.unixMicros(), - accepted == null || accepted.update.isEmpty() ? 0L : latestBallot(accepted.update), - latestBallot(committed.update), + accepted == null || accepted.update.isEmpty() ? 0L : latestBallot(accepted.update.iterator()), + latestBallot(committed.update.iterator()), baseTable ); }); @@ -166,11 +169,7 @@ public class Ballots List memtables = ImmutableList.copyOf(paxos.getTracker().getView().getAllMemtables()); for (Memtable memtable : memtables) { - Partition partition = memtable.getPartition(key); - if (partition == null) - continue; - - Row row = partition.getRow(paxos.metadata.get().comparator.make(metadata.id)); + Row row = getRow(key, metadata, paxos, memtable); if (row == null) continue; @@ -187,10 +186,18 @@ public class Ballots return result; } + private static Row getRow(DecoratedKey key, TableMetadata metadata, ColumnFamilyStore paxos, Memtable memtable) + { + final ClusteringComparator comparator = paxos.metadata.get().comparator; + UnfilteredRowIterator iter = memtable.rowIterator(key, Slices.with(comparator, Slice.make(comparator.make(metadata.id))), ColumnFilter.NONE, false, SSTableReadsListener.NOOP_LISTENER); + if (iter == null || !iter.hasNext()) + return null; + return (Row) iter.next(); + } + public static long latestBallotFromBaseTable(DecoratedKey key, TableMetadata metadata) { SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata, 0, key, Slice.ALL); - ImmutableBTreePartition partition; try (ReadExecutionController controller = cmd.executionController(); UnfilteredPartitionIterator partitions = cmd.executeLocally(controller)) { if (!partitions.hasNext()) @@ -198,10 +205,9 @@ public class Ballots try (UnfilteredRowIterator rows = partitions.next()) { - partition = ImmutableBTreePartition.create(rows); + return latestBallot(rows); } } - return latestBallot(partition); } private static long latestBallotFromBaseMemtable(DecoratedKey key, TableMetadata metadata) @@ -211,20 +217,27 @@ public class Ballots List memtables = ImmutableList.copyOf(table.getTracker().getView().getAllMemtables()); for (Memtable memtable : memtables) { - Partition partition = memtable.getPartition(key); - if (partition == null) - continue; + try (UnfilteredRowIterator partition = memtable.rowIterator(key)) + { + if (partition == null) + continue; - timestamp = max(timestamp, latestBallot((AbstractBTreePartition) partition)); + timestamp = max(timestamp, latestBallot(partition)); + } } return timestamp; } - private static long latestBallot(AbstractBTreePartition partition) + private static long latestBallot(Iterator partition) { long timestamp = 0L; - for (Row row : partition) - timestamp = row.accumulate((cd, v) -> max(v, cd.maxTimestamp()), timestamp); + while (partition.hasNext()) + { + Unfiltered unfiltered = partition.next(); + if (!unfiltered.isRow()) + continue; + timestamp = ((Row) unfiltered).accumulate((cd, v) -> max(v, cd.maxTimestamp()), timestamp); + } return timestamp; } diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java index e742d88056..e35c8a6d0b 100644 --- a/test/unit/org/apache/cassandra/ServerTestUtils.java +++ b/test/unit/org/apache/cassandra/ServerTestUtils.java @@ -172,7 +172,8 @@ public final class ServerTestUtils private static void cleanupDirectory(String dirName) { - cleanupDirectory(new File(dirName)); + if (dirName != null) + cleanupDirectory(new File(dirName)); } /** diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 060e77c180..0459cb3b3b 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -99,6 +99,7 @@ import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.view.TableViews; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; import org.apache.cassandra.dht.Range; @@ -252,7 +253,7 @@ public class Util rm.applyUnsafe(); ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(tableId); - store.forceBlockingFlush(); + Util.flush(store); return store; } @@ -1007,4 +1008,38 @@ public class Util return new File(targetBasePath.toPath().resolve(relative)); } + public static void flush(ColumnFamilyStore cfs) + { + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + } + + public static void flushTable(Keyspace keyspace, String table) + { + flush(keyspace.getColumnFamilyStore(table)); + } + + public static void flushTable(Keyspace keyspace, TableId table) + { + flush(keyspace.getColumnFamilyStore(table)); + } + + public static void flushTable(String keyspace, String table) + { + flushTable(Keyspace.open(keyspace), table); + } + + public static void flush(Keyspace keyspace) + { + FBUtilities.waitOnFutures(keyspace.flush(ColumnFamilyStore.FlushReason.UNIT_TESTS)); + } + + public static void flushKeyspace(String keyspaceName) + { + flush(Keyspace.open(keyspaceName)); + } + + public static void flush(TableViews view) + { + view.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + } } diff --git a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java index 17b06e6d01..1fb37352da 100644 --- a/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java +++ b/test/unit/org/apache/cassandra/batchlog/BatchlogManagerTest.java @@ -156,7 +156,7 @@ public class BatchlogManagerTest } // Flush the batchlog to disk (see CASSANDRA-6822). - Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(); + Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); assertEquals(100, BatchlogManager.instance.countAllBatches() - initialAllBatches); assertEquals(0, BatchlogManager.instance.getTotalBatchesReplayed() - initialReplayedBatches); @@ -240,7 +240,7 @@ public class BatchlogManagerTest } // Flush the batchlog to disk (see CASSANDRA-6822). - Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(); + Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); // Force batchlog replay and wait for it to complete. BatchlogManager.instance.startBatchlogReplay().get(); @@ -368,7 +368,7 @@ public class BatchlogManagerTest assertEquals(1, BatchlogManager.instance.countAllBatches() - initialAllBatches); // Flush the batchlog to disk (see CASSANDRA-6822). - Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(); + Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); assertEquals(1, BatchlogManager.instance.countAllBatches() - initialAllBatches); assertEquals(0, BatchlogManager.instance.getTotalBatchesReplayed() - initialReplayedBatches); diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java index 865ae1f69f..040409e424 100644 --- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java +++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java @@ -76,7 +76,7 @@ public class AutoSavingCacheTest RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), currentTimeMillis(), "key1"); rowBuilder.add(colDef, "val1"); rowBuilder.build().apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(2, cfs.getLiveSSTables().size()); diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 26024030bd..0255cc9e4d 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -59,6 +59,7 @@ import com.datastax.driver.core.exceptions.UnauthorizedException; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.Util; import org.apache.cassandra.auth.AuthCacheService; import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.auth.AuthSchemaChangeListener; @@ -635,7 +636,7 @@ public abstract class CQLTester { ColumnFamilyStore store = getCurrentColumnFamilyStore(keyspace); if (store != null) - store.forceBlockingFlush(); + Util.flush(store); } public void disableCompaction(String keyspace) diff --git a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java index a0653c1f29..eded145c66 100644 --- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java +++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java @@ -409,11 +409,11 @@ public class GcCompactionTest extends CQLTester createTable("create table %s (k int, c1 int, primary key (k, c1)) with compaction = {'class': 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones':'row'}"); execute("delete from %s where k = 1"); Set readers = new HashSet<>(getCurrentColumnFamilyStore().getLiveSSTables()); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); SSTableReader oldSSTable = getNewTable(readers); Thread.sleep(2000); execute("delete from %s where k = 1"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); SSTableReader newTable = getNewTable(readers); CompactionManager.instance.forceUserDefinedCompaction(oldSSTable.getFilename()); diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java index 861b60f5e8..2b0a3c8d60 100644 --- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java +++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java @@ -33,7 +33,6 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.index.Index; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.metrics.CacheMetrics; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.schema.CachingParams; @@ -164,7 +163,7 @@ public class KeyCacheCqlTest extends CQLTester } } - StorageService.instance.forceKeyspaceFlush(KEYSPACE_PER_TEST); + StorageService.instance.forceKeyspaceFlush(KEYSPACE_PER_TEST, ColumnFamilyStore.FlushReason.UNIT_TESTS); for (int pkInt = 0; pkInt < 20; pkInt++) { @@ -570,7 +569,7 @@ public class KeyCacheCqlTest extends CQLTester if (i % 10 == 9) { - Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).forceFlush().get(); + Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS).get(); if (index != null) triggerBlockingFlush(Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).indexManager.getIndexByName(index)); } @@ -580,7 +579,7 @@ public class KeyCacheCqlTest extends CQLTester private static void prepareTable(String table) throws IOException, InterruptedException, java.util.concurrent.ExecutionException { StorageService.instance.disableAutoCompaction(KEYSPACE_PER_TEST, table); - Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).forceFlush().get(); + Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS).get(); Keyspace.open(KEYSPACE_PER_TEST).getColumnFamilyStore(table).truncateBlocking(); } diff --git a/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java b/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java new file mode 100644 index 0000000000..ee5da9253e --- /dev/null +++ b/test/unit/org/apache/cassandra/cql3/MemtableQuickTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.Util; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; + +@RunWith(Parameterized.class) +public class MemtableQuickTest extends CQLTester +{ + static final Logger logger = LoggerFactory.getLogger(MemtableQuickTest.class); + + static final int partitions = 50_000; + static final int rowsPerPartition = 4; + + static final int deletedPartitionsStart = 20_000; + static final int deletedPartitionsEnd = deletedPartitionsStart + 10_000; + + static final int deletedRowsStart = 40_000; + static final int deletedRowsEnd = deletedRowsStart + 5_000; + + @Parameterized.Parameter() + public String memtableClass; + + @Parameterized.Parameters(name = "{0}") + public static List parameters() + { + return ImmutableList.of("skiplist", + "skiplist_sharded", + "skiplist_sharded_locking"); + } + + @BeforeClass + public static void setUp() + { + CQLTester.setUpClass(); + CQLTester.prepareServer(); + CQLTester.disablePreparedReuseForTest(); + logger.info("setupClass done."); + } + + @Test + public void testMemtable() throws Throwable + { + + String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); + String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" + + " with compression = {'enabled': false}" + + " and memtable = '" + memtableClass + "'"); + execute("use " + keyspace + ';'); + + String writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)"; + + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); + cfs.disableAutoCompaction(); + Util.flush(cfs); + + long i; + long limit = partitions; + logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition); + for (i = 0; i < limit; ++i) + { + for (long j = 0; j < rowsPerPartition; ++j) + execute(writeStatement, i, j, i + j); + } + + logger.info("Deleting partitions between {} and {}", deletedPartitionsStart, deletedPartitionsEnd); + for (i = deletedPartitionsStart; i < deletedPartitionsEnd; ++i) + { + // no partition exists, but we will create a tombstone + execute("DELETE FROM " + table + " WHERE userid = ?", i); + } + + logger.info("Deleting rows between {} and {}", deletedRowsStart, deletedRowsEnd); + for (i = deletedRowsStart; i < deletedRowsEnd; ++i) + { + // no row exists, but we will create a tombstone (and partition) + execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L); + } + + logger.info("Reading {} partitions", partitions); + for (i = 0; i < limit; ++i) + { + UntypedResultSet result = execute("SELECT * FROM " + table + " WHERE userid = ?", i); + if (i >= deletedPartitionsStart && i < deletedPartitionsEnd) + assertEmpty(result); + else + { + int start = 0; + if (i >= deletedRowsStart && i < deletedRowsEnd) + start = 1; + Object[][] rows = new Object[rowsPerPartition - start][]; + for (long j = start; j < rowsPerPartition; ++j) + rows[(int) (j - start)] = row(i, j, i + j); + assertRows(result, rows); + } + } + + int deletedPartitions = deletedPartitionsEnd - deletedPartitionsStart; + int deletedRows = deletedRowsEnd - deletedRowsStart; + logger.info("Selecting *"); + UntypedResultSet result = execute("SELECT * FROM " + table); + assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows); + + Util.flush(cfs); + + logger.info("Selecting *"); + result = execute("SELECT * FROM " + table); + assertRowCount(result, rowsPerPartition * (partitions - deletedPartitions) - deletedRows); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java b/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java index 7d6c6dd304..63ff055d2b 100644 --- a/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java +++ b/test/unit/org/apache/cassandra/cql3/MemtableSizeTest.java @@ -18,33 +18,53 @@ package org.apache.cassandra.cql3; +import java.util.List; + import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.Memtable; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.ObjectSizes; +@RunWith(Parameterized.class) public class MemtableSizeTest extends CQLTester { - static String keyspace; - String table; - ColumnFamilyStore cfs; + static final Logger logger = LoggerFactory.getLogger(MemtableSizeTest.class); - int partitions = 50_000; - int rowsPerPartition = 4; + static final int partitions = 50_000; + static final int rowsPerPartition = 4; - int deletedPartitions = 10_000; - int deletedRows = 5_000; + static final int deletedPartitions = 10_000; + static final int deletedRows = 5_000; + + @Parameterized.Parameter(0) + public String memtableClass; + + @Parameterized.Parameter(1) + public int differencePerPartition; + + @Parameterized.Parameters(name = "{0}") + public static List parameters() + { + return ImmutableList.of(new Object[]{"skiplist", 50}, + new Object[]{"skiplist_sharded", 60}); + } // must be within 50 bytes per partition of the actual size - final int MAX_DIFFERENCE = (partitions + deletedPartitions + deletedRows) * 50; + final long MAX_DIFFERENCE_PARTITIONS = (partitions + deletedPartitions + deletedRows); @BeforeClass public static void setUp() @@ -52,42 +72,44 @@ public class MemtableSizeTest extends CQLTester CQLTester.setUpClass(); CQLTester.prepareServer(); CQLTester.disablePreparedReuseForTest(); - System.err.println("setupClass done."); + logger.info("setupClass done."); } @Test - public void testTruncationReleasesLogSpace() + public void testSize() { - Util.flakyTest(this::testSize, 2, "Fails occasionally, see CASSANDRA-16684"); + Util.flakyTest(this::testSizeFlaky, 2, "Fails occasionally, see CASSANDRA-16684"); } - private void testSize() + private void testSizeFlaky() { try { - keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); - table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid)) with compression = {'enabled': false}"); + String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); + String table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))" + + " with compression = {'enabled': false}" + + " and memtable = '" + memtableClass + "'"); execute("use " + keyspace + ';'); String writeStatement = "INSERT INTO " + table + "(userid,picid,commentid)VALUES(?,?,?)"; - cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); cfs.disableAutoCompaction(); - cfs.forceBlockingFlush(); + Util.flush(cfs); long deepSizeBefore = ObjectSizes.measureDeep(cfs.getTracker().getView().getCurrentMemtable()); - System.out.printf("Memtable deep size before %s\n%n", - FBUtilities.prettyPrintMemory(deepSizeBefore)); + logger.info("Memtable deep size before {}\n", + FBUtilities.prettyPrintMemory(deepSizeBefore)); long i; long limit = partitions; - System.out.println("Writing " + partitions + " partitions of " + rowsPerPartition + " rows"); + logger.info("Writing {} partitions of {} rows", partitions, rowsPerPartition); for (i = 0; i < limit; ++i) { for (long j = 0; j < rowsPerPartition; ++j) execute(writeStatement, i, j, i + j); } - System.out.println("Deleting " + deletedPartitions + " partitions"); + logger.info("Deleting {} partitions", deletedPartitions); limit += deletedPartitions; for (; i < limit; ++i) { @@ -95,7 +117,7 @@ public class MemtableSizeTest extends CQLTester execute("DELETE FROM " + table + " WHERE userid = ?", i); } - System.out.println("Deleting " + deletedRows + " rows"); + logger.info("Deleting {} rows", deletedRows); limit += deletedRows; for (; i < limit; ++i) { @@ -103,31 +125,28 @@ public class MemtableSizeTest extends CQLTester execute("DELETE FROM " + table + " WHERE userid = ? AND picid = ?", i, 0L); } - if (!cfs.getLiveSSTables().isEmpty()) - System.out.println("Warning: " + cfs.getLiveSSTables().size() + " sstables created."); + logger.info("Warning: " + cfs.getLiveSSTables().size() + " sstables created."); Memtable memtable = cfs.getTracker().getView().getCurrentMemtable(); - long actualHeap = memtable.getAllocator().onHeap().owns(); - System.out.printf("Memtable in %s mode: %d ops, %s serialized bytes, %s (%.0f%%) on heap, %s (%.0f%%) off-heap%n", - DatabaseDescriptor.getMemtableAllocationType(), - memtable.getOperations(), - FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()), - FBUtilities.prettyPrintMemory(actualHeap), - 100 * memtable.getAllocator().onHeap().ownershipRatio(), - FBUtilities.prettyPrintMemory(memtable.getAllocator().offHeap().owns()), - 100 * memtable.getAllocator().offHeap().ownershipRatio()); + Memtable.MemoryUsage usage = Memtable.getMemoryUsage(memtable); + long actualHeap = usage.ownsOnHeap; + logger.info("Memtable in {} mode: {} ops, {} serialized bytes, {}\n", + DatabaseDescriptor.getMemtableAllocationType(), + memtable.operationCount(), + FBUtilities.prettyPrintMemory(memtable.getLiveDataSize()), + usage); long deepSizeAfter = ObjectSizes.measureDeep(memtable); - System.out.printf("Memtable deep size %s\n%n", - FBUtilities.prettyPrintMemory(deepSizeAfter)); + logger.info("Memtable deep size {}\n", + FBUtilities.prettyPrintMemory(deepSizeAfter)); long expectedHeap = deepSizeAfter - deepSizeBefore; String message = String.format("Expected heap usage close to %s, got %s.\n", FBUtilities.prettyPrintMemory(expectedHeap), FBUtilities.prettyPrintMemory(actualHeap)); - System.out.println(message); - Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= MAX_DIFFERENCE); + logger.info(message); + Assert.assertTrue(message, Math.abs(actualHeap - expectedHeap) <= MAX_DIFFERENCE_PARTITIONS * differencePerPartition); } catch (Throwable throwable) { diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java index 1ebd9ec96e..49195f57cf 100644 --- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java +++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java @@ -26,6 +26,7 @@ import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.config.Config.DiskFailurePolicy; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogSegment; import org.apache.cassandra.db.Keyspace; @@ -115,7 +116,10 @@ public class OutOfSpaceTest extends CQLTester { try { - Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()).forceFlush().get(); + Keyspace.open(KEYSPACE) + .getColumnFamilyStore(currentTable()) + .forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS) + .get(); fail("FSWriteError expected."); } catch (ExecutionException e) diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsPartialTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsPartialTest.java index a827be3031..a05621cf42 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsPartialTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsPartialTest.java @@ -23,8 +23,8 @@ import java.util.concurrent.TimeUnit; import org.junit.Ignore; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; /* ViewComplexTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167) * Any changes here check if they apply to the other classes: @@ -64,17 +64,17 @@ public class ViewComplexDeletionsPartialTest extends ViewAbstractParameterizedTe updateView("UPDATE %s USING TIMESTAMP 10 SET b=1 WHERE k=1 AND c=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, 1, null, 1)); assertRows(executeView("SELECT * FROM %s"), row(1, 1)); updateView("DELETE b FROM %s USING TIMESTAMP 11 WHERE k=1 AND c=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertEmpty(execute("SELECT * from %s")); assertEmpty(executeView("SELECT * FROM %s")); updateView("UPDATE %s USING TIMESTAMP 1 SET a=1 WHERE k=1 AND c=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, 1, 1, null)); assertRows(executeView("SELECT * FROM %s"), row(1, 1)); @@ -205,27 +205,27 @@ public class ViewComplexDeletionsPartialTest extends ViewAbstractParameterizedTe execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) using timestamp 0", 1, 1, 1, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT * FROM %s"), row(1, 1, 1, 1)); // remove view row updateView("UPDATE %s using timestamp 1 set b = null WHERE a=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT * FROM %s")); // remove base row, no view updated generated. updateView("DELETE FROM %s using timestamp 2 where a=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT * FROM %s")); // restor view row with b,c column. d is still tombstone updateView("UPDATE %s using timestamp 3 set b = 1,c = 1 where a=1"); // upsert if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT * FROM %s"), row(1, 1, 1, null)); } diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsTest.java index 388b8a20b6..d9fe7f1212 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsTest.java @@ -24,13 +24,13 @@ import java.util.stream.Collectors; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -75,35 +75,35 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest updateView("Insert into %s (p, v1, v2) values (3, 1, 3) using timestamp 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v2, WRITETIME(v2) from %s WHERE v1 = ? AND p = ?", 1, 3), row(3, 1L)); // sstable-2 updateView("Delete from %s using timestamp 2 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s")); // sstable-3 updateView("Insert into %s (p, v1) values (3, 1) using timestamp 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, null, null)); // sstable-4 updateView("UPdate %s using timestamp 4 set v1 = 2 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(2, 3, null, null)); // sstable-5 updateView("UPdate %s using timestamp 5 set v1 = 1 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, null, null)); @@ -152,7 +152,7 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest updateView("Insert into %s (p1, p2, v1, v2) values (1, 2, 3, 4) using timestamp 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, v2, WRITETIME(v2) from %s WHERE p1 = ? AND p2 = ?", 1, 2), row(3, 4, 1L)); @@ -160,14 +160,14 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest updateView("Delete from %s using timestamp 2 where p1 = 1 and p2 = 2;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // view are empty assertRowsIgnoringOrder(executeView("SELECT * FROM %s")); // insert PK with TS=3 updateView("Insert into %s (p1, p2) values (1, 2) using timestamp 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // deleted column in MV remained dead assertRowsIgnoringOrder(executeView("SELECT * FROM %s"), row(2, 1, null, null)); @@ -177,21 +177,21 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest // reset values updateView("Insert into %s (p1, p2, v1, v2) values (1, 2, 3, 4) using timestamp 10;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, v2, WRITETIME(v2) from %s WHERE p1 = ? AND p2 = ?", 1, 2), row(3, 4, 10L)); updateView("UPDATE %s using timestamp 20 SET v2 = 5 WHERE p1 = 1 and p2 = 2"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, v2, WRITETIME(v2) from %s WHERE p1 = ? AND p2 = ?", 1, 2), row(3, 5, 20L)); updateView("DELETE FROM %s using timestamp 10 WHERE p1 = 1 and p2 = 2"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, v2, WRITETIME(v2) from %s WHERE p1 = ? AND p2 = ?", 1, 2), row(null, 5, 20L)); @@ -211,21 +211,21 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest updateView("Insert into %s (p, v1, v2) values (3, 1, 5) using timestamp 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v2, WRITETIME(v2) from %s WHERE v1 = ? AND p = ?", 1, 3), row(5, 1L)); // remove row/mv TS=2 updateView("Delete from %s using timestamp 2 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // view are empty assertRowsIgnoringOrder(executeView("SELECT * FROM %s")); // insert PK with TS=3 updateView("Insert into %s (p, v1) values (3, 1) using timestamp 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // deleted column in MV remained dead assertRowsIgnoringOrder(executeView("SELECT * FROM %s"), row(1, 3, null)); @@ -233,7 +233,7 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest updateView("Insert into %s (p, v1, v2) values (3, 1, 5) using timestamp 2;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // deleted column in MV remained dead assertRowsIgnoringOrder(executeView("SELECT * FROM %s"), row(1, 3, null)); assertRowsIgnoringOrder(executeView("SELECT * from %s limit 1"), row(1, 3, null)); @@ -242,7 +242,7 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest executeNet("UPDATE %s USING TIMESTAMP 3 SET v2 = ? WHERE p = ?", 4, 3); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, 4, 3L)); @@ -260,11 +260,11 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest ColumnFamilyStore batchlog = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.BATCHES); batchlog.disableAutoCompaction(); - batchlog.forceBlockingFlush(); + Util.flush(batchlog); int batchlogSSTables = batchlog.getLiveSSTables().size(); updateView("INSERT INTO %s(k1, v1) VALUES(1, 1)"); - batchlog.forceBlockingFlush(); + Util.flush(batchlog); assertEquals(batchlogSSTables, batchlog.getLiveSSTables().size()); } } diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessLimitTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessLimitTest.java index 73f6b742b8..0d3ab67a21 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessLimitTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessLimitTest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; import static org.junit.Assert.assertEquals; @@ -77,8 +78,8 @@ public class ViewComplexLivenessLimitTest extends ViewAbstractParameterizedTest if (flush) { - ks.getColumnFamilyStore(mv1).forceBlockingFlush(); - ks.getColumnFamilyStore(mv2).forceBlockingFlush(); + Util.flushTable(ks, mv1); + Util.flushTable(ks, mv2); } for (String view : Arrays.asList(mv1, mv2)) diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessTest.java index cbb2a5f927..99caecc54c 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexLivenessTest.java @@ -20,9 +20,9 @@ package org.apache.cassandra.cql3; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -115,7 +115,7 @@ public class ViewComplexLivenessTest extends ViewAbstractParameterizedTest assertRowsIgnoringOrder(executeView("SELECT p, v1, v2 from %s"), row(1, 1, 1)); updateView("Update %s set v1 = null WHERE p = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT p, v1, v2 from %s")); cfs.forceMajorCompaction(); // before gc grace second, strict-liveness tombstoned dead row remains @@ -128,7 +128,7 @@ public class ViewComplexLivenessTest extends ViewAbstractParameterizedTest assertEquals(0, cfs.getLiveSSTables().size()); updateView("Update %s using ttl 5 set v1 = 1 WHERE p = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT p, v1, v2 from %s"), row(1, 1, 1)); cfs.forceMajorCompaction(); // before ttl+gc_grace_second, strict-liveness ttled dead row remains diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexTTLTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexTTLTest.java index 3a66118ff3..93c25c9881 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexTTLTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexTTLTest.java @@ -20,8 +20,8 @@ package org.apache.cassandra.cql3; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -66,7 +66,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("UPDATE %s SET a = 1 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, 1, null)); assertRows(executeView("SELECT * from %s"), row(1, 1, null)); @@ -74,7 +74,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("DELETE a FROM %s WHERE k = 1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s")); assertEmpty(executeView("SELECT * from %s")); @@ -82,7 +82,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("INSERT INTO %s (k) VALUES (1);"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, null, null)); assertEmpty(executeView("SELECT * from %s")); @@ -90,7 +90,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TTL 5 SET a = 10 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, 10, null)); assertRows(executeView("SELECT * from %s"), row(10, 1, null)); @@ -98,7 +98,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("UPDATE %s SET b = 100 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, 10, 100)); assertRows(executeView("SELECT * from %s"), row(10, 1, 100)); @@ -113,7 +113,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("DELETE b FROM %s WHERE k=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(execute("SELECT * from %s"), row(1, null, null)); assertEmpty(executeView("SELECT * from %s")); @@ -121,7 +121,7 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("DELETE FROM %s WHERE k=1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertEmpty(execute("SELECT * from %s")); assertEmpty(executeView("SELECT * from %s")); @@ -198,11 +198,11 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("INSERT INTO %s (p, c, v) VALUES (0, 0, 0) using timestamp 1;"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); updateView("INSERT INTO %s (p, c, v) VALUES (0, 0, 0) USING TTL 3 and timestamp 1;"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); Thread.sleep(4000); @@ -213,11 +213,11 @@ public class ViewComplexTTLTest extends ViewAbstractParameterizedTest updateView("INSERT INTO %s (p, c, v) VALUES (0, 0, 0) USING TTL 3 and timestamp 1;"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); updateView("INSERT INTO %s (p, c, v) VALUES (0, 0, 0) USING timestamp 1;"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); Thread.sleep(4000); diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java index 998bf7637b..3231d3b851 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexTest.java @@ -29,9 +29,9 @@ import java.util.Map; import com.google.common.base.Objects; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.fail; @@ -73,37 +73,37 @@ public class ViewComplexTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TIMESTAMP 1 set v1 =1 where p1 = 1 AND p2 = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s"), row(1, 1, 1, null)); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s"), row(1, 1, 1, null)); updateView("UPDATE %s USING TIMESTAMP 2 set v1 = null, v2 = 1 where p1 = 1 AND p2 = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, 1)); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, 1)); updateView("UPDATE %s USING TIMESTAMP 2 set v2 = null where p1 = 1 AND p2 = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s")); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s")); updateView("INSERT INTO %s (p1,p2) VALUES(1,1) USING TIMESTAMP 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, null)); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, null)); updateView("DELETE FROM %s USING TIMESTAMP 4 WHERE p1 =1 AND p2 = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s")); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s")); updateView("UPDATE %s USING TIMESTAMP 5 set v2 = 1 where p1 = 1 AND p2 = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, 1)); assertRowsIgnoringOrder(executeView("SELECT p1, p2, v1, v2 from %s"), row(1, 1, null, 1)); } diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexTombstoneTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexTombstoneTest.java index c88e175c9e..a873135c0b 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexTombstoneTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexTombstoneTest.java @@ -25,12 +25,12 @@ import java.util.stream.Collectors; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.io.sstable.SSTableIdFactory; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.utils.FBUtilities; /* ViewComplexTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167) * Any changes here check if they apply to the other classes: @@ -70,14 +70,14 @@ public class ViewComplexTombstoneTest extends ViewAbstractParameterizedTest updateView("Insert into %s (p, v1, v2) values (3, 1, 3) using timestamp 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v2, WRITETIME(v2) from %s WHERE v1 = ? AND p = ?", 1, 3), row(3, 1L)); // sstable 2 updateView("UPdate %s using timestamp 2 set v2 = null where p = 3"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v2, WRITETIME(v2) from %s WHERE v1 = ? AND p = ?", 1, 3), row(null, null)); @@ -85,14 +85,14 @@ public class ViewComplexTombstoneTest extends ViewAbstractParameterizedTest updateView("UPdate %s using timestamp 3 set v1 = 2 where p = 3"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(2, 3, null, null)); // sstable 4 updateView("UPdate %s using timestamp 4 set v1 = 1 where p = 3"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, null, null)); diff --git a/test/unit/org/apache/cassandra/cql3/ViewComplexUpdatesTest.java b/test/unit/org/apache/cassandra/cql3/ViewComplexUpdatesTest.java index d63d3a684d..74f4038503 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewComplexUpdatesTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewComplexUpdatesTest.java @@ -22,8 +22,8 @@ import java.util.concurrent.TimeUnit; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; /* ViewComplexTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167) * Any changes here check if they apply to the other classes: @@ -63,7 +63,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TIMESTAMP 0 SET v1 = 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, 1, null)); assertRowsIgnoringOrder(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0)); @@ -71,7 +71,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("DELETE v1 FROM %s USING TIMESTAMP 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertEmpty(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); assertEmpty(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); @@ -80,7 +80,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TIMESTAMP 1 SET v1 = 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertEmpty(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); assertEmpty(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); @@ -88,7 +88,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TIMESTAMP 2 SET v2 = 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, null, 1)); assertRowsIgnoringOrder(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0)); @@ -96,7 +96,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("DELETE v1 FROM %s USING TIMESTAMP 3 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, null, 1)); assertRowsIgnoringOrder(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0)); @@ -104,7 +104,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("DELETE v2 FROM %s USING TIMESTAMP 4 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertEmpty(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); assertEmpty(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0)); @@ -112,7 +112,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s USING TTL 3 SET v2 = 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, null, 1)); assertRowsIgnoringOrder(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0)); @@ -125,7 +125,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s SET v2 = 1 WHERE p = 0 AND c = 0"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0, null, 1)); assertRowsIgnoringOrder(executeView("SELECT * from %s WHERE c = ? AND p = ?", 0, 0), row(0, 0)); @@ -161,23 +161,23 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s SET l=l+[1,2,3] WHERE k = 1 AND c = 1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(executeView("SELECT * from %s"), row(1, 1, null, null)); updateView("UPDATE %s SET l=l-[1,2] WHERE k = 1 AND c = 1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(executeView("SELECT * from %s"), row(1, 1, null, null)); updateView("UPDATE %s SET b=3 WHERE k=1 AND c=1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(executeView("SELECT * from %s"), row(1, 1, null, 3)); updateView("UPDATE %s SET b=null, l=l-[3], s=s-{3} WHERE k = 1 AND c = 1"); if (flush) { - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); ks.getColumnFamilyStore(mv).forceMajorCompaction(); } assertRowsIgnoringOrder(execute("SELECT k,c,a,b from %s")); @@ -185,7 +185,7 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest updateView("UPDATE %s SET m=m+{3:3}, l=l-[1], s=s-{2} WHERE k = 1 AND c = 1"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT k,c,a,b from %s"), row(1, 1, null, null)); assertRowsIgnoringOrder(executeView("SELECT * from %s"), row(1, 1, null, null)); @@ -223,23 +223,23 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest // reset value updateView("Insert into %s (p, v1, v2) values (3, 1, 3) using timestamp 6;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, 3, 6L)); // increase pk's timestamp to 20 updateView("Insert into %s (p) values (3) using timestamp 20;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, 3, 6L)); // change v1's to 2 and remove existing view row with ts7 updateView("UPdate %s using timestamp 7 set v1 = 2 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(2, 3, 3, 6L)); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s" + " limit 1"), row(2, 3, 3, 6L)); // change v1's to 1 and remove existing view row with ts8 updateView("UPdate %s using timestamp 8 set v1 = 1 where p = 3;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT v1, p, v2, WRITETIME(v2) from %s"), row(1, 3, 3, 6L)); } @@ -269,41 +269,41 @@ public class ViewComplexUpdatesTest extends ViewAbstractParameterizedTest ks.getColumnFamilyStore(mv).disableAutoCompaction(); updateView("DELETE FROM %s USING TIMESTAMP 0 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // sstable-1, Set initial values TS=1 updateView("INSERT INTO %s(k, a, b) VALUES (1, 1, 1) USING TIMESTAMP 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 1, 1)); updateView("UPDATE %s USING TIMESTAMP 10 SET b = 2 WHERE k = 1;"); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 1, 2)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 1, 2)); updateView("UPDATE %s USING TIMESTAMP 2 SET a = 2 WHERE k = 1;"); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 2, 2)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); ks.getColumnFamilyStore(mv).forceMajorCompaction(); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 2, 2)); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s limit 1"), row(1, 2, 2)); updateView("UPDATE %s USING TIMESTAMP 11 SET a = 1 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 1, 2)); assertRowsIgnoringOrder(execute("SELECT k,a,b from %s"), row(1, 1, 2)); // set non-key base column as tombstone, view row is removed with shadowable updateView("UPDATE %s USING TIMESTAMP 12 SET a = null WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s")); assertRowsIgnoringOrder(execute("SELECT k,a,b from %s"), row(1, null, 2)); // column b should be alive updateView("UPDATE %s USING TIMESTAMP 13 SET a = 1 WHERE k = 1;"); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(executeView("SELECT k,a,b from %s"), row(1, 1, 2)); assertRowsIgnoringOrder(execute("SELECT k,a,b from %s"), row(1, 1, 2)); diff --git a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java index 685e679361..47d5b8e237 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java +++ b/test/unit/org/apache/cassandra/cql3/ViewFiltering1Test.java @@ -27,9 +27,9 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.utils.FBUtilities; /* ViewFilteringTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167) * Any changes here check if they apply to the other classes @@ -99,7 +99,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) using timestamp 0", 1, 1, 1, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // views should be updated. assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1, 1)); @@ -111,7 +111,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 1 set c = ? WHERE a=?", 0, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowCount(execute("SELECT * FROM " + mv1), 0); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -122,7 +122,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 2 set c = ? WHERE a=?", 1, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // row should be back in views. assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1, 1)); @@ -134,7 +134,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 3 set d = ? WHERE a=?", 0, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1, 0)); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -145,7 +145,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 4 set c = ? WHERE a=?", 0, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowCount(execute("SELECT * FROM " + mv1), 0); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -156,7 +156,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 5 set d = ? WHERE a=?", 1, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // should not update as c=0 assertRowCount(execute("SELECT * FROM " + mv1), 0); @@ -179,7 +179,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 7 set b = ? WHERE a=?", 2, 1); if (flush) { - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); for (String view : getViews()) ks.getColumnFamilyStore(view).forceMajorCompaction(); } @@ -193,7 +193,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("DELETE b, c FROM %s using timestamp 6 WHERE a=?", 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * FROM %s"), row(1, 2, null, 1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1)); @@ -205,7 +205,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("DELETE FROM %s using timestamp 8 where a=?", 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowCount(execute("SELECT * FROM " + mv1), 0); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -216,7 +216,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("UPDATE %s using timestamp 9 set b = ?,c = ? where a=?", 1, 1, 1); // upsert if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1, null)); assertRows(execute("SELECT * FROM " + mv2)); @@ -227,7 +227,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("DELETE FROM %s using timestamp 10 where a=?", 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowCount(execute("SELECT * FROM " + mv1), 0); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -238,7 +238,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) using timestamp 11", 1, 1, 1, 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // row should be back in views. assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1, 1)); @@ -250,7 +250,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest updateView("DELETE FROM %s using timestamp 12 where a=?", 1); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowCount(execute("SELECT * FROM " + mv1), 0); assertRowCount(execute("SELECT * FROM " + mv2), 0); @@ -307,7 +307,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest list(1, 1, 2), set(1, 2), map(1, 1, 2, 2)); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1), row(1, 1, 1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv2), row(1, 1)); @@ -315,7 +315,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest assertRowsIgnoringOrder(execute("SELECT * FROM " + mv4), row(1, 1)); execute("UPDATE %s SET l=l-[1] WHERE a = 1 AND b = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv2)); @@ -323,7 +323,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest assertRowsIgnoringOrder(execute("SELECT * FROM " + mv4), row(1, 1)); execute("UPDATE %s SET s=s-{2}, m=m-{2} WHERE a = 1 AND b = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT a,b,c FROM %s"), row(1, 1, 1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1)); @@ -332,7 +332,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest assertRowsIgnoringOrder(execute("SELECT * FROM " + mv4), row(1, 1)); execute("UPDATE %s SET m=m-{1} WHERE a = 1 AND b = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT a,b,c FROM %s"), row(1, 1, 1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1)); @@ -342,7 +342,7 @@ public class ViewFiltering1Test extends ViewAbstractParameterizedTest // filter conditions result not changed execute("UPDATE %s SET l=l+[2], s=s-{0}, m=m+{3:3} WHERE a = 1 AND b = 1"); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRowsIgnoringOrder(execute("SELECT a,b,c FROM %s"), row(1, 1, 1)); assertRowsIgnoringOrder(execute("SELECT * FROM " + mv1)); diff --git a/test/unit/org/apache/cassandra/cql3/ViewFiltering2Test.java b/test/unit/org/apache/cassandra/cql3/ViewFiltering2Test.java index 8ce1bd5496..17709a800c 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewFiltering2Test.java +++ b/test/unit/org/apache/cassandra/cql3/ViewFiltering2Test.java @@ -23,8 +23,9 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.Util; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.utils.FBUtilities; /* ViewFilteringTest class has been split into multiple ones because of timeout issues (CASSANDRA-16670, CASSANDRA-17167) * Any changes here check if they apply to the other classes @@ -301,14 +302,14 @@ public class ViewFiltering2Test extends ViewAbstractParameterizedTest assertRows(executeView("SELECT d FROM %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //update c's timestamp TS=2 executeNet("UPDATE %s USING TIMESTAMP 2 SET c = ? WHERE a = ? and b = ? ", 1, 0, 0); assertRows(executeView("SELECT d FROM %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //change c's value and TS=3, tombstones c=1 and adds c=0 record executeNet("UPDATE %s USING TIMESTAMP 3 SET c = ? WHERE a = ? and b = ? ", 0, 0, 0); @@ -317,7 +318,7 @@ public class ViewFiltering2Test extends ViewAbstractParameterizedTest if (flush) { ks.getColumnFamilyStore(mv).forceMajorCompaction(); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); } //change c's value back to 1 with TS=4, check we can see d @@ -325,7 +326,7 @@ public class ViewFiltering2Test extends ViewAbstractParameterizedTest if (flush) { ks.getColumnFamilyStore(mv).forceMajorCompaction(); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); } assertRows(executeView("SELECT d, e FROM %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0, null)); @@ -335,14 +336,14 @@ public class ViewFiltering2Test extends ViewAbstractParameterizedTest assertRows(executeView("SELECT d, e FROM %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0, 1)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //Change d value @ TS=2 executeNet("UPDATE %s USING TIMESTAMP 2 SET d = ? WHERE a = ? and b = ? ", 2, 0, 0); assertRows(executeView("SELECT d FROM %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(2)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //Change d value @ TS=3 executeNet("UPDATE %s USING TIMESTAMP 3 SET d = ? WHERE a = ? and b = ? ", 1, 0, 0); @@ -429,7 +430,7 @@ public class ViewFiltering2Test extends ViewAbstractParameterizedTest for (int i = 0; i < 100; i++) updateView("INSERT into %s (k,c,val)VALUES(?,?,?)", 0, i % 2, "baz"); - Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(); + Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); Assert.assertEquals(2, execute("select * from %s").size()); Assert.assertEquals(2, executeView("select * from %s").size()); diff --git a/test/unit/org/apache/cassandra/cql3/ViewRangesTest.java b/test/unit/org/apache/cassandra/cql3/ViewRangesTest.java index bb2ea676d4..5d6e8a7588 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewRangesTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewRangesTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.cql3; import org.junit.Assert; import org.junit.Test; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; /* @@ -55,7 +56,7 @@ public class ViewRangesTest extends ViewAbstractTest updateView("DELETE FROM %s USING TIMESTAMP 10 WHERE k1 = 1 and c1=1"); if (flush) - Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(); + Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); String table = KEYSPACE + "." + currentTable(); updateView("BEGIN BATCH " + diff --git a/test/unit/org/apache/cassandra/cql3/ViewTest.java b/test/unit/org/apache/cassandra/cql3/ViewTest.java index 75efa5703d..d82fd322e1 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import com.datastax.driver.core.ResultSet; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -144,7 +145,7 @@ public class ViewTest extends ViewAbstractTest for (int i = 0; i < 100; i++) updateView("INSERT into %s (k,c,val)VALUES(?,?,?)", 0, i % 2, "baz"); - Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(); + Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); Assert.assertEquals(2, execute("select * from %s").size()); Assert.assertEquals(2, executeView("select * from %s").size()); @@ -323,7 +324,7 @@ public class ViewTest extends ViewAbstractTest assertRows(executeView("SELECT a, b, c from %s WHERE b = ?", 1), row(0, 1, null)); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentView()); - cfs.forceBlockingFlush(); + Util.flush(cfs); Assert.assertEquals(1, cfs.getLiveSSTables().size()); } @@ -436,22 +437,22 @@ public class ViewTest extends ViewAbstractTest for (int i = 0; i < 1024; i++) execute("INSERT into %s (k,c,val)VALUES(?,?,?)", i, i, String.valueOf(i)); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (int i = 0; i < 1024; i++) execute("INSERT into %s (k,c,val)VALUES(?,?,?)", i, i, String.valueOf(i)); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (int i = 0; i < 1024; i++) execute("INSERT into %s (k,c,val)VALUES(?,?,?)", i, i, String.valueOf(i)); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (int i = 0; i < 1024; i++) execute("INSERT into %s (k,c,val)VALUES(?,?,?)", i, i, String.valueOf(i)); - cfs.forceBlockingFlush(); + Util.flush(cfs); String mv1 = createViewAsync("CREATE MATERIALIZED VIEW %s AS SELECT * FROM %s " + "WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)"); diff --git a/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java b/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java index 6bfbb5c099..9c45936391 100644 --- a/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java +++ b/test/unit/org/apache/cassandra/cql3/ViewTimesTest.java @@ -26,9 +26,9 @@ import org.junit.Test; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Row; +import org.apache.cassandra.Util; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.utils.FBUtilities; import org.assertj.core.api.Assertions; import static org.junit.Assert.assertEquals; @@ -106,18 +106,18 @@ public class ViewTimesTest extends ViewAbstractTest assertRows(executeView("SELECT d from %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // change c's value and TS=3, tombstones c=1 and adds c=0 record executeNet("UPDATE %s USING TIMESTAMP 3 SET c = ? WHERE a = ? and b = ? ", 0, 0, 0); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); assertRows(executeView("SELECT d from %s WHERE c = ? and a = ? and b = ?", 1, 0, 0)); if(flush) { ks.getColumnFamilyStore(currentView()).forceMajorCompaction(); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); } @@ -126,7 +126,7 @@ public class ViewTimesTest extends ViewAbstractTest if (flush) { ks.getColumnFamilyStore(currentView()).forceMajorCompaction(); - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); } assertRows(executeView("SELECT d,e from %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0, null)); @@ -136,7 +136,7 @@ public class ViewTimesTest extends ViewAbstractTest assertRows(executeView("SELECT d,e from %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(0, 1)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //Change d value @ TS=2 @@ -144,7 +144,7 @@ public class ViewTimesTest extends ViewAbstractTest assertRows(executeView("SELECT d from %s WHERE c = ? and a = ? and b = ?", 1, 0, 0), row(2)); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); //Change d value @ TS=3 executeNet("UPDATE %s USING TIMESTAMP 3 SET d = ? WHERE a = ? and b = ? ", 1, 0, 0); diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index 4f41f4ac7c..5263a6d609 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -845,6 +845,7 @@ public class DescribeStatementTest extends CQLTester " AND comment = ''\n" + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + " AND crc_check_chance = 1.0\n" + " AND default_time_to_live = 0\n" + " AND extensions = {}\n" + @@ -865,6 +866,7 @@ public class DescribeStatementTest extends CQLTester " AND comment = ''\n" + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}\n" + " AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}\n" + + " AND memtable = 'default'\n" + " AND crc_check_chance = 1.0\n" + " AND extensions = {}\n" + " AND gc_grace_seconds = 864000\n" + diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java index c6d6390503..bac7e7917c 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/CrcCheckChanceTest.java @@ -24,6 +24,8 @@ import java.util.concurrent.Future; import org.junit.Test; import org.junit.Assert; + +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; @@ -68,7 +70,7 @@ public class CrcCheckChanceTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(CQLTester.KEYSPACE).getColumnFamilyStore(currentTable()); ColumnFamilyStore indexCfs = cfs.indexManager.getAllIndexColumnFamilyStores().iterator().next(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Assert.assertEquals(0.99, cfs.getCrcCheckChance(), 0.0); Assert.assertEquals(0.99, cfs.getLiveSSTables().iterator().next().getCrcCheckChance(), 0.0); @@ -96,19 +98,19 @@ public class CrcCheckChanceTest extends CQLTester execute("INSERT INTO %s(p, c, v) values (?, ?, ?)", "p1", "k2", "v2"); execute("INSERT INTO %s(p, s) values (?, ?)", "p2", "sv2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); execute("INSERT INTO %s(p, c, v, s) values (?, ?, ?, ?)", "p1", "k1", "v1", "sv1"); execute("INSERT INTO %s(p, c, v) values (?, ?, ?)", "p1", "k2", "v2"); execute("INSERT INTO %s(p, s) values (?, ?)", "p2", "sv2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); execute("INSERT INTO %s(p, c, v, s) values (?, ?, ?, ?)", "p1", "k1", "v1", "sv1"); execute("INSERT INTO %s(p, c, v) values (?, ?, ?)", "p1", "k2", "v2"); execute("INSERT INTO %s(p, s) values (?, ?)", "p2", "sv2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.forceMajorCompaction(); //Now let's change via JMX @@ -182,7 +184,7 @@ public class CrcCheckChanceTest extends CQLTester execute("INSERT INTO %s(p, c, v) values (?, ?, ?)", "p1", "k2", "v2"); execute("INSERT INTO %s(p, s) values (?, ?)", "p2", "sv2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); } DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(1); diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java index 5d367de49c..9ecc3ae6f1 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTableMetadataTrackingTest.java @@ -19,6 +19,7 @@ package org.apache.cassandra.cql3.validation.miscellaneous; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -40,7 +41,7 @@ public class SSTableMetadataTrackingTest extends CQLTester createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b))"); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("INSERT INTO %s (a,b,c) VALUES (1,1,'1') using timestamp 9999"); - cfs.forceBlockingFlush(); + Util.flush(cfs); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); assertEquals(Integer.MAX_VALUE, metadata.maxLocalDeletionTime); @@ -57,7 +58,7 @@ public class SSTableMetadataTrackingTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("INSERT INTO %s (a,b,c) VALUES (1,1,'1') using timestamp 10000"); execute("DELETE FROM %s USING TIMESTAMP 9999 WHERE a = 1 and b = 1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); assertEquals(10000, metadata.maxTimestamp); @@ -76,7 +77,7 @@ public class SSTableMetadataTrackingTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("INSERT INTO %s (a,b,c) VALUES (1,1,'1') using timestamp 10000"); execute("DELETE FROM %s USING TIMESTAMP 9999 WHERE a = 1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); assertEquals(10000, metadata.maxTimestamp); @@ -95,7 +96,7 @@ public class SSTableMetadataTrackingTest extends CQLTester createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 10000"); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("DELETE FROM %s USING TIMESTAMP 9999 WHERE a = 1 and b = 1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); @@ -115,7 +116,7 @@ public class SSTableMetadataTrackingTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("DELETE FROM %s USING TIMESTAMP 9999 WHERE a = 1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); @@ -135,7 +136,7 @@ public class SSTableMetadataTrackingTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("INSERT INTO %s (a) VALUES (1) USING TIMESTAMP 9999"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); @@ -154,7 +155,7 @@ public class SSTableMetadataTrackingTest extends CQLTester createTable("CREATE TABLE %s (a int, PRIMARY KEY (a))"); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); execute("DELETE FROM %s USING TIMESTAMP 9999 WHERE a=1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); StatsMetadata metadata = cfs.getLiveSSTables().iterator().next().getSSTableMetadata(); assertEquals(9999, metadata.minTimestamp); diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java index c4a376648d..fddd59bc4c 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AlterTest.java @@ -19,18 +19,23 @@ package org.apache.cassandra.cql3.validation.operations; import java.util.UUID; +import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.SkipListMemtable; +import org.apache.cassandra.db.memtable.TestMemtable; import org.apache.cassandra.dht.OrderPreservingPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspaceTables; import org.apache.cassandra.service.StorageService; @@ -39,6 +44,7 @@ import org.apache.cassandra.utils.FBUtilities; import static java.lang.String.format; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -548,72 +554,104 @@ public class AlterTest extends CQLTester } } + @Test + public void testAlterTableWithMemtable() throws Throwable + { + createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"); + assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory()); + assertSchemaOption("memtable", null); + + testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("test_shortname", SkipListMemtable.FACTORY, SkipListMemtable.class); + + // verify memtable does not change on other ALTER + alterTable("ALTER TABLE %s" + + " WITH compression = {'class': 'LZ4Compressor'};"); + assertSchemaOption("memtable", "test_shortname"); + + testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class); + + + assertAlterTableThrowsException(ConfigurationException.class, + "The 'class_name' option must be specified.", + "ALTER TABLE %s" + + " WITH memtable = 'test_empty_class';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Memtable class org.apache.cassandra.db.memtable.SkipListMemtable does not accept any futher parameters, but {invalid=throw} were given.", + "ALTER TABLE %s" + + " WITH memtable = 'test_invalid_param';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Could not create memtable factory for class NotExisting", + "ALTER TABLE %s" + + " WITH memtable = 'test_unknown_class';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Memtable class org.apache.cassandra.db.memtable.TestMemtable does not accept any futher parameters, but {invalid=throw} were given.", + "ALTER TABLE %s" + + " WITH memtable = 'test_invalid_extra_param';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Could not create memtable factory for class " + CreateTest.InvalidMemtableFactoryMethod.class.getName(), + "ALTER TABLE %s" + + " WITH memtable = 'test_invalid_factory_method';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Could not create memtable factory for class " + CreateTest.InvalidMemtableFactoryField.class.getName(), + "ALTER TABLE %s" + + " WITH memtable = 'test_invalid_factory_field';"); + + assertAlterTableThrowsException(ConfigurationException.class, + "Memtable configuration \"unknown\" not found.", + "ALTER TABLE %s" + + " WITH memtable = 'unknown';"); + } + + void assertSchemaOption(String option, Object expected) throws Throwable + { + assertRows(execute(format("SELECT " + option + " FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspaceTables.TABLES), + KEYSPACE, + currentTable()), + row(expected)); + } + + private void testMemtableConfig(String memtableConfig, Memtable.Factory factoryInstance, Class memtableClass) throws Throwable + { + alterTable("ALTER TABLE %s" + + " WITH memtable = '" + memtableConfig + "';"); + assertSame(factoryInstance, getCurrentColumnFamilyStore().metadata().params.memtable.factory()); + Assert.assertTrue(memtableClass.isInstance(getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable())); + assertSchemaOption("memtable", MemtableParams.DEFAULT.configurationKey().equals(memtableConfig) ? null : memtableConfig); + } + @Test public void testAlterTableWithCompression() throws Throwable { createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'chunk_length_in_kb' : 64 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "64", "class", "org.apache.cassandra.io.compress.LZ4Compressor")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 2 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "2.0"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "2.0")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 1 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.0"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor", "min_compress_ratio", "1.0")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'LZ4Compressor', 'min_compress_ratio' : 0 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")); alterTable("ALTER TABLE %s WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };"); alterTable("ALTER TABLE %s WITH compression = { 'enabled' : 'false'};"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("enabled", "false"))); + assertSchemaOption("compression", map("enabled", "false")); assertAlterTableThrowsException(ConfigurationException.class, "Missing sub-option 'class' for the 'compression' option.", diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java index 0773c97531..b72fd81ed7 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/CreateTest.java @@ -21,14 +21,19 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; +import java.util.Map; import java.util.UUID; +import org.junit.Assert; import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.Duration; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.SkipListMemtable; +import org.apache.cassandra.db.memtable.TestMemtable; import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -38,6 +43,7 @@ import org.apache.cassandra.locator.AbstractEndpointSnitch; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspaceTables; @@ -53,6 +59,7 @@ import static org.apache.cassandra.cql3.Duration.NANOS_PER_MILLI; import static org.apache.cassandra.cql3.Duration.NANOS_PER_MINUTE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -569,97 +576,125 @@ public class CreateTest extends CQLTester assertInvalidSyntaxMessage("no viable alternative at input 'WITH'", stmt); } + public static class InvalidMemtableFactoryMethod + { + @SuppressWarnings("unused") + public static String factory(Map options) + { + return "invalid"; + } + } + + public static class InvalidMemtableFactoryField + { + @SuppressWarnings("unused") + public static String FACTORY = "invalid"; + } + + @Test + public void testCreateTableWithMemtable() throws Throwable + { + createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"); + assertSame(MemtableParams.DEFAULT.factory(), getCurrentColumnFamilyStore().metadata().params.memtable.factory()); + + assertSchemaOption("memtable", null); + + testMemtableConfig("skiplist", SkipListMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("skiplist_remapped", SkipListMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("test_fullname", TestMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("test_shortname", SkipListMemtable.FACTORY, SkipListMemtable.class); + testMemtableConfig("default", MemtableParams.DEFAULT.factory(), SkipListMemtable.class); + + assertThrowsConfigurationException("The 'class_name' option must be specified.", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_empty_class';"); + + assertThrowsConfigurationException("The 'class_name' option must be specified.", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_missing_class';"); + + assertThrowsConfigurationException("Memtable class org.apache.cassandra.db.memtable.SkipListMemtable does not accept any futher parameters, but {invalid=throw} were given.", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_invalid_param';"); + + assertThrowsConfigurationException("Could not create memtable factory for class NotExisting", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_unknown_class';"); + + assertThrowsConfigurationException("Memtable class org.apache.cassandra.db.memtable.TestMemtable does not accept any futher parameters, but {invalid=throw} were given.", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_invalid_extra_param';"); + + assertThrowsConfigurationException("Could not create memtable factory for class " + InvalidMemtableFactoryMethod.class.getName(), + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_invalid_factory_method';"); + + assertThrowsConfigurationException("Could not create memtable factory for class " + InvalidMemtableFactoryField.class.getName(), + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'test_invalid_factory_field';"); + + assertThrowsConfigurationException("Memtable configuration \"unknown\" not found.", + "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = 'unknown';"); + } + + private void testMemtableConfig(String memtableConfig, Memtable.Factory factoryInstance, Class memtableClass) throws Throwable + { + createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + + " WITH memtable = '" + memtableConfig + "';"); + assertSame(factoryInstance, getCurrentColumnFamilyStore().metadata().params.memtable.factory()); + Assert.assertTrue(memtableClass.isInstance(getCurrentColumnFamilyStore().getTracker().getView().getCurrentMemtable())); + + assertSchemaOption("memtable", MemtableParams.DEFAULT.configurationKey().equals(memtableConfig) ? null : memtableConfig); + } + + void assertSchemaOption(String option, Object expected) throws Throwable + { + assertRows(execute(format("SELECT " + option + " FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", + SchemaConstants.SCHEMA_KEYSPACE_NAME, + SchemaKeyspaceTables.TABLES), + KEYSPACE, + currentTable()), + row(expected)); + } + @Test public void testCreateTableWithCompression() throws Throwable { createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.LZ4Compressor")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'class' : 'SnappyCompressor', 'chunk_length_in_kb' : 32, 'enabled' : true };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'chunk_length_kb' : 32 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "32", "class", "org.apache.cassandra.io.compress.SnappyCompressor")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 2 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "2.0"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "2.0")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 1 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.0"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor", "min_compress_ratio", "1.0")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : 'SnappyCompressor', 'min_compress_ratio' : 0 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor"))); + assertSchemaOption("compression", map("chunk_length_in_kb", "16", "class", "org.apache.cassandra.io.compress.SnappyCompressor")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'sstable_compression' : '', 'chunk_length_kb' : 32 };"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("enabled", "false"))); + assertSchemaOption("compression", map("enabled", "false")); createTable("CREATE TABLE %s (a text, b int, c int, primary key (a, b))" + " WITH compression = { 'enabled' : 'false'};"); - - assertRows(execute(format("SELECT compression FROM %s.%s WHERE keyspace_name = ? and table_name = ?;", - SchemaConstants.SCHEMA_KEYSPACE_NAME, - SchemaKeyspaceTables.TABLES), - KEYSPACE, - currentTable()), - row(map("enabled", "false"))); + assertSchemaOption("compression", map("enabled", "false")); assertThrowsConfigurationException("Missing sub-option 'class' for the 'compression' option.", "CREATE TABLE %s (a text, b int, c int, primary key (a, b))" diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java index c97e5106fd..aaaecd0da9 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.cql3.validation.operations; import java.io.IOException; +import org.apache.cassandra.Util; import org.apache.cassandra.io.util.File; import org.junit.After; import org.junit.Before; @@ -269,7 +270,7 @@ public class TTLTest extends CQLTester // Maybe Flush Keyspace ks = Keyspace.open(keyspace()); if (flush) - FBUtilities.waitOnFutures(ks.flush()); + Util.flush(ks); // Verify data verifyData(simple); diff --git a/test/unit/org/apache/cassandra/db/CleanupTest.java b/test/unit/org/apache/cassandra/db/CleanupTest.java index 0c2a969a88..a0eaa5fa60 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTest.java @@ -284,7 +284,7 @@ public class CleanupTest .add("val", VALUE) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Set beforeFirstCleanup = Sets.newHashSet(cfs.getLiveSSTables()); @@ -433,7 +433,7 @@ public class CleanupTest .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } protected List getMaxTimestampList(ColumnFamilyStore cfs) diff --git a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java index 18431d3b46..d611bfa71d 100644 --- a/test/unit/org/apache/cassandra/db/CleanupTransientTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTransientTest.java @@ -181,7 +181,7 @@ public class CleanupTransientTest .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } protected List getMaxTimestampList(ColumnFamilyStore cfs) diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java index 21417ed787..a215d4c0b3 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyMetricTest.java @@ -70,7 +70,7 @@ public class ColumnFamilyMetricTest { applyMutation(cfs.metadata(), String.valueOf(j), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros()); } - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstables = cfs.getLiveSSTables(); long size = 0; for (SSTableReader reader : sstables) @@ -152,7 +152,7 @@ public class ColumnFamilyMetricTest applyMutation(store.metadata(), "1", bytes(1), FBUtilities.timestampMicros()); // Flushing first SSTable - store.forceBlockingFlush(); + Util.flush(store); long[] estimatedColumnCountHistogram = store.metric.estimatedColumnCountHistogram.getValue(); assertNumberOfNonZeroValue(estimatedColumnCountHistogram, 1); @@ -165,7 +165,7 @@ public class ColumnFamilyMetricTest applyMutation(store.metadata(), "2", bytes(2), FBUtilities.timestampMicros()); // Flushing second SSTable - store.forceBlockingFlush(); + Util.flush(store); estimatedColumnCountHistogram = store.metric.estimatedColumnCountHistogram.getValue(); assertNumberOfNonZeroValue(estimatedColumnCountHistogram, 1); diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index b512a5d284..633328d4c8 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -111,14 +111,14 @@ public class ColumnFamilyStoreTest .add("val", "asdf") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, "key1") .clustering("Column1") .add("val", "asdf") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts Util.getAll(Util.cmd(cfs, "key1").includeRow("c1").build()); @@ -188,7 +188,7 @@ public class ColumnFamilyStoreTest assertRangeCount(cfs, col, val, 2); // flush. - cfs.forceBlockingFlush(); + Util.flush(cfs); // insert, don't flush new RowUpdateBuilder(cfs.metadata(), 1, "key3").clustering("Column1").add("val", "val1").build().applyUnsafe(); @@ -203,7 +203,7 @@ public class ColumnFamilyStoreTest assertRangeCount(cfs, col, val, 2); // flush - cfs.forceBlockingFlush(); + Util.flush(cfs); // re-verify delete. // first breakage is right here because of CASSANDRA-1837. assertRangeCount(cfs, col, val, 2); @@ -221,7 +221,7 @@ public class ColumnFamilyStoreTest assertRangeCount(cfs, col, val, 4); // and it remains so after flush. (this wasn't failing before, but it's good to check.) - cfs.forceBlockingFlush(); + Util.flush(cfs); assertRangeCount(cfs, col, val, 4); } @@ -273,7 +273,7 @@ public class ColumnFamilyStoreTest .add("val", "asdf") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // snapshot cfs.snapshot("basic", null, false, false); @@ -305,9 +305,9 @@ public class ColumnFamilyStoreTest { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (SSTableReader liveSSTable : cfs.getLiveSSTables()) { @@ -449,7 +449,7 @@ public class ColumnFamilyStoreTest public void reTest(ColumnFamilyStore cfs, Runnable verify) throws Exception { verify.run(); - cfs.forceBlockingFlush(); + Util.flush(cfs); verify.run(); } @@ -488,10 +488,10 @@ public class ColumnFamilyStoreTest .add("birthdate", 1L) .add("notbirthdate", 2L); new Mutation(builder.build()).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); String snapshotName = "newSnapshot"; - cfs.snapshotWithoutFlush(snapshotName); + cfs.snapshotWithoutMemtable(snapshotName); File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName); SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(snapshotManifestFile); @@ -539,12 +539,12 @@ public class ColumnFamilyStoreTest if (cfs.name.equals(CF_INDEX1)) { new RowUpdateBuilder(cfs.metadata(), 2, "key").add("birthdate", 1L).add("notbirthdate", 2L).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } else { new RowUpdateBuilder(cfs.metadata(), 2, "key").clustering("name").add("val", "2").build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -589,7 +589,7 @@ public class ColumnFamilyStoreTest ColumnFamilyStore.scrubDataDirectories(cfs.metadata()); new RowUpdateBuilder(cfs.metadata(), 2, "key").clustering("name").add("val", "2").build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Nuke the metadata and reload that sstable Collection ssTables = cfs.getLiveSSTables(); diff --git a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java index d2f599d836..34a2b83a12 100644 --- a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java +++ b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java @@ -75,7 +75,7 @@ public class DeletePartitionTest assertTrue(r.getCell(column).value().equals(ByteBufferUtil.bytes("asdf"))); if (flushBeforeRemove) - store.forceBlockingFlush(); + Util.flush(store); // delete the partition new Mutation.PartitionUpdateCollector(KEYSPACE1, key) @@ -84,7 +84,7 @@ public class DeletePartitionTest .applyUnsafe(); if (flushAfterRemove) - store.forceBlockingFlush(); + Util.flush(store); // validate removal ImmutableBTreePartition partitionUnfiltered = Util.getOnlyPartitionUnfiltered(Util.cmd(store, key).build()); diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java index 62e92870a8..53d998a954 100644 --- a/test/unit/org/apache/cassandra/db/ImportTest.java +++ b/test/unit/org/apache/cassandra/db/ImportTest.java @@ -86,8 +86,7 @@ public class ImportTest extends CQLTester { execute("insert into %s (id, d) values (?, ?)", i, i); } - - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -112,14 +111,14 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); File backupdir = moveToBackupDir(sstables); for (int i = 10; i < 20; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -143,7 +142,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); sstables.forEach(s -> s.selfRef().release()); @@ -158,7 +157,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); for (SSTableReader sstable : sstables) @@ -195,7 +194,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); for (SSTableReader sstable : sstables) @@ -277,7 +276,7 @@ public class ImportTest extends CQLTester for (int i = 0; i < 10; i++) { execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } Set toMove = getCurrentColumnFamilyStore().getLiveSSTables(); @@ -306,11 +305,11 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); SSTableReader sstableToCorrupt = getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i + 10, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -327,7 +326,7 @@ public class ImportTest extends CQLTester // now move a correct sstable to another directory to make sure that directory gets properly imported for (int i = 100; i < 130; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set correctSSTables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -403,7 +402,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 1000; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -448,7 +447,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 1000; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); @@ -484,7 +483,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); CacheService.instance.setRowCacheCapacityInMB(1); Set keysToInvalidate = new HashSet<>(); @@ -505,7 +504,7 @@ public class ImportTest extends CQLTester for (int i = 10; i < 20; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set allCachedKeys = new HashSet<>(); @@ -552,7 +551,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); CacheService.instance.setRowCacheCapacityInMB(1); getCurrentColumnFamilyStore().clearUnsafe(); @@ -569,7 +568,7 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int) WITH caching = { 'keys': 'NONE', 'rows_per_partition': 'ALL' }"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); sstables.forEach(s -> s.selfRef().release()); @@ -584,10 +583,10 @@ public class ImportTest extends CQLTester for (int i = 10; i < 20; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); for (int i = 20; i < 30; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set expectedFiles = new HashSet<>(getCurrentColumnFamilyStore().getLiveSSTables()); @@ -633,14 +632,14 @@ public class ImportTest extends CQLTester createTable("create table %s (id int primary key, d int)"); for (int i = 0; i < 10; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); File backupdir = moveToBackupDir(sstables); for (int i = 10; i < 20; i++) execute("insert into %s (id, d) values (?, ?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); sstables = getCurrentColumnFamilyStore().getLiveSSTables(); getCurrentColumnFamilyStore().clearUnsafe(); diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java index f41460adeb..f548f90b0f 100644 --- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java +++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java @@ -132,7 +132,7 @@ public class KeyCacheTest // insert data and force to disk SchemaLoader.insertData(KEYSPACE1, cf, 0, 100); - store.forceBlockingFlush(); + Util.flush(store); // populate the cache readData(KEYSPACE1, cf, 0, 100); @@ -232,7 +232,7 @@ public class KeyCacheTest // insert data and force to disk SchemaLoader.insertData(KEYSPACE1, cf, 0, 100); - store.forceBlockingFlush(); + Util.flush(store); Collection firstFlushTables = ImmutableList.copyOf(store.getLiveSSTables()); @@ -242,7 +242,7 @@ public class KeyCacheTest // insert some new data and force to disk SchemaLoader.insertData(KEYSPACE1, cf, 100, 50); - store.forceBlockingFlush(); + Util.flush(store); // check that it's fine readData(KEYSPACE1, cf, 100, 50); @@ -303,7 +303,7 @@ public class KeyCacheTest new RowUpdateBuilder(cfs.metadata(), 0, "key2").clustering("2").build().applyUnsafe(); // to make sure we have SSTable - cfs.forceBlockingFlush(); + Util.flush(cfs); // reads to cache key position Util.getAll(Util.cmd(cfs, "key1").build()); @@ -426,7 +426,7 @@ public class KeyCacheTest // insert data and force to disk SchemaLoader.insertData(keyspace, cf, 0, numberOfRows); - store.forceBlockingFlush(); + Util.flush(store); } for(Pair entry : tables) { diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java index b0c8011b41..2445fe3aa0 100644 --- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java @@ -85,7 +85,7 @@ public class KeyspaceTest extends CQLTester Util.assertEmpty(Util.cmd(cfs, "0").columns("c").includeRow(1).build()); if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -120,7 +120,7 @@ public class KeyspaceTest extends CQLTester } if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -137,7 +137,7 @@ public class KeyspaceTest extends CQLTester for (String key : new String[]{"0", "2"}) Util.assertEmpty(Util.cmd(cfs, key).build()); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (String key : new String[]{"0", "2"}) Util.assertEmpty(Util.cmd(cfs, key).build()); @@ -209,7 +209,7 @@ public class KeyspaceTest extends CQLTester assertRowsInSlice(cfs, "0", 288, 299, 12, true, prefix); if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -222,7 +222,7 @@ public class KeyspaceTest extends CQLTester for (int i = 0; i < 10; i++) execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (int i = 10; i < 20; i++) { @@ -336,7 +336,7 @@ public class KeyspaceTest extends CQLTester assertRowsInResult(cfs, command); if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -359,7 +359,7 @@ public class KeyspaceTest extends CQLTester assertRowsInResult(cfs, command, 1); if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -372,7 +372,7 @@ public class KeyspaceTest extends CQLTester for (int i = 1; i < 7; i++) execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); // overwrite three rows with -1 for (int i = 1; i < 4; i++) @@ -384,7 +384,7 @@ public class KeyspaceTest extends CQLTester assertRowsInResult(cfs, command, -1, -1, 4); if (round == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -397,7 +397,7 @@ public class KeyspaceTest extends CQLTester for (int i = 1000; i < 2000; i++) execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", "0", i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); validateSliceLarge(cfs); @@ -439,7 +439,7 @@ public class KeyspaceTest extends CQLTester for (int i = 1000 + (j*100); i < 1000 + ((j+1)*100); i++) execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", "0", i, i, (long)i); - cfs.forceBlockingFlush(); + Util.flush(cfs); } ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); diff --git a/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java b/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java index d69025372b..5ecd5b0228 100644 --- a/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/MultiKeyspaceTest.java @@ -20,7 +20,9 @@ package org.apache.cassandra.db; * */ +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; + import org.junit.Test; @@ -38,8 +40,8 @@ public class MultiKeyspaceTest extends CQLTester execute("INSERT INTO multikstest1.standard1 (a, b) VALUES (0, 0)"); execute("INSERT INTO multikstest2.standard1 (a, b) VALUES (0, 0)"); - Keyspace.open("multikstest1").flush(); - Keyspace.open("multikstest2").flush(); + Util.flushKeyspace("multikstest1"); + Util.flushKeyspace("multikstest2"); assertRows(execute("SELECT * FROM multikstest1.standard1"), row(0, 0)); diff --git a/test/unit/org/apache/cassandra/db/NameSortTest.java b/test/unit/org/apache/cassandra/db/NameSortTest.java index 0d7b09c8fc..2fdd735300 100644 --- a/test/unit/org/apache/cassandra/db/NameSortTest.java +++ b/test/unit/org/apache/cassandra/db/NameSortTest.java @@ -84,7 +84,7 @@ public class NameSortTest rub.build().applyUnsafe(); } validateNameSort(cfs); - keyspace.getColumnFamilyStore("Standard1").forceBlockingFlush(); + keyspace.getColumnFamilyStore("Standard1").forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); validateNameSort(cfs); } diff --git a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java index 986a1251d5..20387488ba 100644 --- a/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java +++ b/test/unit/org/apache/cassandra/db/PartitionRangeReadTest.java @@ -100,14 +100,14 @@ public class PartitionRangeReadTest .add("val", "val1") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, "k1") .clustering(new BigInteger(new byte[]{0, 0, 1})) .add("val", "val2") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // fetch by the first column name; we should get the second version of the column value Row row = Util.getOnlyRow(Util.cmd(cfs, "k1").includeRow(new BigInteger(new byte[]{1})).build()); @@ -157,7 +157,7 @@ public class PartitionRangeReadTest builder.build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); ColumnMetadata cDef = cfs.metadata().getColumn(ByteBufferUtil.bytes("val")); diff --git a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java index 9e8f913501..0ce0c14ac3 100644 --- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java +++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java @@ -85,7 +85,7 @@ public class RangeTombstoneTest for (int i = 0; i < 40; i += 2) builder.newRow(i).add("val", i); builder.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 22).build().applyUnsafe(); @@ -235,7 +235,7 @@ public class RangeTombstoneTest int nowInSec = FBUtilities.nowInSeconds(); new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); @@ -257,7 +257,7 @@ public class RangeTombstoneTest key = "rt_times2"; int nowInSec = FBUtilities.nowInSeconds(); new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE); @@ -276,7 +276,7 @@ public class RangeTombstoneTest int nowInSec = FBUtilities.nowInSeconds(); new RowUpdateBuilder(cfs.metadata(), nowInSec, 1000L, key).addRangeTombstone(1, 2).build().apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 1000, 1000, nowInSec); @@ -298,9 +298,9 @@ public class RangeTombstoneTest key = "rt_times2"; int nowInSec = FBUtilities.nowInSeconds(); new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1000, nowInSec)).apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE); cfs.forceMajorCompaction(); @@ -328,10 +328,10 @@ public class RangeTombstoneTest for (int i = 10; i < 20; i ++) builder.newRow(i).add("val", i); builder.apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 11).build().apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Thread.sleep(5); cfs.forceMajorCompaction(); @@ -350,10 +350,10 @@ public class RangeTombstoneTest for (int i = 0; i < 40; i += 2) builder.newRow(i).add("val", i); builder.apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 1, 1)).apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Thread.sleep(5); cfs.forceMajorCompaction(); } @@ -370,13 +370,13 @@ public class RangeTombstoneTest for (int i = 10; i < 20; i ++) builder.newRow(i).add("val", i); builder.apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new Mutation(PartitionUpdate.fullPartitionDelete(cfs.metadata(), Util.dk(key), 0, 0)).apply(); UpdateBuilder.create(cfs.metadata(), key).withTimestamp(1).newRow(5).add("val", 5).apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Thread.sleep(5); cfs.forceMajorCompaction(); assertEquals(1, Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).rowCount()); @@ -396,16 +396,16 @@ public class RangeTombstoneTest for (int i = 0; i < 20; i++) builder.newRow(i).add("val", i); builder.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(5, 15).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(5, 10).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 2, key).addRangeTombstone(5, 8).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Partition partition = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()); int nowInSec = FBUtilities.nowInSeconds(); @@ -447,11 +447,11 @@ public class RangeTombstoneTest String key = "k3"; UpdateBuilder.create(cfs.metadata(), key).withTimestamp(0).newRow(2).add("val", 2).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(0, 10).build().applyUnsafe(); UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Get the last value of the row FilteredPartition partition = Util.getOnlyPartition(Util.cmd(cfs, key).build()); @@ -508,10 +508,10 @@ public class RangeTombstoneTest for (int i = 0; i < 10; i++) builder.newRow(i).add("val", i); builder.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, key).addRangeTombstone(0, 7).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(10, index.rowsInserted.size()); @@ -538,10 +538,10 @@ public class RangeTombstoneTest for (int i = 0; i < 10; i += 2) builder.newRow(i).add("val", i); builder.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, key).addRangeTombstone(0, 7).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // there should be 2 sstables assertEquals(2, cfs.getLiveSSTables().size()); @@ -614,7 +614,7 @@ public class RangeTombstoneTest // now re-insert that column UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2).newRow(1).add("val", 1).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // We should have 1 insert and 1 update to the indexed "1" column // CASSANDRA-6640 changed index update to just update, not insert then delete diff --git a/test/unit/org/apache/cassandra/db/ReadCommandTest.java b/test/unit/org/apache/cassandra/db/ReadCommandTest.java index 8be5608626..43a7952175 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandTest.java @@ -220,7 +220,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2")) .clustering("Column1") @@ -248,7 +248,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key")) .clustering("dd") @@ -279,7 +279,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key")) .clustering("dd") @@ -358,7 +358,7 @@ public class ReadCommandTest commands.add(SinglePartitionReadCommand.create(cfs.metadata(), nowInSeconds, columnFilter, rowFilter, DataLimits.NONE, Util.dk(data[1]), sliceFilter)); } - cfs.forceBlockingFlush(); + Util.flush(cfs); ReadQuery query = SinglePartitionReadCommand.Group.create(commands, DataLimits.NONE); @@ -528,7 +528,7 @@ public class ReadCommandTest DataLimits.NONE, Util.dk(data[1]), sliceFilter)); } - cfs.forceBlockingFlush(); + Util.flush(cfs); ReadQuery query = SinglePartitionReadCommand.Group.create(commands, DataLimits.NONE); @@ -604,7 +604,7 @@ public class ReadCommandTest DataLimits.NONE, Util.dk(data[1]), sliceFilter)); } - cfs.forceBlockingFlush(); + Util.flush(cfs); ReadQuery query = SinglePartitionReadCommand.Group.create(commands, DataLimits.NONE); @@ -663,7 +663,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, ByteBufferUtil.bytes("key")) .clustering("dd") @@ -671,7 +671,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstables = new ArrayList<>(cfs.getLiveSSTables()); assertEquals(2, sstables.size()); sstables.sort(SSTableReader.maxTimestampDescending); @@ -710,7 +710,7 @@ public class ReadCommandTest .addLegacyCounterCell("c", 0L) .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); // execute a read and capture the digest @@ -727,7 +727,7 @@ public class ReadCommandTest .addLegacyCounterCell("c", 1L) .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); ByteBuffer digestWithLegacyCounter1 = performReadAndVerifyRepairedInfo(readCommand, 1, 1, true); @@ -742,7 +742,7 @@ public class ReadCommandTest .add("c", 1L) .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); ByteBuffer digestWithCounterCell = performReadAndVerifyRepairedInfo(readCommand, 1, 1, true); @@ -785,7 +785,7 @@ public class ReadCommandTest // Partition with 2 rows, one fully deleted new RowUpdateBuilder(cfs.metadata.get(), 0, keys[3]).clustering("bb").add("a", ByteBufferUtil.bytes("a")).delete("b").build().apply(); RowUpdateBuilder.deleteRow(cfs.metadata(), 0, keys[3], "cc").apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); Map digestsWithTombstones = new HashMap<>(); @@ -853,7 +853,7 @@ public class ReadCommandTest .build()); // Insert and repair insert(cfs, IntStream.range(0, 10), () -> IntStream.range(0, 10)); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); // Insert and leave unrepaired insert(cfs, IntStream.range(0, 10), () -> IntStream.range(10, 20)); @@ -983,7 +983,7 @@ public class ReadCommandTest // Partition with a fully deleted static row and a single, fully deleted regular row RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key")).apply(); RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key"), "cc").apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); try (ReadExecutionController controller = command.executionController(true)) @@ -1033,12 +1033,12 @@ public class ReadCommandTest // Live partition in a repaired sstable, so included in the digest calculation new RowUpdateBuilder(cfs.metadata.get(), 0, ByteBufferUtil.bytes("key-0")).clustering("cc").add("a", ByteBufferUtil.bytes("a")).build().apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); // Fully deleted partition (static and regular rows) in an unrepaired sstable, so not included in the intial digest RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key-1")).apply(); RowUpdateBuilder.deleteRow(cfs.metadata(), 0, ByteBufferUtil.bytes("key-1"), "cc").apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); ByteBuffer digestWithoutPurgedPartition = null; @@ -1081,11 +1081,11 @@ public class ReadCommandTest DecoratedKey key = Util.dk("key"); RowUpdateBuilder.deleteRow(cfs.metadata(), 0, key).apply(); RowUpdateBuilder.deleteRow(cfs.metadata(), 0, key, "cc").apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.getLiveSSTables().forEach(sstable -> mutateRepaired(cfs, sstable, 111, null)); new RowUpdateBuilder(cfs.metadata(), 1, key).clustering("cc").add("a", ByteBufferUtil.bytes("a")).build().apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); int nowInSec = FBUtilities.nowInSeconds() + 10; ReadCommand cmd = Util.cmd(cfs, key).withNowInSeconds(nowInSec).build(); @@ -1126,7 +1126,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build(); assertTrue(cfs.isRowCacheEnabled()); @@ -1214,7 +1214,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, ByteBufferUtil.bytes("key")) .clustering("dd") @@ -1222,7 +1222,7 @@ public class ReadCommandTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstables = new ArrayList<>(cfs.getLiveSSTables()); assertEquals(2, sstables.size()); sstables.forEach(sstable -> assertFalse(sstable.isRepaired() || sstable.isPendingRepair())); @@ -1279,7 +1279,7 @@ public class ReadCommandTest assertEquals(EMPTY_BYTE_BUFFER, digest); // now flush so we have an unrepaired table with the deletion and repeat the check - cfs.forceBlockingFlush(); + Util.flush(cfs); digest = performReadAndVerifyRepairedInfo(readCommand, 0, rowsPerPartition, false); assertEquals(EMPTY_BYTE_BUFFER, digest); } diff --git a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java index d3954394f0..acca6edc4a 100644 --- a/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java +++ b/test/unit/org/apache/cassandra/db/RecoveryManagerFlushedTest.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.io.compress.ZstdCompressor; @@ -101,8 +102,8 @@ public class RecoveryManagerFlushedTest public void testWithFlush() throws Exception { // Flush everything that may be in the commit log now to start fresh - FBUtilities.waitOnFutures(Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).flush()); - FBUtilities.waitOnFutures(Keyspace.open(SchemaConstants.SCHEMA_KEYSPACE_NAME).flush()); + Util.flushKeyspace(SchemaConstants.SYSTEM_KEYSPACE_NAME); + Util.flushKeyspace(SchemaConstants.SCHEMA_KEYSPACE_NAME); CompactionManager.instance.disableAutoCompaction(); @@ -119,7 +120,7 @@ public class RecoveryManagerFlushedTest Keyspace keyspace1 = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace1.getColumnFamilyStore("Standard1"); logger.debug("forcing flush"); - cfs.forceBlockingFlush(); + Util.flush(cfs); logger.debug("begin manual replay"); // replay the commit log (nothing on Standard1 should be replayed since everything was flushed, so only the row on Standard2 diff --git a/test/unit/org/apache/cassandra/db/RemoveCellTest.java b/test/unit/org/apache/cassandra/db/RemoveCellTest.java index 01fe2551f4..1c28d1c820 100644 --- a/test/unit/org/apache/cassandra/db/RemoveCellTest.java +++ b/test/unit/org/apache/cassandra/db/RemoveCellTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; public class RemoveCellTest extends CQLTester @@ -30,7 +31,7 @@ public class RemoveCellTest extends CQLTester String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))"); ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 0, 0, 0L); - cfs.forceBlockingFlush(); + Util.flush(cfs); execute("DELETE c FROM %s USING TIMESTAMP ? WHERE a = ? AND b = ?", 1L, 0, 0); assertRows(execute("SELECT * FROM %s WHERE a = ? AND b = ?", 0, 0), row(0, 0, null)); assertRows(execute("SELECT c FROM %s WHERE a = ? AND b = ?", 0, 0), row(new Object[]{null})); diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java index c44cfb2c8b..0bf8c5df08 100644 --- a/test/unit/org/apache/cassandra/db/RowCacheTest.java +++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java @@ -495,7 +495,7 @@ public class RowCacheTest SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, 0, 100); //force flush for confidence that SSTables exists - cachedStore.forceBlockingFlush(); + Util.flush(cachedStore); ((ClearableHistogram)cachedStore.metric.sstablesPerReadHistogram.cf).clear(); diff --git a/test/unit/org/apache/cassandra/db/RowIterationTest.java b/test/unit/org/apache/cassandra/db/RowIterationTest.java index b0cd4fc1ca..894744a34c 100644 --- a/test/unit/org/apache/cassandra/db/RowIterationTest.java +++ b/test/unit/org/apache/cassandra/db/RowIterationTest.java @@ -36,7 +36,7 @@ public class RowIterationTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); for (int i = 0; i < 10; i++) execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TIMESTAMP ?", i, 0, i, i, (long)i); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(10, execute("SELECT * FROM %s").size()); } @@ -49,7 +49,7 @@ public class RowIterationTest extends CQLTester execute("INSERT INTO %s (a, b) VALUES (?, ?) USING TIMESTAMP ?", 0, 0, 0L); execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ?", 0L, 0); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Delete row in second sstable with higher timestamp execute("INSERT INTO %s (a, b) VALUES (?, ?) USING TIMESTAMP ?", 0, 0, 1L); @@ -57,7 +57,7 @@ public class RowIterationTest extends CQLTester int localDeletionTime = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).partitionLevelDeletion().localDeletionTime(); - cfs.forceBlockingFlush(); + Util.flush(cfs); DeletionTime dt = Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).partitionLevelDeletion(); assertEquals(1L, dt.markedForDeleteAt()); @@ -72,7 +72,7 @@ public class RowIterationTest extends CQLTester // Delete a row in first sstable execute("DELETE FROM %s USING TIMESTAMP ? WHERE a = ?", 0L, 0); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs).build()).isEmpty()); } diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index eb35adf7a4..63ef861101 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -303,6 +303,7 @@ public class SchemaCQLHelperTest extends CQLTester " AND comment = 'comment'\n" + " AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4', 'sstable_size_in_mb': '1'}\n" + " AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor', 'min_compress_ratio': '2.0'}\n" + + " AND memtable = 'default'\n" + " AND crc_check_chance = 0.3\n" + " AND default_time_to_live = 4\n" + " AND extensions = {'ext1': 0x76616c31}\n" + diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index 5d20af52e3..1e13039d64 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -483,7 +483,7 @@ public class ScrubTest new Mutation(update).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long ... values) @@ -507,7 +507,7 @@ public class ScrubTest new Mutation(builder.build()).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } protected static void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException @@ -520,7 +520,7 @@ public class ScrubTest new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } @Test @@ -531,14 +531,14 @@ public class ScrubTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_static_columns"); QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_static_columns (a, b, c, d) VALUES (123, c3db07e8-b602-11e3-bc6b-e0b9a54a6d93, true, 'foobar')", ksName)); - cfs.forceBlockingFlush(); + Util.flush(cfs); CompactionManager.instance.performScrub(cfs, false, true, 2); QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_scrub_validation (a text primary key, b int)", ksName), ConsistencyLevel.ONE); ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation"); new Mutation(UpdateBuilder.create(cfs2.metadata(), "key").newRow().add("b", Int32Type.instance.decompose(1)).build()).apply(); - cfs2.forceBlockingFlush(); + Util.flush(cfs2); CompactionManager.instance.performScrub(cfs2, false, false, 2); } @@ -557,7 +557,7 @@ public class ScrubTest QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'a', 'foo')", ksName)); QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')", ksName)); QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')", ksName)); - cfs.forceBlockingFlush(); + Util.flush(cfs); CompactionManager.instance.performScrub(cfs, true, true, 2); // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away" diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java index 544196f4e9..14137f13dc 100644 --- a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java +++ b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java @@ -300,7 +300,7 @@ public class SecondaryIndexTest new RowUpdateBuilder(cfs.metadata(), 1, "k1").noRowMarker().add("birthdate", 1L).build().applyUnsafe(); // force a flush, so our index isn't being read from a memtable - keyspace.getColumnFamilyStore(WITH_KEYS_INDEX).forceBlockingFlush(); + Util.flushTable(keyspace, WITH_KEYS_INDEX); // now apply another update, but force the index update to be skipped keyspace.apply(new RowUpdateBuilder(cfs.metadata(), 2, "k1").noRowMarker().add("birthdate", 2L).build(), @@ -356,7 +356,7 @@ public class SecondaryIndexTest assertIndexedOne(cfs, col, 10l); // force a flush and retry the query, so our index isn't being read from a memtable - keyspace.getColumnFamilyStore(cfName).forceBlockingFlush(); + Util.flushTable(keyspace, cfName); assertIndexedOne(cfs, col, 10l); // now apply another update, but force the index update to be skipped @@ -522,7 +522,7 @@ public class SecondaryIndexTest new RowUpdateBuilder(cfs.metadata(), 0, "k" + i).noRowMarker().add("birthdate", 1l).build().applyUnsafe(); assertIndexedCount(cfs, ByteBufferUtil.bytes("birthdate"), 1l, 10); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertIndexedCount(cfs, ByteBufferUtil.bytes("birthdate"), 1l, 10); } @@ -537,7 +537,7 @@ public class SecondaryIndexTest new RowUpdateBuilder(cfs.metadata(), 0, "k3").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe(); new RowUpdateBuilder(cfs.metadata(), 0, "k4").clustering("c").add("birthdate", 1L).add("notbirthdate", 3L).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); ReadCommand rc = Util.cmd(cfs) .fromKeyIncl("k1") .toKeyIncl("k3") diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java index 1c891ec2b2..47e866c0dd 100644 --- a/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionReadCommandCQLTest.java @@ -31,10 +31,10 @@ public class SinglePartitionReadCommandCQLTest extends CQLTester { createTable("CREATE TABLE %s (bucket_id TEXT,name TEXT,data TEXT,PRIMARY KEY (bucket_id, name))"); execute("insert into %s (bucket_id, name, data) values ('8772618c9009cf8f5a5e0c18', 'test', 'hello')"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); execute("insert into %s (bucket_id, name, data) values ('8772618c9009cf8f5a5e0c19', 'test2', 'hello');"); execute("delete from %s where bucket_id = '8772618c9009cf8f5a5e0c18'"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); UntypedResultSet res = execute("select * from %s where bucket_id = '8772618c9009cf8f5a5e0c18' and name = 'test'"); assertTrue(res.isEmpty()); } diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java index 692b0c5eab..ab790dca50 100644 --- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java +++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java @@ -180,7 +180,7 @@ public class SinglePartitionSliceCommandTest ck1)); if (flush) - Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_SCLICES).forceBlockingFlush(); + Util.flushTable(KEYSPACE, TABLE_SCLICES); AbstractClusteringIndexFilter clusteringFilter = createClusteringFilter(uniqueCk1, uniqueCk2, isSlice); ReadCommand cmd = SinglePartitionReadCommand.create(CFM_SLICES, @@ -279,7 +279,7 @@ public class SinglePartitionSliceCommandTest } // check (de)serialized iterator for sstable static cell - Schema.instance.getColumnFamilyStoreInstance(metadata.id).forceBlockingFlush(); + Schema.instance.getColumnFamilyStoreInstance(metadata.id).forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); try (ReadExecutionController executionController = cmd.executionController(); UnfilteredPartitionIterator pi = cmd.executeLocally(executionController)) { response = ReadResponse.createDataResponse(pi, cmd, executionController.getRepairedDataInfo()); @@ -318,7 +318,7 @@ public class SinglePartitionSliceCommandTest QueryProcessor.executeOnceInternal("DELETE FROM ks.test_read_rt USING TIMESTAMP 10 WHERE k=1 AND c1=1"); List memtableUnfiltereds = assertQueryReturnsSingleRT(query); - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstableUnfiltereds = assertQueryReturnsSingleRT(query); String errorMessage = String.format("Expected %s but got %s with postfix '%s'", @@ -356,17 +356,17 @@ public class SinglePartitionSliceCommandTest timestamp, "DELETE FROM ks.partition_row_deletion USING TIMESTAMP 10 WHERE k=1"); if (flush && multiSSTable) - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeOnceInternalWithNowAndTimestamp(nowInSec, timestamp, "DELETE FROM ks.partition_row_deletion USING TIMESTAMP 10 WHERE k=1 and c=1"); if (flush) - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeOnceInternal("INSERT INTO ks.partition_row_deletion(k,c,v) VALUES(1,1,1) using timestamp 11"); if (flush) { - cfs.forceBlockingFlush(); + Util.flush(cfs); try { cfs.forceMajorCompaction(); @@ -424,17 +424,17 @@ public class SinglePartitionSliceCommandTest timestamp, "DELETE FROM ks.partition_range_deletion USING TIMESTAMP 10 WHERE k=1"); if (flush && multiSSTable) - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeOnceInternalWithNowAndTimestamp(nowInSec, timestamp, "DELETE FROM ks.partition_range_deletion USING TIMESTAMP 10 WHERE k=1 and c1=1"); if (flush) - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeOnceInternal("INSERT INTO ks.partition_range_deletion(k,c1,c2,v) VALUES(1,1,1,1) using timestamp 11"); if (flush) { - cfs.forceBlockingFlush(); + Util.flush(cfs); try { cfs.forceMajorCompaction(); @@ -544,7 +544,7 @@ public class SinglePartitionSliceCommandTest QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)"); QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1"); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1"); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names @@ -560,7 +560,7 @@ public class SinglePartitionSliceCommandTest new Mutation(builder.build()).apply(); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2"); assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100 AND c1=3 AND c2=2 AND c3=2"); // clustering names diff --git a/test/unit/org/apache/cassandra/db/SnapshotTest.java b/test/unit/org/apache/cassandra/db/SnapshotTest.java index c14a6c77f1..aa726ec3e0 100644 --- a/test/unit/org/apache/cassandra/db/SnapshotTest.java +++ b/test/unit/org/apache/cassandra/db/SnapshotTest.java @@ -35,7 +35,7 @@ public class SnapshotTest extends CQLTester { createTable("create table %s (id int primary key, k int)"); execute("insert into %s (id, k) values (1,1)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables()) { File toc = new File(sstable.descriptor.filenameFor(Component.TOC)); diff --git a/test/unit/org/apache/cassandra/db/TimeSortTest.java b/test/unit/org/apache/cassandra/db/TimeSortTest.java index 8ae05ea957..b45ab9679a 100644 --- a/test/unit/org/apache/cassandra/db/TimeSortTest.java +++ b/test/unit/org/apache/cassandra/db/TimeSortTest.java @@ -36,7 +36,7 @@ public class TimeSortTest extends CQLTester ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 100, 0, 100L); - cfs.forceBlockingFlush(); + Util.flush(cfs); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", 0, 0, 1, 0L); assertRows(execute("SELECT * FROM %s WHERE a = ? AND b >= ? LIMIT 1000", 0, 10), row(0, 100, 0)); @@ -53,7 +53,7 @@ public class TimeSortTest extends CQLTester execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP ?", i, j * 2, 0, (long)j * 2); validateTimeSort(); - cfs.forceBlockingFlush(); + Util.flush(cfs); validateTimeSort(); // interleave some new data to test memtable + sstable diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java index fe739895db..a58837aa08 100644 --- a/test/unit/org/apache/cassandra/db/VerifyTest.java +++ b/test/unit/org/apache/cassandra/db/VerifyTest.java @@ -703,7 +703,7 @@ public class VerifyTest Batch bogus = Batch.createLocal(nextTimeUUID(), 0, Collections.emptyList()); BatchlogManager.store(bogus); ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("batches"); - cfs.forceBlockingFlush(); + Util.flush(cfs); for (SSTableReader sstable : cfs.getLiveSSTables()) { @@ -761,7 +761,7 @@ public class VerifyTest .apply(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } protected void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException @@ -773,7 +773,7 @@ public class VerifyTest .apply(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } protected long simpleFullChecksum(String filename) throws IOException diff --git a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java index 9040f1197c..9f9b7bb310 100644 --- a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java +++ b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java @@ -28,6 +28,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; @@ -81,7 +82,7 @@ public class SSTableReverseIteratorTest QueryProcessor.executeInternal(String.format("UPDATE %s.%s SET v1=? WHERE k=? AND c=?", KEYSPACE, table), bytes(0x20000), key, 2); QueryProcessor.executeInternal(String.format("UPDATE %s.%s SET v1=? WHERE k=? AND c=?", KEYSPACE, table), bytes(0x20000), key, 3); - tbl.forceBlockingFlush(); + Util.flush(tbl); SSTableReader sstable = Iterables.getOnlyElement(tbl.getLiveSSTables()); DecoratedKey dk = tbl.getPartitioner().decorateKey(Int32Type.instance.decompose(key)); RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ); diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java index 4725bcf399..531ca87bee 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogCQLTest.java @@ -45,7 +45,7 @@ public class CommitLogCQLTest extends CQLTester // Calling switchMemtable directly applies Flush even though memtable is empty. This can happen with some races // (flush with recycling by segment manager). It should still tell commitlog that the memtable's region is clean. // CASSANDRA-12436 - cfs.switchMemtable(); + cfs.switchMemtable(ColumnFamilyStore.FlushReason.UNIT_TESTS); execute("INSERT INTO %s (idx, data) VALUES (?, ?)", 15, Integer.toString(17)); diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java index 83723c5c5b..02ce3f6438 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogReaderTest.java @@ -27,6 +27,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.Config; @@ -261,7 +262,9 @@ public class CommitLogReaderTest extends CQLTester for (int i = midpoint; i < entryCount; i++) execute("INSERT INTO %s (idx, data) VALUES (?, ?)", i, Integer.toString(i)); - Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(); + Keyspace.open(keyspace()) + .getColumnFamilyStore(currentTable()) + .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); return result; } } diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java index af33c79a14..076f2fbad2 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogSegmentManagerCDCTest.java @@ -34,6 +34,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.commitlog.CommitLogSegment.CDCState; @@ -76,7 +77,9 @@ public class CommitLogSegmentManagerCDCTest extends CQLTester // Confirm that, on flush+recyle, we see files show up in cdc_raw CommitLogSegmentManagerCDC cdcMgr = (CommitLogSegmentManagerCDC)CommitLog.instance.segmentManager; - Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush(); + Keyspace.open(keyspace()) + .getColumnFamilyStore(currentTable()) + .forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); CommitLog.instance.forceRecycleAllSegments(); cdcMgr.awaitManagementTasksCompletion(); Assert.assertTrue("Expected files to be moved to overflow.", getCDCRawCount() > 0); diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java index c3c096144e..8f014353b3 100644 --- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java +++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java @@ -48,8 +48,11 @@ import org.junit.runners.Parameterized.Parameters; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.SkipListMemtable; import org.apache.cassandra.io.compress.ZstdCompressor; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; @@ -146,22 +149,25 @@ public abstract class CommitLogTest SchemaLoader.prepareServer(); StorageService.instance.getTokenMetadata().updateHostId(UUID.randomUUID(), FBUtilities.getBroadcastAddressAndPort()); + MemtableParams skipListMemtable = MemtableParams.get("skiplist"); + TableMetadata.Builder custom = TableMetadata.builder(KEYSPACE1, CUSTOM1) .addPartitionKeyColumn("k", IntegerType.instance) .addClusteringColumn("c1", MapType.getInstance(UTF8Type.instance, UTF8Type.instance, false)) .addClusteringColumn("c2", SetType.getInstance(UTF8Type.instance, false)) - .addStaticColumn("s", IntegerType.instance); + .addStaticColumn("s", IntegerType.instance) + .memtable(skipListMemtable); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), - SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance), - SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance), + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD1, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable), + SchemaLoader.standardCFMD(KEYSPACE1, STANDARD2, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable), custom); SchemaLoader.createKeyspace(KEYSPACE2, KeyspaceParams.simpleTransient(1), - SchemaLoader.standardCFMD(KEYSPACE2, STANDARD1, 0, AsciiType.instance, BytesType.instance), - SchemaLoader.standardCFMD(KEYSPACE2, STANDARD2, 0, AsciiType.instance, BytesType.instance)); + SchemaLoader.standardCFMD(KEYSPACE2, STANDARD1, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable), + SchemaLoader.standardCFMD(KEYSPACE2, STANDARD2, 0, AsciiType.instance, BytesType.instance).memtable(skipListMemtable)); CompactionManager.instance.disableAutoCompaction(); testKiller = new KillerForTests(); @@ -217,7 +223,7 @@ public abstract class CommitLogTest public void testHeaderOnlyFileFiltering() throws Exception { Assume.assumeTrue(!DatabaseDescriptor.getEncryptionContext().isEnabled()); - + File directory = new File(Files.createTempDir()); CommitLogDescriptor desc1 = new CommitLogDescriptor(CommitLogDescriptor.current_version, 1, null, DatabaseDescriptor.getEncryptionContext()); @@ -953,7 +959,7 @@ public abstract class CommitLogTest { try (Closeable c = Util.markDirectoriesUnwriteable(cfs)) { - cfs.forceBlockingFlush(); + Util.flush(cfs); } catch (Throwable t) { @@ -963,7 +969,7 @@ public abstract class CommitLogTest } } else - cfs.forceBlockingFlush(); + Util.flush(cfs); } } finally @@ -995,7 +1001,7 @@ public abstract class CommitLogTest Memtable current = cfs.getTracker().getView().getCurrentMemtable(); if (i == 2) - current.makeUnflushable(); + ((SkipListMemtable) current).makeUnflushable(); flushAction.accept(cfs, current); } @@ -1017,7 +1023,7 @@ public abstract class CommitLogTest { try { - cfs.forceBlockingFlush(); + Util.flush(cfs); } catch (Throwable t) { @@ -1025,7 +1031,7 @@ public abstract class CommitLogTest while (!(t instanceof FSWriteError)) t = t.getCause(); // Wait for started flushes to complete. - cfs.switchMemtableIfCurrent(current); + waitForStartedFlushes(cfs, current); } }; @@ -1037,9 +1043,14 @@ public abstract class CommitLogTest CommitLog.instance.forceRecycleAllSegments(); // Wait for started flushes to complete. - cfs.switchMemtableIfCurrent(current); + waitForStartedFlushes(cfs, current); }; + private void waitForStartedFlushes(ColumnFamilyStore cfs, Memtable current) + { + FBUtilities.waitOnFuture(cfs.switchMemtableIfCurrent(current, ColumnFamilyStore.FlushReason.UNIT_TESTS)); + } + @Test public void testOutOfOrderFlushRecovery() throws ExecutionException, InterruptedException, IOException { diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java index 4092f541f4..bd4b28fc03 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AbstractCompactionStrategyTest.java @@ -139,6 +139,6 @@ public class AbstractCompactionStrategyTest .add("val", "val") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } } diff --git a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java index f12c2c84e1..b0bb23cc64 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java @@ -27,6 +27,7 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; @@ -86,7 +87,7 @@ public class AbstractPendingRepairTest extends AbstractRepairTest int pk = nextSSTableKey++; Set pre = cfs.getLiveSSTables(); QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), pk, pk); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set post = cfs.getLiveSSTables(); Set diff = new HashSet<>(post); diff.removeAll(pre); diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java index 08c76bfa27..56d6c4082a 100644 --- a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java @@ -38,6 +38,7 @@ import org.junit.Test; import org.apache.cassandra.cache.AutoSavingCache; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.view.View; import org.apache.cassandra.db.view.ViewBuilderTask; @@ -67,7 +68,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName); @@ -111,7 +112,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName); @@ -134,7 +135,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } Set sstables = getCurrentColumnFamilyStore().getLiveSSTables(); try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstables, OperationType.INDEX_SUMMARY)) @@ -159,7 +160,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (k1, c1, val) VALUES (" + i + ", 2, 3)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } execute(String.format("CREATE MATERIALIZED VIEW %s.view1 AS SELECT k1, c1, val FROM %s.%s WHERE k1 IS NOT NULL AND c1 IS NOT NULL AND val IS NOT NULL PRIMARY KEY (val, k1, c1)", keyspace(), keyspace(), currentTable())); View view = Iterables.getOnlyElement(getCurrentColumnFamilyStore().viewManager); @@ -183,7 +184,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null); @@ -208,7 +209,7 @@ public class ActiveCompactionsTest extends CQLTester for (int i = 0; i < 5; i++) { execute("INSERT INTO %s (pk, ck, a, b) VALUES (" + i + ", 2, 3, 4)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null); diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java index 5f2464935f..c858930509 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionBytemanTest.java @@ -67,7 +67,7 @@ public class AntiCompactionBytemanTest extends CQLTester execute("insert into %s (id, i) values (1, 1)"); execute("insert into %s (id, i) values (2, 1)"); execute("insert into %s (id, i) values (3, 1)"); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); UntypedResultSet res = execute("select token(id) as tok from %s"); Iterator it = res.iterator(); List tokens = new ArrayList<>(); diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index 8a772f21b2..d3112d8ad4 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -31,6 +31,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; + +import org.apache.cassandra.Util; import org.apache.cassandra.io.util.File; import org.junit.Assert; import org.junit.BeforeClass; @@ -299,7 +301,7 @@ public class AntiCompactionTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); } @Test @@ -442,7 +444,7 @@ public class AntiCompactionTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); return store; } diff --git a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java index f63f4bbaa0..67421ba6d2 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CancelCompactionsTest.java @@ -462,7 +462,7 @@ public class CancelCompactionsTest extends CQLTester for (int i = 0; i < 10; i++) { execute("insert into %s (id, something) values (?,?)", i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); } AbstractCompactionTask ct = null; diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java index b38732e8e3..9d81b61ed3 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionControllerTest.java @@ -90,7 +90,7 @@ public class CompactionControllerTest extends SchemaLoader { assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp1); //memtable only - cfs.forceBlockingFlush(); + Util.flush(cfs); assertTrue(controller.getPurgeEvaluator(key).test(Long.MAX_VALUE)); //no memtables and no sstables } @@ -98,7 +98,7 @@ public class CompactionControllerTest extends SchemaLoader // create another sstable applyMutation(cfs.metadata(), key, timestamp2); - cfs.forceBlockingFlush(); + Util.flush(cfs); // check max purgeable timestamp when compacting the first sstable with and without a memtable try (CompactionController controller = new CompactionController(cfs, compacting, 0)) @@ -111,7 +111,7 @@ public class CompactionControllerTest extends SchemaLoader } // check max purgeable timestamp again without any sstables but with different insertion orders on the memtable - cfs.forceBlockingFlush(); + Util.flush(cfs); //newest to oldest try (CompactionController controller = new CompactionController(cfs, null, 0)) @@ -123,7 +123,7 @@ public class CompactionControllerTest extends SchemaLoader assertPurgeBoundary(controller.getPurgeEvaluator(key), timestamp3); //memtable only } - cfs.forceBlockingFlush(); + Util.flush(cfs); //oldest to newest try (CompactionController controller = new CompactionController(cfs, null, 0)) @@ -151,14 +151,14 @@ public class CompactionControllerTest extends SchemaLoader // create sstable with tombstone that should be expired in no older timestamps applyDeleteMutation(cfs.metadata(), key, timestamp2); - cfs.forceBlockingFlush(); + Util.flush(cfs); // first sstable with tombstone is compacting Set compacting = Sets.newHashSet(cfs.getLiveSSTables()); // create another sstable with more recent timestamp applyMutation(cfs.metadata(), key, timestamp1); - cfs.forceBlockingFlush(); + Util.flush(cfs); // second sstable is overlapping Set overlapping = Sets.difference(Sets.newHashSet(cfs.getLiveSSTables()), compacting); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java index 0aab021d00..ff3f210ec2 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java @@ -466,7 +466,7 @@ public class CompactionIteratorTest extends CQLTester createTable("CREATE TABLE %s (pk text, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2))"); for (int i = 0; i < 10; i++) execute("insert into %s (pk, ck1, ck2, v) values (?, ?, ?, ?)", "key", i, i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); DatabaseDescriptor.setSnapshotOnDuplicateRowDetection(true); TableMetadata metadata = getCurrentColumnFamilyStore().metadata(); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java index d0b2e80020..9960e8e0fe 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java @@ -509,7 +509,7 @@ public class CompactionStrategyManagerTest .build() .applyUnsafe(); Set before = cfs.getLiveSSTables(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set after = cfs.getLiveSSTables(); return Iterables.getOnlyElement(Sets.difference(after, before)); } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java index 4698e6e0a7..24b0c3dfed 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java @@ -30,6 +30,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; @@ -73,10 +74,10 @@ public class CompactionTaskTest cfs.getCompactionStrategyManager().disable(); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);"); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);"); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set sstables = cfs.getLiveSSTables(); Assert.assertEquals(2, sstables.size()); @@ -113,13 +114,13 @@ public class CompactionTaskTest { cfs.getCompactionStrategyManager().disable(); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstables = new ArrayList<>(cfs.getLiveSSTables()); Assert.assertEquals(4, sstables.size()); @@ -162,13 +163,13 @@ public class CompactionTaskTest { cfs.getCompactionStrategyManager().disable(); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (1, 1);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (2, 2);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (3, 3);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); QueryProcessor.executeInternal("INSERT INTO ks.tbl (k, v) VALUES (4, 4);"); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set sstables = cfs.getLiveSSTables(); Assert.assertEquals(4, sstables.size()); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java index 6ea05b2ce6..462d406df9 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java @@ -130,7 +130,7 @@ public class CompactionsBytemanTest extends CQLTester execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1); Util.spinAssertEquals(true, () -> CompactionManager.instance.compactingCF.count(cfs) == 0, 5); - cfs.forceBlockingFlush(); + Util.flush(cfs); Util.spinAssertEquals(true, () -> CompactionManager.instance.compactingCF.count(cfs) == 0, 5); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(cfs)); @@ -148,7 +148,7 @@ public class CompactionsBytemanTest extends CQLTester { execute("INSERT INTO %s (id, val) values (2, 'immortal')"); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } private void createLowGCGraceTable(){ @@ -195,7 +195,7 @@ public class CompactionsBytemanTest extends CQLTester { execute("insert into %s (k, c, v) values (?, ?, ?)", i, j, i*j); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null, false); for (int i = 0; i < 5; i++) @@ -204,7 +204,7 @@ public class CompactionsBytemanTest extends CQLTester { execute("insert into %s (k, c, v) values (?, ?, ?)", i, j, i*j); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } assertTrue(cfs.getTracker().getCompacting().isEmpty()); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java index 0af6be339c..69741fd820 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java @@ -30,6 +30,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.config.Config; @@ -214,7 +215,7 @@ public class CompactionsCQLTest extends CQLTester getCurrentColumnFamilyStore().setCompactionParameters(localOptions); assertTrue(verifyStrategies(getCurrentColumnFamilyStore().getCompactionStrategyManager(), DateTieredCompactionStrategy.class)); // Invalidate disk boundaries to ensure that boundary invalidation will not cause the old strategy to be reloaded - getCurrentColumnFamilyStore().invalidateDiskBoundaries(); + getCurrentColumnFamilyStore().invalidateLocalRanges(); // altering something non-compaction related execute("ALTER TABLE %s WITH gc_grace_seconds = 1000"); // should keep the local compaction strat @@ -295,7 +296,7 @@ public class CompactionsCQLTest extends CQLTester RangeTombstone rt = new RangeTombstone(Slice.ALL, new DeletionTime(System.currentTimeMillis(), -1)); RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, 22).clustering(33).addRangeTombstone(rt); rub.build().apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); compactAndValidate(); readAndValidate(true); readAndValidate(false); @@ -309,7 +310,7 @@ public class CompactionsCQLTest extends CQLTester // write a standard tombstone with negative local deletion time (LDTs are not set by user and should not be negative): RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), -1, System.currentTimeMillis() * 1000, 22).clustering(33).delete("b"); rub.build().apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); compactAndValidate(); readAndValidate(true); readAndValidate(false); @@ -323,7 +324,7 @@ public class CompactionsCQLTest extends CQLTester // write a partition deletion with negative local deletion time (LDTs are not set by user and should not be negative):: PartitionUpdate pu = PartitionUpdate.simpleBuilder(getCurrentColumnFamilyStore().metadata(), 22).nowInSec(-1).delete().build(); new Mutation(pu).apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); compactAndValidate(); readAndValidate(true); readAndValidate(false); @@ -336,7 +337,7 @@ public class CompactionsCQLTest extends CQLTester prepare(); // write a row deletion with negative local deletion time (LDTs are not set by user and should not be negative): RowUpdateBuilder.deleteRowAt(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, -1, 22, 33).apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); compactAndValidate(); readAndValidate(true); readAndValidate(false); @@ -358,7 +359,7 @@ public class CompactionsCQLTest extends CQLTester DatabaseDescriptor.setColumnIndexSize(1024); prepareWide(); RowUpdateBuilder.deleteRowAt(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, -1, 22, 33).apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); readAndValidate(true); readAndValidate(false); DatabaseDescriptor.setColumnIndexSize(maxSizePre); @@ -374,7 +375,7 @@ public class CompactionsCQLTest extends CQLTester prepareWide(); RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), -1, System.currentTimeMillis() * 1000, 22).clustering(33).delete("b"); rub.build().apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); readAndValidate(true); readAndValidate(false); DatabaseDescriptor.setColumnIndexSize(maxSizePre); @@ -391,7 +392,7 @@ public class CompactionsCQLTest extends CQLTester RangeTombstone rt = new RangeTombstone(Slice.ALL, new DeletionTime(System.currentTimeMillis(), -1)); RowUpdateBuilder rub = new RowUpdateBuilder(getCurrentColumnFamilyStore().metadata(), System.currentTimeMillis() * 1000, 22).clustering(33).addRangeTombstone(rt); rub.build().apply(); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); readAndValidate(true); readAndValidate(false); DatabaseDescriptor.setColumnIndexSize(maxSizePreKiB); @@ -413,7 +414,7 @@ public class CompactionsCQLTest extends CQLTester { execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } assertEquals(50, cfs.getLiveSSTables().size()); LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); @@ -430,7 +431,7 @@ public class CompactionsCQLTest extends CQLTester ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); cfs.disableAutoCompaction(); execute("insert into %s (id, id2, t) values (?, ?, ?)", 1,1,"L1"); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.forceMajorCompaction(); SSTableReader l1sstable = cfs.getLiveSSTables().iterator().next(); assertEquals(1, l1sstable.getSSTableLevel()); @@ -444,7 +445,7 @@ public class CompactionsCQLTest extends CQLTester { execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } assertEquals(51, cfs.getLiveSSTables().size()); @@ -471,14 +472,14 @@ public class CompactionsCQLTest extends CQLTester r.nextBytes(b); execute("insert into %s (id, x) values (?, ?)", i, ByteBuffer.wrap(b)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); getCurrentColumnFamilyStore().disableAutoCompaction(); for (int i = 0; i < 1000; i++) { r.nextBytes(b); execute("insert into %s (id, x) values (?, ?)", i, ByteBuffer.wrap(b)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) getCurrentColumnFamilyStore().getCompactionStrategyManager().getUnrepairedUnsafe().first(); LeveledCompactionTask lcsTask; @@ -505,7 +506,7 @@ public class CompactionsCQLTest extends CQLTester r.nextBytes(b); execute("insert into %s (id, x) values (?, ?)", i, ByteBuffer.wrap(b)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); // now we have a bunch of sstables in L2 and one in L0 - bump the L0 one to L1: for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables()) { @@ -647,7 +648,7 @@ public class CompactionsCQLTest extends CQLTester { execute("INSERT INTO %s (id, b) VALUES (?, ?)", i, String.valueOf(i)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false); if (deletedCell) @@ -655,7 +656,7 @@ public class CompactionsCQLTest extends CQLTester else execute("DELETE FROM %s WHERE id = ?", 50); getCurrentColumnFamilyStore().setNeverPurgeTombstones(false); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Thread.sleep(2000); // wait for gcgs to pass getCurrentColumnFamilyStore().forceMajorCompaction(); assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), false); @@ -664,7 +665,7 @@ public class CompactionsCQLTest extends CQLTester else execute("DELETE FROM %s WHERE id = ?", 44); getCurrentColumnFamilyStore().setNeverPurgeTombstones(true); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); Thread.sleep(1100); getCurrentColumnFamilyStore().forceMajorCompaction(); assertTombstones(getCurrentColumnFamilyStore().getLiveSSTables().iterator().next(), true); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java index a0d52aa6cc..4f1b639d70 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java @@ -102,14 +102,14 @@ public class CompactionsPurgeTest .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); // deletes for (int i = 0; i < 10; i++) { RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); // resurrect one column RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, key); @@ -117,7 +117,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // major compact and test that all columns but the resurrected one is completely gone FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false)); @@ -146,14 +146,14 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); // deletes for (int i = 0; i < 10; i++) { RowUpdateBuilder.deleteRow(cfs.metadata(), Long.MAX_VALUE, key, String.valueOf(i)).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); // major compact - tombstones should be purged FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false)); @@ -164,7 +164,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.invalidateCachedPartition(dk(key)); @@ -191,13 +191,13 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); new Mutation.PartitionUpdateCollector(KEYSPACE1, dk(key)) .add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), Long.MAX_VALUE, FBUtilities.nowInSeconds())) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // major compact - tombstones should be purged FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false)); @@ -208,7 +208,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.invalidateCachedPartition(dk(key)); @@ -235,11 +235,11 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), Long.MAX_VALUE, dk(key)) .addRangeTombstone(String.valueOf(0), String.valueOf(9)).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // major compact - tombstones should be purged FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, Integer.MAX_VALUE, false)); @@ -250,7 +250,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.invalidateCachedPartition(dk(key)); @@ -278,7 +278,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); // deletes for (int i = 0; i < 10; i++) @@ -286,7 +286,7 @@ public class CompactionsPurgeTest RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } DecoratedKey key1 = Util.dk("key1"); @@ -294,7 +294,7 @@ public class CompactionsPurgeTest // flush, remember the current sstable and then resurrect one column // for first key. Then submit minor compaction on remembered sstables. - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstablesIncomplete = cfs.getLiveSSTables(); RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata(), 2, "key1"); @@ -302,7 +302,7 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE)) { Iterables.getOnlyElement(tasks).execute(ActiveCompactionsTracker.NOOP); @@ -343,16 +343,16 @@ public class CompactionsPurgeTest .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // delete c1 RowUpdateBuilder.deleteRow(cfs.metadata(), 10, key3, "c1").applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstablesIncomplete = cfs.getLiveSSTables(); // delete c2 so we have new delete in a diffrent SSTable RowUpdateBuilder.deleteRow(cfs.metadata(), 9, key3, "c2").applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // compact the sstables with the c1/c2 data and the c1 tombstone try (CompactionTasks tasks = cfs.getCompactionStrategyManager().getUserDefinedTasks(sstablesIncomplete, Integer.MAX_VALUE)) @@ -393,7 +393,7 @@ public class CompactionsPurgeTest { RowUpdateBuilder.deleteRow(cfs.metadata(), 1, key, String.valueOf(i)).applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(String.valueOf(cfs.getLiveSSTables()), 1, cfs.getLiveSSTables().size()); // inserts & deletes were in the same memtable -> only deletes in sstable // compact and test that the row is completely gone @@ -438,7 +438,7 @@ public class CompactionsPurgeTest assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).isEmpty()); // flush and major compact - cfs.forceBlockingFlush(); + Util.flush(cfs); Util.compactAll(cfs, Integer.MAX_VALUE).get(); // Since we've force purging (by passing MAX_VALUE for gc_before), the row should have been invalidated and we should have no deletion info anymore @@ -474,7 +474,7 @@ public class CompactionsPurgeTest assertFalse(partition.partitionLevelDeletion().isLive()); // flush and major compact (with tombstone purging) - cfs.forceBlockingFlush(); + Util.flush(cfs); Util.compactAll(cfs, Integer.MAX_VALUE).get(); assertFalse(Util.getOnlyPartitionUnfiltered(Util.cmd(cfs, key).build()).isEmpty()); @@ -504,14 +504,14 @@ public class CompactionsPurgeTest // write a row out to one sstable QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", keyspace, table, 1, "foo", 1)); - cfs.forceBlockingFlush(); + Util.flush(cfs); UntypedResultSet result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(1, result.size()); // write a row tombstone out to a second sstable QueryProcessor.executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); - cfs.forceBlockingFlush(); + Util.flush(cfs); // basic check that the row is considered deleted assertEquals(2, cfs.getLiveSSTables().size()); @@ -529,14 +529,14 @@ public class CompactionsPurgeTest // write a row out to one sstable QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v1, v2) VALUES (%d, '%s', %d)", keyspace, table, 1, "foo", 1)); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(2, cfs.getLiveSSTables().size()); result = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s WHERE k = %d", keyspace, table, 1)); assertEquals(1, result.size()); // write a row tombstone out to a different sstable QueryProcessor.executeInternal(String.format("DELETE FROM %s.%s WHERE k = %d", keyspace, table, 1)); - cfs.forceBlockingFlush(); + Util.flush(cfs); // compact the two sstables with a gcBefore that *does* allow the row tombstone to be purged FBUtilities.waitOnFutures(CompactionManager.instance.submitMaximal(cfs, (int) (System.currentTimeMillis() / 1000) + 10000, false)); diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java index 23e3be0b65..fcf5c51da0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java @@ -141,7 +141,7 @@ public class CompactionsTest long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); //ttl=3s - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); long originalSize = store.getLiveSSTables().iterator().next().uncompressedLength(); @@ -183,11 +183,11 @@ public class CompactionsTest //Populate sstable1 with with keys [0..9] populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); //ttl=3s - store.forceBlockingFlush(); + Util.flush(store); //Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1) long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); //ttl=3s - store.forceBlockingFlush(); + Util.flush(store); assertEquals(2, store.getLiveSSTables().size()); @@ -267,7 +267,7 @@ public class CompactionsTest .add("val", "val1") .build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstables = cfs.getLiveSSTables(); assertEquals(1, sstables.size()); @@ -302,7 +302,7 @@ public class CompactionsTest notYetDeletedRowUpdateBuilder.clustering("02").add("val", "a"); //Range tombstone doesn't cover this (timestamp 3 > 2) notYetDeletedRowUpdateBuilder.build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } @Test @@ -387,7 +387,7 @@ public class CompactionsTest rowUpdateBuilder.clustering("c").add("val", "a"); rowUpdateBuilder.build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstablesBefore = cfs.getLiveSSTables(); @@ -405,7 +405,7 @@ public class CompactionsTest // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0 Thread.sleep(1000); - cfs.forceBlockingFlush(); + Util.flush(cfs); Collection sstablesAfter = cfs.getLiveSSTables(); Collection toCompact = new ArrayList(); @@ -487,7 +487,7 @@ public class CompactionsTest insertRowWithKey(i + 100); insertRowWithKey(i + 200); } - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); SSTableReader sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java index 9fe20c8b66..33dfc25599 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CorruptedSSTablesCompactionsTest.java @@ -165,7 +165,7 @@ public class CorruptedSSTablesCompactionsTest maxTimestampExpected = Math.max(timestamp, maxTimestampExpected); inserted.add(key); } - cfs.forceBlockingFlush(); + Util.flush(cfs); CompactionsTest.assertMaxTimestamp(cfs, maxTimestampExpected); assertEquals(inserted.toString(), inserted.size(), Util.getAll(Util.cmd(cfs).build()).size()); } diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java index f75842d05a..4df210a839 100644 --- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java @@ -231,9 +231,9 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstrs = new ArrayList<>(cfs.getLiveSSTables()); @@ -267,9 +267,9 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } - cfs.forceBlockingFlush(); + Util.flush(cfs); Iterable filtered; List sstrs = new ArrayList<>(cfs.getLiveSSTables()); @@ -304,7 +304,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader expiredSSTable = cfs.getLiveSSTables().iterator().next(); Thread.sleep(10); @@ -313,7 +313,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(cfs.getLiveSSTables().size(), 2); Map options = new HashMap<>(); @@ -357,7 +357,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", bigValue).build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } // and small ones: for (int r = 0; r < numSSTables / 2; r++) @@ -366,7 +366,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader new RowUpdateBuilder(cfs.metadata(), timestamp, key.getKey()) .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Map options = new HashMap<>(); options.put(SizeTieredCompactionStrategyOptions.MIN_SSTABLE_SIZE_KEY, "1"); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 6e0cebbf2d..4b9b527017 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -143,7 +143,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } waitForLeveling(cfs); @@ -198,7 +198,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } waitForLeveling(cfs); @@ -272,7 +272,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } waitForLeveling(cfs); @@ -309,9 +309,9 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } - cfs.forceBlockingFlush(); + Util.flush(cfs); LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getStrategies().get(1).get(0); cfs.forceMajorCompaction(); @@ -350,7 +350,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } waitForLeveling(cfs); cfs.disableAutoCompaction(); @@ -425,7 +425,7 @@ public class LeveledCompactionStrategyTest update.newRow("column" + c).add("val", value); update.applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } // create 20 more sstables with 10 containing data for key1 and other 10 containing data for key2 @@ -435,7 +435,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -480,7 +480,7 @@ public class LeveledCompactionStrategyTest update.newRow("column" + c).add("val", value); update.applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } // create 20 more sstables with 10 containing data for key1 and other 10 containing data for key2 @@ -492,7 +492,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } } @@ -538,7 +538,7 @@ public class LeveledCompactionStrategyTest for (int c = 0; c < columns; c++) update.newRow("column" + c).add("val", value); update.applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) (cfs.getCompactionStrategyManager()).getStrategies().get(1).get(0); // get readers for level 0 sstables @@ -797,7 +797,7 @@ public class LeveledCompactionStrategyTest update.newRow("column" + c).add("val", value); update.applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); long [] levelSizes = cfs.getPerLevelSizeBytes(); diff --git a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java index 241003ad55..089ede818e 100644 --- a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java @@ -22,6 +22,7 @@ import java.util.Collection; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -72,13 +73,13 @@ public class NeverPurgeTest extends CQLTester { execute("INSERT INTO %s (a, b, c) VALUES (" + j + ", 2, '3')"); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } execute("UPDATE %s SET c = null WHERE a=1 AND b=2"); execute("DELETE FROM %s WHERE a=2 AND b=2"); execute("DELETE FROM %s WHERE a=3"); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.enableAutoCompaction(); while (cfs.getLiveSSTables().size() > 1 || !cfs.getTracker().getCompacting().isEmpty()) Thread.sleep(100); @@ -92,7 +93,7 @@ public class NeverPurgeTest extends CQLTester execute("INSERT INTO %s (a, b, c) VALUES (1, 2, '3')"); execute(deletionStatement); Thread.sleep(1000); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.forceMajorCompaction(); verifyContainsTombstones(cfs.getLiveSSTables(), 1); } diff --git a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java index 0c469dc534..9bb2abd0d0 100644 --- a/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/OneCompactionTest.java @@ -71,7 +71,7 @@ public class OneCompactionTest .applyUnsafe(); inserted.add(key); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(inserted.size(), Util.getAll(Util.cmd(store).build()).size()); } CompactionManager.instance.performMaximal(store, false); diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java index fe35599ced..bb1d1f0c5d 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java @@ -25,6 +25,7 @@ import java.util.Random; import org.apache.commons.lang3.StringUtils; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -44,7 +45,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester createTable("create table %s (id int primary key, t text) with compaction = {'class':'LeveledCompactionStrategy','single_sstable_uplevel':true}"); ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); execute("insert into %s (id, t) values (1, 'meep')"); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.COMPACTION)) @@ -97,7 +98,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester execute("insert into %s (id, id2, t) values (?, ?, ?)", i, j, value); } if (i % 100 == 0) - cfs.forceBlockingFlush(); + Util.flush(cfs); } // now we have a bunch of data in L0, first compaction will be a normal one, containing all sstables: LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) cfs.getCompactionStrategyManager().getUnrepairedUnsafe().first(); @@ -125,7 +126,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester createTable("create table %s (id int primary key, t text) with compaction = {'class':'LeveledCompactionStrategy','single_sstable_uplevel':true}"); ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); execute("insert into %s (id, t) values (1, 'meep')"); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS); diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java index 00c4a86e0d..fc98a9cc1f 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java @@ -28,6 +28,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; @@ -165,9 +166,9 @@ public class SizeTieredCompactionStrategyTest new RowUpdateBuilder(cfs.metadata(), 0, key) .clustering("column").add("val", value) .build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstrs = new ArrayList<>(cfs.getLiveSSTables()); Pair, Double> bucket; diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java index bf95dd712e..0711f4ae04 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.*; @@ -91,7 +92,7 @@ public class TTLExpiryTest .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 2L, 1, key) .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() @@ -102,7 +103,7 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 4L, 1, key) .add("col1", ByteBufferUtil.EMPTY_BYTE_BUFFER) @@ -114,7 +115,7 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 6L, 3, key) @@ -127,7 +128,7 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set sstables = Sets.newHashSet(cfs.getLiveSSTables()); int now = (int)(System.currentTimeMillis() / 1000); @@ -170,7 +171,7 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key) .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) @@ -180,7 +181,7 @@ public class TTLExpiryTest .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); // To reproduce #10944, we need to avoid the optimization that get rid of full sstable because everything // is known to be gcAble, so keep some data non-expiring in that case. new RowUpdateBuilder(cfs.metadata(), timestamp, force10944Bug ? 0 : 1, key) @@ -189,14 +190,14 @@ public class TTLExpiryTest .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key) .add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getLiveSSTables().size()); cfs.enableAutoCompaction(true); @@ -218,32 +219,32 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key) .add("col2", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), timestamp, 1, key) .add("col3", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); String noTTLKey = "nottl"; new RowUpdateBuilder(cfs.metadata(), timestamp, noTTLKey) .add("col311", ByteBufferUtil.EMPTY_BYTE_BUFFER) .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getLiveSSTables().size()); cfs.enableAutoCompaction(true); assertEquals(1, cfs.getLiveSSTables().size()); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(cfs.metadata()), - DataRange.allData(cfs.getPartitioner()), - SSTableReadsListener.NOOP_LISTENER); + UnfilteredPartitionIterator scanner = sstable.partitionIterator(ColumnFilter.all(cfs.metadata()), + DataRange.allData(cfs.getPartitioner()), + SSTableReadsListener.NOOP_LISTENER); assertTrue(scanner.hasNext()); while(scanner.hasNext()) { @@ -267,7 +268,7 @@ public class TTLExpiryTest .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader blockingSSTable = cfs.getSSTables(SSTableSet.LIVE).iterator().next(); for (int i = 0; i < 10; i++) { @@ -276,7 +277,7 @@ public class TTLExpiryTest .delete("col1") .build() .applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Multimap blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(SSTableSet.LIVE), (int) (System.currentTimeMillis() / 1000) + 100); assertEquals(1, blockers.keySet().size()); diff --git a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java index 38033f156e..a9b148eedd 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java @@ -181,7 +181,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } // Decrement the timestamp to simulate a timestamp in the past hour for (int r = 3; r < 5; r++) @@ -191,10 +191,10 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader new RowUpdateBuilder(cfs.metadata(), r, key.getKey()) .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); } - cfs.forceBlockingFlush(); + Util.flush(cfs); HashMultimap buckets = HashMultimap.create(); List sstrs = new ArrayList<>(cfs.getLiveSSTables()); @@ -237,7 +237,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } // Reset the buckets, overfill it now @@ -272,7 +272,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader expiredSSTable = cfs.getLiveSSTables().iterator().next(); Thread.sleep(10); @@ -282,7 +282,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(cfs.getLiveSSTables().size(), 2); Map options = new HashMap<>(); @@ -324,7 +324,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader expiredSSTable = cfs.getLiveSSTables().iterator().next(); Thread.sleep(10); @@ -337,7 +337,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader .clustering("column") .add("val", value).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(cfs.getLiveSSTables().size(), 2); Map options = new HashMap<>(); diff --git a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java index 98abd0c093..a641be1b53 100644 --- a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java @@ -23,6 +23,7 @@ import java.util.*; import com.google.common.primitives.Longs; import org.junit.*; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.*; @@ -232,7 +233,7 @@ public class CompactionAwareWriterTest extends CQLTester execute(String.format("INSERT INTO %s.%s(k, t, v) VALUES (?, ?, ?)", KEYSPACE, TABLE), i, j, b); ColumnFamilyStore cfs = getColumnFamilyStore(); - cfs.forceBlockingFlush(); + Util.flush(cfs); if (cfs.getLiveSSTables().size() > 1) { // we want just one big sstable to avoid doing actual compaction in compact() above diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java index 1e0d15786f..781a46884d 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java @@ -30,7 +30,6 @@ import org.junit.Test; import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction.ReaderState.Action; @@ -77,7 +76,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest public void testUpdates() // (including obsoletion) { ColumnFamilyStore cfs = MockSchema.newCFS(); - Tracker tracker = new Tracker(null, false); + Tracker tracker = Tracker.newDummyTracker(); SSTableReader[] readers = readersArray(0, 3, cfs); SSTableReader[] readers2 = readersArray(0, 4, cfs); SSTableReader[] readers3 = readersArray(0, 4, cfs); @@ -141,7 +140,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest public void testCancellation() { ColumnFamilyStore cfs = MockSchema.newCFS(); - Tracker tracker = new Tracker(null, false); + Tracker tracker = Tracker.newDummyTracker(); List readers = readers(0, 3, cfs); tracker.addInitialSSTables(readers); LifecycleTransaction txn = tracker.tryModify(readers, OperationType.UNKNOWN); @@ -185,7 +184,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest public void testSplit() { ColumnFamilyStore cfs = MockSchema.newCFS(); - Tracker tracker = new Tracker(null, false); + Tracker tracker = Tracker.newDummyTracker(); List readers = readers(0, 4, cfs); tracker.addInitialSSTables(readers); LifecycleTransaction txn = tracker.tryModify(readers, OperationType.UNKNOWN); @@ -271,7 +270,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest private static Tracker tracker(ColumnFamilyStore cfs, List readers) { - Tracker tracker = new Tracker(new Memtable(new AtomicReference<>(CommitLogPosition.NONE), cfs), false); + Tracker tracker = new Tracker(cfs, cfs.createMemtable(new AtomicReference<>(CommitLogPosition.NONE)), false); tracker.addInitialSSTables(readers); return tracker; } diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java index 4390b20781..e39f71f025 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java @@ -24,23 +24,22 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; - import javax.annotation.Nullable; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.notifications.*; import org.apache.cassandra.schema.CachingParams; @@ -85,7 +84,7 @@ public class TrackerTest public void testTryModify() { ColumnFamilyStore cfs = MockSchema.newCFS(); - Tracker tracker = new Tracker(null, false); + Tracker tracker = Tracker.newDummyTracker(); List readers = ImmutableList.of(MockSchema.sstable(0, true, cfs), MockSchema.sstable(1, cfs), MockSchema.sstable(2, cfs)); tracker.addInitialSSTables(copyOf(readers)); Assert.assertNull(tracker.tryModify(ImmutableList.of(MockSchema.sstable(0, cfs)), OperationType.COMPACTION)); @@ -108,7 +107,7 @@ public class TrackerTest public void testApply() { final ColumnFamilyStore cfs = MockSchema.newCFS(); - final Tracker tracker = new Tracker(null, false); + final Tracker tracker = Tracker.newDummyTracker(); final View resultView = ViewTest.fakeView(0, 0, cfs); final AtomicInteger count = new AtomicInteger(); tracker.apply(new Predicate() @@ -277,15 +276,15 @@ public class TrackerTest Tracker tracker = cfs.getTracker(); tracker.subscribe(listener); - Memtable prev1 = tracker.switchMemtable(true, new Memtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()), cfs)); + Memtable prev1 = tracker.switchMemtable(true, cfs.createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()))); OpOrder.Group write1 = cfs.keyspace.writeOrder.getCurrent(); OpOrder.Barrier barrier1 = cfs.keyspace.writeOrder.newBarrier(); - prev1.setDiscarding(barrier1, new AtomicReference<>(CommitLog.instance.getCurrentPosition())); + prev1.switchOut(barrier1, new AtomicReference<>(CommitLog.instance.getCurrentPosition())); barrier1.issue(); - Memtable prev2 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()), cfs)); + Memtable prev2 = tracker.switchMemtable(false, cfs.createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()))); OpOrder.Group write2 = cfs.keyspace.writeOrder.getCurrent(); OpOrder.Barrier barrier2 = cfs.keyspace.writeOrder.newBarrier(); - prev2.setDiscarding(barrier2, new AtomicReference<>(CommitLog.instance.getCurrentPosition())); + prev2.switchOut(barrier2, new AtomicReference<>(CommitLog.instance.getCurrentPosition())); barrier2.issue(); Memtable cur = tracker.getView().getCurrentMemtable(); OpOrder.Group writecur = cfs.keyspace.writeOrder.getCurrent(); @@ -325,7 +324,7 @@ public class TrackerTest tracker = cfs.getTracker(); listener = new MockListener(false); tracker.subscribe(listener); - prev1 = tracker.switchMemtable(false, new Memtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()), cfs)); + prev1 = tracker.switchMemtable(false, cfs.createMemtable(new AtomicReference<>(CommitLog.instance.getCurrentPosition()))); tracker.markFlushing(prev1); reader = MockSchema.sstable(0, 10, true, cfs); cfs.invalidate(false); @@ -348,7 +347,7 @@ public class TrackerTest { ColumnFamilyStore cfs = MockSchema.newCFS(); SSTableReader r1 = MockSchema.sstable(0, cfs), r2 = MockSchema.sstable(1, cfs); - Tracker tracker = new Tracker(null, false); + Tracker tracker = Tracker.newDummyTracker(); MockListener listener = new MockListener(false); tracker.subscribe(listener); tracker.notifyAdded(singleton(r1), false); diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java index fd32087023..eb162d59b9 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java @@ -34,9 +34,9 @@ import org.junit.Test; import org.junit.Assert; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.MockSchema; diff --git a/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java b/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java new file mode 100644 index 0000000000..ec10c1a815 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/memtable/TestMemtable.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.memtable; + +import java.util.Map; + +public class TestMemtable +{ + public static Memtable.Factory factory(Map options) + { + String skiplist = options.remove("skiplist"); + if (Boolean.parseBoolean(skiplist)) + return SkipListMemtable.FACTORY; + else + return FACTORY; + } + + public static Memtable.Factory FACTORY = SkipListMemtable::new; +} diff --git a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java index 97127836b7..79be4c8643 100644 --- a/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/AbstractPendingAntiCompactionTest.java @@ -28,6 +28,7 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.QueryProcessor; @@ -109,7 +110,7 @@ public abstract class AbstractPendingAntiCompactionTest int val = i * rowsPerSSTable; // multiplied to prevent ranges from overlapping for (int j = 0; j < rowsPerSSTable; j++) QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, cfs.getTableName()), val + j, val + j); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(num, cfs.getLiveSSTables().size()); } diff --git a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java index 4063898661..bcf75827b0 100644 --- a/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java +++ b/test/unit/org/apache/cassandra/db/repair/CompactionManagerGetSSTablesForValidationTest.java @@ -29,6 +29,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.TableMetadata; @@ -94,7 +95,7 @@ public class CompactionManagerGetSSTablesForValidationTest for (int i=0; i<3; i++) { QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES(?, ?)", ks, tbl), i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); } Assert.assertEquals(3, cfs.getLiveSSTables().size()); diff --git a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java index 7fa8d5675b..a559478b87 100644 --- a/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java @@ -40,6 +40,8 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; + +import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.FutureTask; import org.apache.cassandra.concurrent.ImmediateExecutor; @@ -125,12 +127,12 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest { QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i); } - cfs.forceBlockingFlush(); + Util.flush(cfs); for (int i = 8; i < 12; i++) { QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", ks, tbl), i, i); } - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(2, cfs.getLiveSSTables().size()); Token left = ByteOrderedPartitioner.instance.getToken(ByteBufferUtil.bytes((int) 6)); diff --git a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java index f62a72cd8c..0c4a79d2a1 100644 --- a/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java +++ b/test/unit/org/apache/cassandra/db/rows/ThrottledUnfilteredIteratorTest.java @@ -110,7 +110,7 @@ public class ThrottledUnfilteredIteratorTest extends CQLTester // flush and generate 1 sstable ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.disableAutoCompaction(); cfs.forceMajorCompaction(); @@ -145,7 +145,7 @@ public class ThrottledUnfilteredIteratorTest extends CQLTester // flush and generate 1 sstable ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.disableAutoCompaction(); cfs.forceMajorCompaction(); @@ -203,7 +203,7 @@ public class ThrottledUnfilteredIteratorTest extends CQLTester // flush and generate 1 sstable ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()); - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.disableAutoCompaction(); cfs.forceMajorCompaction(); @@ -621,7 +621,7 @@ public class ThrottledUnfilteredIteratorTest extends CQLTester new RowUpdateBuilder(cfs.metadata(), 1, key).addRangeTombstone(10, 22).build().applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); builder = UpdateBuilder.create(cfs.metadata(), key).withTimestamp(2); for (int i = 1; i < 40; i += 2) diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java index 1e4acc457e..a65b2ea3a0 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Queue; +import org.apache.cassandra.Util; import org.apache.cassandra.io.sstable.Descriptor; import org.junit.BeforeClass; import org.junit.Test; @@ -102,7 +103,7 @@ public class CassandraEntireSSTableStreamWriterTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java index 9d663b5862..93b6e71e27 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java @@ -25,6 +25,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -78,7 +79,7 @@ public class CassandraOutgoingFileTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java index 999a44eeb8..0e5187c3d0 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java @@ -26,6 +26,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -80,7 +81,7 @@ public class CassandraStreamHeaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java index a0d409e75d..e7795c83f3 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamManagerTest.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.Util; import org.apache.cassandra.locator.RangesAtEndpoint; import org.junit.Assert; import org.junit.Before; @@ -118,7 +119,7 @@ public class CassandraStreamManagerTest { Set before = cfs.getLiveSSTables(); queryable.run(); - cfs.forceBlockingFlush(); + Util.flush(cfs); Set after = cfs.getLiveSSTables(); Set diff = Sets.difference(after, before); diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java index 9eb3e14be1..8b63ba53b4 100644 --- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java @@ -45,6 +45,7 @@ import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; @@ -128,7 +129,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); Token start = ByteOrderedPartitioner.instance.getTokenFactory().fromString(Long.toHexString(0)); diff --git a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java index 2e2ee8ff1e..e44cbcde8e 100644 --- a/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java +++ b/test/unit/org/apache/cassandra/db/transform/DuplicateRowCheckerTest.java @@ -65,7 +65,7 @@ public class DuplicateRowCheckerTest extends CQLTester createTable("CREATE TABLE %s (pk text, ck1 int, ck2 int, v int, PRIMARY KEY (pk, ck1, ck2))"); for (int i = 0; i < 10; i++) execute("insert into %s (pk, ck1, ck2, v) values (?, ?, ?, ?)", "key", i, i, i); - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); metadata = getCurrentColumnFamilyStore().metadata(); cfs = getCurrentColumnFamilyStore(); diff --git a/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java b/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java index 2341c730a4..c6df89860c 100644 --- a/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java +++ b/test/unit/org/apache/cassandra/db/view/ViewBuilderTaskTest.java @@ -24,6 +24,7 @@ import java.util.stream.IntStream; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.SystemKeyspace; @@ -84,8 +85,8 @@ public class ViewBuilderTaskTest extends CQLTester int expectedRowsInView) throws Throwable { // Truncate the materialized view (not the base table) - cfs.viewManager.forceBlockingFlush(); - cfs.viewManager.truncateBlocking(cfs.forceBlockingFlush(), System.currentTimeMillis()); + Util.flush(cfs.viewManager); + cfs.viewManager.truncateBlocking(cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS), System.currentTimeMillis()); assertRowCount(execute("SELECT * FROM " + viewName), 0); // Get the tokens from the referenced inserted rows diff --git a/test/unit/org/apache/cassandra/index/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/CustomIndexTest.java index 9cf3b10568..0d7fc9f59e 100644 --- a/test/unit/org/apache/cassandra/index/CustomIndexTest.java +++ b/test/unit/org/apache/cassandra/index/CustomIndexTest.java @@ -639,7 +639,7 @@ public class CustomIndexTest extends CQLTester try { - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); fail("Exception should have been propagated"); } catch (Throwable t) @@ -661,7 +661,7 @@ public class CustomIndexTest extends CQLTester // Insert a single wide partition to be indexed for (int i = 0; i < totalRows; i++) execute("INSERT INTO %s (k, c, v) VALUES (0, ?, ?)", i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Create the index, which won't automatically start building String indexName = "build_single_partition_idx"; @@ -718,7 +718,7 @@ public class CustomIndexTest extends CQLTester execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 5, 3, 3); execute("DELETE FROM %s WHERE k = ?", 5); - cfs.forceBlockingFlush(); + Util.flush(cfs); String indexName = "partition_index_test_idx"; createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v) USING '%s'", @@ -780,7 +780,7 @@ public class CustomIndexTest extends CQLTester // Insert a single row partition to be indexed for (int i = 0; i < totalRows; i++) execute("INSERT INTO %s (k, c, v) VALUES (0, ?, ?)", i, i); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Create the index, which won't automatically start building String indexName = "partition_overindex_test_idx"; @@ -806,7 +806,7 @@ public class CustomIndexTest extends CQLTester // Insert a single range tombstone execute("DELETE FROM %s WHERE k=1 and c > 2"); - cfs.forceBlockingFlush(); + Util.flush(cfs); // Create the index, which won't automatically start building String indexName = "range_tombstone_idx"; diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java index 51bb6bbf6a..04579b0399 100644 --- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java +++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java @@ -30,6 +30,7 @@ import java.util.stream.StreamSupport; import com.google.common.collect.ImmutableSet; +import org.apache.cassandra.Util; import org.apache.cassandra.index.TargetParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,7 +136,7 @@ public class CustomCassandraIndex implements Index public Callable getBlockingFlushTask() { return () -> { - indexCfs.forceBlockingFlush(); + Util.flush(indexCfs); return null; }; } @@ -597,7 +598,7 @@ public class CustomCassandraIndex implements Index CompactionManager.instance.interruptCompactionForCFs(cfss, (sstable) -> true, true); CompactionManager.instance.waitForCessation(cfss, (sstable) -> true); indexCfs.keyspace.writeOrder.awaitNewBarrier(); - indexCfs.forceBlockingFlush(); + Util.flush(indexCfs); indexCfs.readOrdering.awaitNewBarrier(); indexCfs.invalidate(); } @@ -622,7 +623,7 @@ public class CustomCassandraIndex implements Index private void buildBlocking() { - baseCfs.forceBlockingFlush(); + Util.flush(baseCfs); try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.selectFunction(SSTableSet.CANONICAL)); Refs sstables = viewFragment.refs) @@ -646,7 +647,7 @@ public class CustomCassandraIndex implements Index ImmutableSet.copyOf(sstables)); Future future = CompactionManager.instance.submitIndexBuild(builder); FBUtilities.waitOnFuture(future); - indexCfs.forceBlockingFlush(); + Util.flush(indexCfs); } logger.info("Index build of {} complete", metadata.name); } diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index e757c45f78..8c223a5a7f 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -37,6 +37,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; @@ -544,7 +545,7 @@ public class SASIIndexTest if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); final UntypedResultSet results = executeCQL(FTS_CF_NAME, "SELECT * FROM %s.%s WHERE artist LIKE 'lady%%'"); Assert.assertNotNull(results); @@ -896,7 +897,7 @@ public class SASIIndexTest rm3.build().apply(); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); final ByteBuffer dataOutputId = UTF8Type.instance.decompose("/data/output/id"); @@ -1058,7 +1059,7 @@ public class SASIIndexTest { setMinIndexInterval(minIndexInterval); IndexSummaryManager.instance.redistributeSummaries(); - store.forceBlockingFlush(); + Util.flush(store); Set rows = getIndexed(store, 100, buildExpression(firstName, Operator.LIKE_CONTAINS, UTF8Type.instance.decompose("a"))); Assert.assertEquals(rows.toString(), expected, rows.size()); @@ -1302,7 +1303,7 @@ public class SASIIndexTest rm.build().apply(); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); Set rows; @@ -1374,7 +1375,7 @@ public class SASIIndexTest rm.build().apply(); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); Set rows; @@ -1435,7 +1436,7 @@ public class SASIIndexTest rows = getIndexed(store, 10, buildExpression(comment, Operator.LIKE_MATCHES, bigValue.duplicate())); Assert.assertEquals(0, rows.size()); - store.forceBlockingFlush(); + Util.flush(store); rows = getIndexed(store, 10, buildExpression(comment, Operator.LIKE_MATCHES, bigValue.duplicate())); Assert.assertEquals(0, rows.size()); @@ -1538,7 +1539,7 @@ public class SASIIndexTest update(rm, fullName, UTF8Type.instance.decompose("利久 寺地"), 8000); rm.build().apply(); - store.forceBlockingFlush(); + Util.flush(store); Set rows; @@ -1575,7 +1576,7 @@ public class SASIIndexTest rm.build().apply(); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); Set rows; @@ -1660,7 +1661,7 @@ public class SASIIndexTest rm.build().apply(); // first flush would make interval for name - 'johnny' -> 'pavel' - store.forceBlockingFlush(); + Util.flush(store); rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key6")); update(rm, name, UTF8Type.instance.decompose("Jason"), 6000); @@ -1675,7 +1676,7 @@ public class SASIIndexTest rm.build().apply(); // this flush is going to produce range - 'jason' -> 'vijay' - store.forceBlockingFlush(); + Util.flush(store); // make sure that overlap of the prefixes is properly handled across sstables // since simple interval tree lookup is not going to cover it, prefix lookup actually required. @@ -1839,7 +1840,7 @@ public class SASIIndexTest executeCQL(CLUSTERING_CF_NAME_1 ,"INSERT INTO %s.%s (name, nickname, location, age, height, score) VALUES (?, ?, ?, ?, ?, ?)", "Jordan", "jrwest", "US", 27, 182, 1.0); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); UntypedResultSet results; @@ -1926,7 +1927,7 @@ public class SASIIndexTest executeCQL(CLUSTERING_CF_NAME_2 ,"INSERT INTO %s.%s (name, nickname, location, age, height, score) VALUES (?, ?, ?, ?, ?, ?)", "Christopher", "chis", "US", 27, 180, 1.0); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); results = executeCQL(CLUSTERING_CF_NAME_2 ,"SELECT * FROM %s.%s WHERE location LIKE 'US' AND age = 43 ALLOW FILTERING"); Assert.assertNotNull(results); @@ -1952,7 +1953,7 @@ public class SASIIndexTest executeCQL(STATIC_CF_NAME, "INSERT INTO %s.%s (sensor_id,date,value,variance) VALUES(?, ?, ?, ?)", 1, 20160403L, 24.96, 4); if (shouldFlush) - store.forceBlockingFlush(); + Util.flush(store); executeCQL(STATIC_CF_NAME, "INSERT INTO %s.%s (sensor_id,sensor_type) VALUES(?, ?)", 2, "PRESSURE"); executeCQL(STATIC_CF_NAME, "INSERT INTO %s.%s (sensor_id,date,value,variance) VALUES(?, ?, ?, ?)", 2, 20160401L, 1.03, 9); @@ -1960,7 +1961,7 @@ public class SASIIndexTest executeCQL(STATIC_CF_NAME, "INSERT INTO %s.%s (sensor_id,date,value,variance) VALUES(?, ?, ?, ?)", 2, 20160403L, 1.01, 4); if (shouldFlush) - store.forceBlockingFlush(); + Util.flush(store); UntypedResultSet results; @@ -2041,7 +2042,7 @@ public class SASIIndexTest executeCQL(CLUSTERING_CF_NAME_1, "INSERT INTO %s.%s (name, location, age, height, score) VALUES (?, ?, ?, ?, ?)", "Pavel", "BY", 28, 182, 2.0); executeCQL(CLUSTERING_CF_NAME_1, "INSERT INTO %s.%s (name, nickname, location, age, height, score) VALUES (?, ?, ?, ?, ?, ?)", "Jordan", "jrwest", "US", 27, 182, 1.0); - store.forceBlockingFlush(); + Util.flush(store); SSTable ssTable = store.getSSTables(SSTableSet.LIVE).iterator().next(); Path path = FileSystems.getDefault().getPath(ssTable.getFilename().replace("-Data", "-SI_" + CLUSTERING_CF_NAME_1 + "_age")); @@ -2080,7 +2081,7 @@ public class SASIIndexTest executeCQL(CLUSTERING_CF_NAME_1, "INSERT INTO %s.%s (name, nickname) VALUES (?, ?)", "Alex", "ifesdjeen"); - store.forceBlockingFlush(); + Util.flush(store); for (Index index : store.indexManager.listIndexes()) { @@ -2206,7 +2207,7 @@ public class SASIIndexTest { Keyspace keyspace = Keyspace.open(KS_NAME); for (String table : Arrays.asList(containsTable, prefixTable, analyzedPrefixTable)) - keyspace.getColumnFamilyStore(table).forceBlockingFlush(); + Util.flushTable(keyspace, table); } UntypedResultSet results; @@ -2440,7 +2441,7 @@ public class SASIIndexTest Assert.assertTrue(rangesSize(beforeFlushMemtable, expression) > 0); - store.forceBlockingFlush(); + Util.flush(store); IndexMemtable afterFlushMemtable = index.getCurrentMemtable(); @@ -2618,7 +2619,7 @@ public class SASIIndexTest ColumnFamilyStore store = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME); if (forceFlush) - store.forceBlockingFlush(); + Util.flush(store); return store; } diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java index 8760f43b80..be8b162766 100644 --- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java +++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Lists; import org.junit.Assert; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.compaction.CompactionInterruptedException; @@ -96,7 +97,7 @@ public class DiskSpaceMetricsTest extends CQLTester execute("INSERT INTO %s (pk) VALUES (?)", base + i); // flush to write the sstable - cfs.forceBlockingFlush(); + Util.flush(cfs); } private void assertDiskSpaceEqual(ColumnFamilyStore cfs) diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java index a751984831..08072f694f 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterReopenTest.java @@ -78,12 +78,12 @@ public class CompressedSequentialWriterReopenTest extends CQLTester { execute("insert into %s (id, t) values (?, ?)", i, ByteBuffer.wrap(blob)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); for (int i = 0; i < 10000; i++) { execute("insert into %s (id, t) values (?, ?)", i, ByteBuffer.wrap(blob)); } - getCurrentColumnFamilyStore().forceBlockingFlush(); + flush(); DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMiB(1); getCurrentColumnFamilyStore().forceMajorCompaction(); } diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java index f7cf6e9311..9522bbd12a 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java @@ -191,7 +191,7 @@ public class IndexSummaryManagerTest .build() .applyUnsafe(); } - futures.add(cfs.forceFlush()); + futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)); } for (Future future : futures) { @@ -516,7 +516,7 @@ public class IndexSummaryManagerTest .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); List sstables = new ArrayList<>(cfs.getLiveSSTables()); assertEquals(1, sstables.size()); @@ -581,7 +581,7 @@ public class IndexSummaryManagerTest .build() .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } assertTrue(manager.getAverageIndexInterval() >= cfs.metadata().params.minIndexInterval); diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java index 49872d5b15..57e4d4e4bd 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java @@ -125,7 +125,7 @@ public class IndexSummaryRedistributionTest .build() .applyUnsafe(); } - futures.add(cfs.forceFlush()); + futures.add(cfs.forceFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS)); } for (Future future : futures) { diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 9ffdb3ad60..063afe52db 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -633,7 +633,7 @@ public class LegacySSTableTest } } - StorageService.instance.forceKeyspaceFlush("legacy_tables"); + StorageService.instance.forceKeyspaceFlush("legacy_tables", ColumnFamilyStore.FlushReason.UNIT_TESTS); File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", BigFormat.latestVersion)); ksDir.tryCreateDirectories(); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java index 525020ce23..ec01865786 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java @@ -117,7 +117,7 @@ public class SSTableCorruptionDetectionTest extends SSTableWriterTestBase .add("reg2", ByteBuffer.wrap(reg2)); writer.append(builder.build().unfilteredIterator()); } - cfs.forceBlockingFlush(); + Util.flush(cfs); ssTableReader = writer.finish(true); txn.update(ssTableReader, false); @@ -214,11 +214,11 @@ public class SSTableCorruptionDetectionTest extends SSTableWriterTestBase for (int i = 0; i < numberOfPks; i++) { DecoratedKey dk = Util.dk(String.format("pkvalue_%07d", i)); - try (UnfilteredRowIterator rowIter = sstable.iterator(dk, - Slices.ALL, - ColumnFilter.all(cfs.metadata()), - false, - SSTableReadsListener.NOOP_LISTENER)) + try (UnfilteredRowIterator rowIter = sstable.rowIterator(dk, + Slices.ALL, + ColumnFilter.all(cfs.metadata()), + false, + SSTableReadsListener.NOOP_LISTENER)) { while (rowIter.hasNext()) { diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java index 3dc61517b0..6d79ed446d 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableLoaderTest.java @@ -139,7 +139,7 @@ public class SSTableLoaderTest } ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); - cfs.forceBlockingFlush(); // wait for sstables to be on disk else we won't be able to stream them + Util.flush(cfs); // wait for sstables to be on disk else we won't be able to stream them final CountDownLatch latch = new CountDownLatch(1); SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false)); @@ -180,7 +180,7 @@ public class SSTableLoaderTest } ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2); - cfs.forceBlockingFlush(); // wait for sstables to be on disk else we won't be able to stream them + Util.flush(cfs); // wait for sstables to be on disk else we won't be able to stream them //make sure we have some tables... assertTrue(Objects.requireNonNull(dataDir.tryList()).length > 0); @@ -228,14 +228,14 @@ public class SSTableLoaderTest } ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); - cfs.forceBlockingFlush(); // wait for sstables to be on disk else we won't be able to stream them + Util.flush(cfs); // wait for sstables to be on disk else we won't be able to stream them final CountDownLatch latch = new CountDownLatch(1); SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false), 1, KEYSPACE2); loader.stream(Collections.emptySet(), completionStreamListener(latch)).get(); cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); - cfs.forceBlockingFlush(); + Util.flush(cfs); List partitions = Util.getAll(Util.cmd(cfs).build()); @@ -267,7 +267,7 @@ public class SSTableLoaderTest } ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_BACKUPS); - cfs.forceBlockingFlush(); // wait for sstables to be on disk else we won't be able to stream them + Util.flush(cfs); // wait for sstables to be on disk else we won't be able to stream them final CountDownLatch latch = new CountDownLatch(1); SSTableLoader loader = new SSTableLoader(dataDir, new TestClient(), new OutputHandler.SystemOutput(false, false)); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java index aecddf9ca3..2e5a17ad42 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java @@ -95,7 +95,7 @@ public class SSTableMetadataTest .build() .applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); int ttltimestamp = (int) (System.currentTimeMillis() / 1000); int firstDelTime = 0; @@ -113,7 +113,7 @@ public class SSTableMetadataTest .applyUnsafe(); ttltimestamp = (int) (System.currentTimeMillis() / 1000); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(2, store.getLiveSSTables().size()); List sstables = new ArrayList<>(store.getLiveSSTables()); if (sstables.get(0).getSSTableMetadata().maxLocalDeletionTime < sstables.get(1).getSSTableMetadata().maxLocalDeletionTime) @@ -163,7 +163,7 @@ public class SSTableMetadataTest .build() .applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); int ttltimestamp = (int) (System.currentTimeMillis() / 1000); int firstMaxDelTime = 0; @@ -175,7 +175,7 @@ public class SSTableMetadataTest RowUpdateBuilder.deleteRow(store.metadata(), timestamp + 1, "deletetest", "todelete").applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(2, store.getLiveSSTables().size()); boolean foundDelete = false; for (SSTableReader sstable : store.getLiveSSTables()) @@ -212,7 +212,7 @@ public class SSTableMetadataTest .applyUnsafe(); } } - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); for (SSTableReader sstable : store.getLiveSSTables()) { @@ -233,7 +233,7 @@ public class SSTableMetadataTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); store.forceMajorCompaction(); assertEquals(1, store.getLiveSSTables().size()); for (SSTableReader sstable : store.getLiveSSTables()) @@ -260,7 +260,7 @@ public class SSTableMetadataTest ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); cfs.truncateBlocking(); @@ -271,7 +271,7 @@ public class SSTableMetadataTest cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); cfs.truncateBlocking(); @@ -282,7 +282,7 @@ public class SSTableMetadataTest cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); cfs.truncateBlocking(); @@ -292,7 +292,7 @@ public class SSTableMetadataTest cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertFalse(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); cfs.truncateBlocking(); } */ diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index 3b96ff83ac..f064f19fd9 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -124,7 +124,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); List> ranges = new ArrayList<>(); @@ -168,7 +168,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); // check that all our keys are found correctly @@ -208,7 +208,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); clearAndLoad(store); assert store.metric.maxPartitionSize.getValue() != 0; @@ -236,7 +236,7 @@ public class SSTableReaderTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); SSTableReader sstable = store.getLiveSSTables().iterator().next(); assertEquals(0, sstable.getReadMeter().count()); @@ -267,7 +267,7 @@ public class SSTableReaderTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); SSTableReader sstable = store.getLiveSSTables().iterator().next(); @@ -298,7 +298,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); // check if opening and querying works assertIndexQueryWorks(store); @@ -320,7 +320,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); SSTableReader sstable = store.getLiveSSTables().iterator().next(); @@ -355,7 +355,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); SSTableReader sstable = store.getLiveSSTables().iterator().next(); @@ -422,7 +422,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); SSTableReader sstable = store.getLiveSSTables().iterator().next(); Descriptor desc = sstable.descriptor; @@ -523,7 +523,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); for(ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores()) { @@ -551,7 +551,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); - store.forceBlockingFlush(); + Util.flush(store); boolean foundScanner = false; for (SSTableReader s : store.getLiveSSTables()) { @@ -583,7 +583,7 @@ public class SSTableReaderTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); // construct a range which is present in the sstable, but whose @@ -620,7 +620,7 @@ public class SSTableReaderTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); Collection sstables = store.getLiveSSTables(); @@ -697,7 +697,7 @@ public class SSTableReaderTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); Collection sstables = store.getLiveSSTables(); @@ -808,7 +808,7 @@ public class SSTableReaderTest .build() .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); return Sets.difference(cfs.getLiveSSTables(), before).iterator().next(); } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 070b078cb9..c88e0b09f7 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -83,7 +83,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase .build() .apply(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); Set sstables = new HashSet<>(cfs.getLiveSSTables()); assertEquals(1, sstables.size()); assertEquals(sstables.iterator().next().bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount()); @@ -700,7 +700,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase .build() .apply(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); cfs.forceMajorCompaction(); validateKeys(keyspace); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java index eff95fccbb..922200ab3b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DataRange; @@ -182,9 +183,9 @@ public class SSTableScannerTest assert boundaries.length % 2 == 0; for (DataRange range : dataRanges(sstable.metadata(), scanStart, scanEnd)) { - try(ISSTableScanner scanner = sstable.getScanner(ColumnFilter.all(sstable.metadata()), - range, - SSTableReadsListener.NOOP_LISTENER)) + try(UnfilteredPartitionIterator scanner = sstable.partitionIterator(ColumnFilter.all(sstable.metadata()), + range, + SSTableReadsListener.NOOP_LISTENER)) { for (int b = 0; b < boundaries.length; b += 2) for (int i = boundaries[b]; i <= boundaries[b + 1]; i++) @@ -215,7 +216,7 @@ public class SSTableScannerTest for (int i = 2; i < 10; i++) insertRowWithKey(store.metadata(), i); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); SSTableReader sstable = store.getLiveSSTables().iterator().next(); @@ -321,7 +322,7 @@ public class SSTableScannerTest for (int i = 0; i < 3; i++) for (int j = 2; j < 10; j++) insertRowWithKey(store.metadata(), i * 100 + j); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); SSTableReader sstable = store.getLiveSSTables().iterator().next(); @@ -441,7 +442,7 @@ public class SSTableScannerTest store.disableAutoCompaction(); insertRowWithKey(store.metadata(), 205); - store.forceBlockingFlush(); + Util.flush(store); assertEquals(1, store.getLiveSSTables().size()); SSTableReader sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java index 1ebb3c6fb4..5d20367663 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java @@ -216,11 +216,11 @@ public class SSTableWriterTest extends SSTableWriterTestBase try { DecoratedKey dk = Util.dk("large_value"); - UnfilteredRowIterator rowIter = sstable.iterator(dk, - Slices.ALL, - ColumnFilter.all(cfs.metadata()), - false, - SSTableReadsListener.NOOP_LISTENER); + UnfilteredRowIterator rowIter = sstable.rowIterator(dk, + Slices.ALL, + ColumnFilter.all(cfs.metadata()), + false, + SSTableReadsListener.NOOP_LISTENER); while (rowIter.hasNext()) { rowIter.next(); diff --git a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java index 7ae69ea38e..14b48b7855 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java @@ -24,9 +24,9 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DiskBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; @@ -69,7 +69,7 @@ public class RangeAwareSSTableWriterTest { SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, 0, 1); - cfs.forceBlockingFlush(); + Util.flush(cfs); LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM); diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java index ddd3a132ff..e6d20207df 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java @@ -118,7 +118,7 @@ public class BigTableZeroCopyWriterTest .applyUnsafe(); expectedRowCount++; } - store.forceBlockingFlush(); + Util.flush(store); sstable = store.getLiveSSTables().iterator().next(); } @@ -191,11 +191,11 @@ public class BigTableZeroCopyWriterTest for (int i = 0; i < store.metadata().params.minIndexInterval; i++) { DecoratedKey dk = Util.dk(String.valueOf(i)); - UnfilteredRowIterator rowIter = sstable.iterator(dk, - Slices.ALL, - ColumnFilter.all(store.metadata()), - false, - SSTableReadsListener.NOOP_LISTENER); + UnfilteredRowIterator rowIter = sstable.rowIterator(dk, + Slices.ALL, + ColumnFilter.all(store.metadata()), + false, + SSTableReadsListener.NOOP_LISTENER); while (rowIter.hasNext()) { rowIter.next(); diff --git a/test/unit/org/apache/cassandra/repair/ValidatorTest.java b/test/unit/org/apache/cassandra/repair/ValidatorTest.java index 4c78b743ad..86704d3d40 100644 --- a/test/unit/org/apache/cassandra/repair/ValidatorTest.java +++ b/test/unit/org/apache/cassandra/repair/ValidatorTest.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.compaction.CompactionsTest; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -187,7 +188,7 @@ public class ValidatorTest CompactionsTest.populate(keyspace, columnFamily, 0, n, 0); //ttl=3s - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); // wait enough to force single compaction @@ -244,7 +245,7 @@ public class ValidatorTest // 2 ** 14 rows would normally use 2^14 leaves, but with only 1 meg we should only use 2^12 CompactionsTest.populate(keyspace, columnFamily, 0, 1 << 14, 0); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); // wait enough to force single compaction @@ -303,7 +304,7 @@ public class ValidatorTest // 2 ** 14 rows would normally use 2^14 leaves, but with only 1 meg we should only use 2^12 CompactionsTest.populate(keyspace, columnFamily, 0, 1 << 14, 0); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertEquals(1, cfs.getLiveSSTables().size()); // wait enough to force single compaction diff --git a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java index b240d2131f..b3bab20c4d 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/PendingRepairStatTest.java @@ -30,6 +30,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; @@ -114,7 +115,7 @@ public class PendingRepairStatTest extends AbstractRepairTest int key = startKey + i; QueryProcessor.executeInternal(String.format("INSERT INTO %s.%s (k, v) VALUES (?, ?)", cfm.keyspace, cfm.name), key, key); } - cfs.forceBlockingFlush(); + Util.flush(cfs); return Iterables.getOnlyElement(Sets.difference(cfs.getLiveSSTables(), existing)); } diff --git a/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java b/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java new file mode 100644 index 0000000000..7db5204a9f --- /dev/null +++ b/test/unit/org/apache/cassandra/schema/MemtableParamsTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.schema; + +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.config.InheritingClass; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.db.memtable.SkipListMemtableFactory; +import org.apache.cassandra.exceptions.ConfigurationException; + +import static org.junit.Assert.assertEquals; + +public class MemtableParamsTest +{ + static final ParameterizedClass DEFAULT = SkipListMemtableFactory.CONFIGURATION; + + @Test + public void testDefault() + { + Map map = MemtableParams.expandDefinitions(ImmutableMap.of()); + assertEquals(ImmutableMap.of("default", DEFAULT), map); + } + + @Test + public void testDefaultRemapped() + { + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("remap", new InheritingClass("default", null, null)) + ); + assertEquals(ImmutableMap.of("default", DEFAULT, + "remap", DEFAULT), + map); + } + + @Test + public void testOne() + { + final InheritingClass one = new InheritingClass(null, "SkipList", null); + Map map = MemtableParams.expandDefinitions(ImmutableMap.of("one", one)); + assertEquals(ImmutableMap.of("default", DEFAULT, + "one", one), + map); + } + + @Test + public void testExtends() + { + final InheritingClass one = new InheritingClass(null, "SkipList", null); + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("one", one, + "two", new InheritingClass("one", + null, + ImmutableMap.of("extra", "value"))) + ); + + assertEquals(ImmutableMap.of("default", DEFAULT, + "one", one, + "two", new ParameterizedClass("SkipList", + ImmutableMap.of("extra", "value"))), + map); + } + + @Test + public void testExtendsReplace() + { + final InheritingClass one = new InheritingClass(null, + "SkipList", + ImmutableMap.of("extra", "valueOne")); + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("one", one, + "two", new InheritingClass("one", + null, + ImmutableMap.of("extra", "value"))) + ); + assertEquals(ImmutableMap.of("default", DEFAULT, + "one", one, + "two", new ParameterizedClass("SkipList", + ImmutableMap.of("extra", "value"))), + map); + } + + @Test + public void testDoubleExtends() + { + final InheritingClass one = new InheritingClass(null, "SkipList", null); + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("one", one, + "two", new InheritingClass("one", + null, + ImmutableMap.of("param", "valueTwo", + "extra", "value")), + "three", new InheritingClass("two", + "OtherClass", + ImmutableMap.of("param", "valueThree", + "extraThree", "three"))) + ); + assertEquals(ImmutableMap.of("default", DEFAULT, + "one", one, + "two", new ParameterizedClass("SkipList", + ImmutableMap.of("param", "valueTwo", + "extra", "value")), + "three", new ParameterizedClass("OtherClass", + ImmutableMap.of("param", "valueThree", + "extra", "value", + "extraThree", "three"))), + map); + } + + @Test + public void testInvalidExtends() + { + final InheritingClass one = new InheritingClass(null, "SkipList", null); + try + { + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("two", new InheritingClass("one", + null, + ImmutableMap.of("extra", "value")), + "one", one) + ); + Assert.fail("Expected exception."); + } + catch (ConfigurationException e) + { + // expected + } + } + + @Test + public void testInvalidSelfExtends() + { + try + { + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("one", new InheritingClass("one", + null, + ImmutableMap.of("extra", "value"))) + ); + Assert.fail("Expected exception."); + } + catch (ConfigurationException e) + { + // expected + } + } + + @Test + public void testReplaceDefault() + { + final InheritingClass one = new InheritingClass(null, + "SkipList", + ImmutableMap.of("extra", "valueOne")); + Map map = MemtableParams.expandDefinitions(ImmutableMap.of("default", one)); + assertEquals(ImmutableMap.of("default", one), map); + } + + @Test + public void testDefaultExtends() + { + final InheritingClass one = new InheritingClass(null, + "SkipList", + ImmutableMap.of("extra", "valueOne")); + Map map = MemtableParams.expandDefinitions + ( + ImmutableMap.of("one", one, + "default", new InheritingClass("one", null, ImmutableMap.of())) + ); + assertEquals(ImmutableMap.of("one", one, + "default", one), + map); + } + // Note: The factories constructed from these parameters are tested in the CreateTest and AlterTest. +} diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerDropKSTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerDropKSTest.java index a75116671e..c045b556d6 100644 --- a/test/unit/org/apache/cassandra/schema/MigrationManagerDropKSTest.java +++ b/test/unit/org/apache/cassandra/schema/MigrationManagerDropKSTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; @@ -71,7 +72,7 @@ public class MigrationManagerDropKSTest "dropKs", "col" + i, "anyvalue"); ColumnFamilyStore cfs = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name); assertNotNull(cfs); - cfs.forceBlockingFlush(); + Util.flush(cfs); assertTrue(!cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().isEmpty()); SchemaTestUtil.announceKeyspaceDrop(ks.name); diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java index 3dd889d2b8..e1556b7a52 100644 --- a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java +++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java @@ -176,7 +176,7 @@ public class MigrationManagerTest // flush to exercise more than just hitting the memtable ColumnFamilyStore cfs = Keyspace.open(ksName).getColumnFamilyStore(tableName); assertNotNull(cfs); - cfs.forceBlockingFlush(); + Util.flush(cfs); // and make sure we get out what we put in UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", ksName, tableName)); @@ -199,7 +199,7 @@ public class MigrationManagerTest "dropCf", "col" + i, "anyvalue"); ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name); assertNotNull(store); - store.forceBlockingFlush(); + Util.flush(store); assertTrue(store.getDirectories().sstableLister(Directories.OnTxnErr.THROW).list().size() > 0); SchemaTestUtil.announceTableDrop(ks.name, cfm.name); @@ -248,7 +248,7 @@ public class MigrationManagerTest "key0", "col0", "val0"); ColumnFamilyStore store = Keyspace.open(cfm.keyspace).getColumnFamilyStore(cfm.name); assertNotNull(store); - store.forceBlockingFlush(); + Util.flush(store); UntypedResultSet rows = QueryProcessor.executeInternal("SELECT * FROM newkeyspace1.newstandard1"); assertRows(rows, row("key0", "col0", "val0")); @@ -301,7 +301,7 @@ public class MigrationManagerTest ColumnFamilyStore cfs = Keyspace.open(newKs.name).getColumnFamilyStore(newCf.name); assertNotNull(cfs); - cfs.forceBlockingFlush(); + Util.flush(cfs); UntypedResultSet rows = QueryProcessor.executeInternal(String.format("SELECT * FROM %s.%s", EMPTY_KEYSPACE, tableName)); assertRows(rows, row("key0", "col0", "val0")); @@ -456,7 +456,7 @@ public class MigrationManagerTest TABLE1i), "key0", "col0", 1L, 1L); - cfs.forceBlockingFlush(); + Util.flush(cfs); ColumnFamilyStore indexCfs = cfs.indexManager.getIndexByName(indexName) .getBackingTable() .orElseThrow(throwAssert("Cannot access index cfs")); diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 0f2eb67a8d..fa172ca12f 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -35,6 +35,8 @@ import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.SkipListMemtable; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -87,7 +89,7 @@ public class MockSchema public static Memtable memtable(ColumnFamilyStore cfs) { - return new Memtable(cfs.metadata()); + return SkipListMemtable.FACTORY.create(null, cfs.metadata, cfs); } public static SSTableReader sstable(int generation, ColumnFamilyStore cfs) diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 031ed57ee9..70d40d89f8 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -328,7 +328,7 @@ public class ActiveRepairServiceTest .build() .applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); } } diff --git a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java index d9b2b1144a..09fda7c5f5 100644 --- a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java +++ b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.EncryptionOptions; import org.apache.cassandra.cql3.CQLTester; @@ -122,7 +123,7 @@ public class ClientWarningsTest extends CQLTester client.execute(query); } ColumnFamilyStore store = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable()); - store.forceBlockingFlush(); + Util.flush(store); for (int i = 0; i < iterations; i++) { @@ -132,7 +133,7 @@ public class ClientWarningsTest extends CQLTester i), QueryOptions.DEFAULT); client.execute(query); } - store.forceBlockingFlush(); + Util.flush(store); { QueryMessage query = new QueryMessage(String.format("SELECT * FROM %s.%s WHERE pk = 1", diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java index 518a71d9a8..d1733b4b63 100644 --- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosMockUpdateSupplier.java @@ -24,8 +24,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.Memtable; import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; diff --git a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java index 4b9dbe6379..fdb524ef30 100644 --- a/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedTrackerIntegrationTest.java @@ -23,8 +23,10 @@ import com.google.common.collect.Lists; import org.junit.*; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.TableMetadata; @@ -130,7 +132,7 @@ public class PaxosUncommittedTrackerIntegrationTest Assert.assertEquals(key, Iterators.getOnlyElement(iterator).getKey()); } - PAXOS_CFS.forceBlockingFlush(); + Util.flush(PAXOS_CFS); PaxosState.commitDirect(proposal.agreed()); try (CloseableIterator iterator = tracker.uncommittedKeyIterator(cfm.id, ALL_RANGES)) diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java index ea564647ee..2542202302 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java @@ -98,7 +98,7 @@ public class RangeCommandIteratorTest builder.add("val", String.valueOf(i)); builder.build().applyUnsafe(); } - cfs.forceBlockingFlush(); + Util.flush(cfs); PartitionRangeReadCommand command = (PartitionRangeReadCommand) Util.cmd(cfs).build(); AbstractBounds keyRange = command.dataRange().keyRange(); diff --git a/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java index 56afd66b24..34ce09c3f6 100644 --- a/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java +++ b/test/unit/org/apache/cassandra/streaming/EntireSSTableStreamingCorrectFilesCountTest.java @@ -37,6 +37,7 @@ import io.netty.channel.ChannelOutboundHandlerAdapter; import io.netty.channel.ChannelPromise; import io.netty.channel.embedded.EmbeddedChannel; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; @@ -102,7 +103,7 @@ public class EntireSSTableStreamingCorrectFilesCountTest .applyUnsafe(); } - store.forceBlockingFlush(); + Util.flush(store); CompactionManager.instance.performMaximal(store, false); sstable = store.getLiveSSTables().iterator().next(); diff --git a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java index b2966e62c7..dad1e7924f 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamTransferTaskTest.java @@ -31,9 +31,9 @@ import org.junit.BeforeClass; import org.junit.After; import org.junit.Test; -import io.netty.channel.embedded.EmbeddedChannel; import org.junit.Assert; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -99,7 +99,7 @@ public class StreamTransferTaskTest for (int i = 0; i < 2; i++) { SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, i, 1); - cfs.forceBlockingFlush(); + Util.flush(cfs); } // create streaming task that streams those two sstables @@ -150,7 +150,7 @@ public class StreamTransferTaskTest for (int i = 0; i < 2; i++) { SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, i, 1); - cfs.forceBlockingFlush(); + Util.flush(cfs); } // create streaming task that streams those two sstables diff --git a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java index 095951aff8..cffd3b2a48 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamingTransferTest.java @@ -175,7 +175,7 @@ public class StreamingTransferTest long timestamp = 1234; for (int i = 1; i <= 3; i++) mutator.mutate("key" + i, "col" + i, timestamp); - cfs.forceBlockingFlush(); + Util.flush(cfs); Util.compactAll(cfs, Integer.MAX_VALUE).get(); assertEquals(1, cfs.getLiveSSTables().size()); @@ -363,7 +363,7 @@ public class StreamingTransferTest .build() .apply(); - cfs.forceBlockingFlush(); + Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); cfs.clearUnsafe(); @@ -555,7 +555,7 @@ public class StreamingTransferTest // write a lot more data so the data is spread in more than 1 chunk. for (int i = 1; i <= 6000; i++) mutator.mutate("key" + i, "col" + i, System.currentTimeMillis()); - cfs.forceBlockingFlush(); + Util.flush(cfs); Util.compactAll(cfs, Integer.MAX_VALUE).get(); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); cfs.clearUnsafe(); diff --git a/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java index 5b02ccf1ba..14e1b93d6f 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java @@ -119,7 +119,7 @@ public class StandaloneSplitterWithCQLTesterTest extends CQLTester executeFormattedQuery(formatQuery("INSERT INTO %s (id, val) VALUES (?, ?)"), "mockData" + i, "mockData" + i); ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); - cfs.forceBlockingFlush(); + org.apache.cassandra.Util.flush(cfs); Set sstables = cfs.getLiveSSTables(); sstableFileName = sstables.iterator().next().getFilename(); diff --git a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java index 97349b1c2a..e9d2070570 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java @@ -141,7 +141,7 @@ public class StandaloneUpgraderOnSStablesTest private List getSStableFiles(String ks, String table) throws StartupException { ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(table); - cfs.forceBlockingFlush(); + org.apache.cassandra.Util.flush(cfs); ColumnFamilyStore.scrubDataDirectories(cfs.metadata()); Set sstables = cfs.getLiveSSTables(); diff --git a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java index 2741ca46c0..65dd1255be 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java @@ -215,6 +215,6 @@ public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils .apply(); } - cfs.forceBlockingFlush(); + org.apache.cassandra.Util.flush(cfs); } }