diff --git a/CHANGES.txt b/CHANGES.txt index 7e3930c375..72bca4b3c7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.2 + * CEP-17: SSTable API (CASSANDRA-17056) * Gossip stateMapOrdering does not have correct ordering when both EndpointState are in the bootstrapping set (CASSANDRA-18292) * Snapshot only sstables containing mismatching ranges on preview repair mismatch (CASSANDRA-17561) * More accurate skipping of sstables in read path (CASSANDRA-18134) diff --git a/NEWS.txt b/NEWS.txt index 72cabc5402..9a52d6b003 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -125,7 +125,8 @@ New features - On virtual tables, it is not strictly necessary to specify `ALLOW FILTERING` for select statements which would normally require it, except `system_views.system_logs`. - More accurate skipping of sstables in read path due to better handling of min/max clustering and lower bound; - SSTable format has been bumped to 'nc' because there are new fields in stats metadata + SSTable format has been bumped to 'nc' because there are new fields in stats metadata\ + - SSTal Upgrading --------- @@ -140,6 +141,7 @@ Upgrading upgrades involving 3.x and 4.x nodes. The fix for that issue makes it can now appear during rolling upgrades from 4.1.0 or 4.0.0-4.0.7. If that is your case, please use protocol v4 or higher in your driver. See CASSANDRA-17507 for further details. + - Added API for alternative sstable implementations. For details, see src/java/org/apache/cassandra/io/sstable/SSTable_API.md Deprecation ----------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 5a5ddd28f9..746fd5743e 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1910,3 +1910,14 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check +# Supported sstable formats +# This is a list of elements consisting of class_name and parameters, where class_name should point to the class +# implementing org.apache.cassandra.io.sstable.format.SSTableFormat. Parameters must include unique 'id' integer +# which is used in some serialization to denote the format type in a compact way (such as local key cache); and 'name' +# which will be used to recognize the format type - in particular that name will be used in sstable file names and in +# stream headers so the name has to be the same for the same format across all the nodes in the cluster. +sstable_formats: + - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat + parameters: + id: 0 + name: big diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index f2e59bafbe..243c070a32 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -17,13 +17,13 @@ */ package org.apache.cassandra.cache; -import java.io.BufferedInputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.NoSuchFileException; -import java.util.*; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -33,19 +33,28 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionInfo.Unit; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.compaction.CompactionInfo.Unit; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.util.ChecksummedRandomAccessReader; +import org.apache.cassandra.io.util.ChecksummedSequentialWriter; +import org.apache.cassandra.io.util.CorruptFileException; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Pair; @@ -59,8 +68,9 @@ public class AutoSavingCache extends InstrumentingCache extends InstrumentingCache extends InstrumentingCache extends InstrumentingCache extends InstrumentingCache>> futures = new ArrayDeque<>(); long loadByNanos = start + TimeUnit.SECONDS.toNanos(DatabaseDescriptor.getCacheLoadTimeout()); @@ -268,7 +281,7 @@ public class AutoSavingCache extends InstrumentingCache x.path.equals(fileName))); } - @VisibleForTesting - public void enable(boolean enabled) - { - ChunkCache.enabled = enabled; - cache.invalidateAll(); - metrics.reset(); - } - // TODO: Invalidate caches for obsoleted/MOVED_START tables? /** @@ -319,4 +318,4 @@ public class ChunkCache .map(policy -> policy.weightedSize().orElseGet(cache::estimatedSize)) .orElseGet(cache::estimatedSize); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index 71dfd7f364..73aa67b9a5 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -341,6 +341,9 @@ public enum CassandraRelevantProperties SEED_COUNT_WARN_THRESHOLD("cassandra.seed_count_warn_threshold"), + SSTABLE_FORMAT_DEFAULT("cassandra.sstable.format.default"), + + /** When enabled, recursive directory deletion will be executed using a unix command `rm -rf` instead of traversing * and removing individual files. This is now used only tests, but eventually we will make it true by default.*/ USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"), @@ -356,6 +359,7 @@ public enum CassandraRelevantProperties * can be also done manually for that particular case: {@code flush(SchemaConstants.SCHEMA_KEYSPACE_NAME);}. */ FLUSH_LOCAL_SCHEMA_CHANGES("cassandra.test.flush_local_schema_changes", "true"), + TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS("cassandra.streaminghistogram.roundseconds", "60"), ; CassandraRelevantProperties(String key, String defaultVal) @@ -561,6 +565,21 @@ public enum CassandraRelevantProperties return Enum.valueOf(defaultValue.getDeclaringClass(), toUppercase ? value.toUpperCase() : value); } + /** + * Gets the value of a system property as an enum, optionally calling {@link String#toUpperCase()} first. + * If the value is missing, the default value for this property is used + * + * @param toUppercase before converting to enum + * @param enumClass enumeration class + * @param type + * @return enum value + */ + public > T getEnum(boolean toUppercase, Class enumClass) + { + String value = System.getProperty(key, defaultVal); + return Enum.valueOf(enumClass, toUppercase ? value.toUpperCase() : value); + } + /** * Sets the value into system properties. * @param value to set diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 4082d8eb41..848939acf1 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -21,25 +21,27 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.function.Supplier; - import javax.annotation.Nullable; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.fql.FullQueryLoggerOptions; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.service.StartupChecks.StartupCheckType; /** @@ -70,6 +72,9 @@ public class Config */ public static final String PROPERTY_PREFIX = "cassandra."; + public static final String SSTABLE_FORMAT_ID = "id"; + public static final String SSTABLE_FORMAT_NAME = "name"; + public String cluster_name = "Test Cluster"; public String authenticator; public String authorizer; @@ -352,6 +357,10 @@ public class Config public String[] data_file_directories = new String[0]; + public List sstable_formats = ImmutableList.of(new ParameterizedClass(BigFormat.class.getName(),// "org.apache.cassandra.io.sstable.format.big.BigFormat", + ImmutableMap.of(SSTABLE_FORMAT_ID, "0", + SSTABLE_FORMAT_NAME, "big"))); + /** * The directory to use for storing the system keyspaces data. * If unspecified the data will be stored in the first of the data_file_directories. diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 3e1630c561..e314f0a677 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -33,6 +33,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -43,12 +45,14 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; import java.util.regex.Pattern; - +import java.util.stream.Collectors; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; @@ -78,6 +82,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.fql.FullQueryLoggerOptions; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.util.DiskOptimizationStrategy; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; @@ -185,6 +190,8 @@ public class DatabaseDescriptor private static GuardrailsOptions guardrails; private static StartupChecksOptions startupChecksOptions; + private static Map>> sstableFormatFactories; + private static Function commitLogSegmentMgrProvider = c -> DatabaseDescriptor.isCDCEnabled() ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation()) : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation()); @@ -247,6 +254,8 @@ public class DatabaseDescriptor setConfig(loadConfig()); + applySSTableFormats(); + applySimpleConfig(); applyPartitioner(); @@ -264,6 +273,14 @@ public class DatabaseDescriptor clientInitialization(true); } + /** + * Equivalent to {@link #clientInitialization(boolean) clientInitialization(true, Config::new)}. + */ + public static void clientInitialization(boolean failIfDaemonOrTool) + { + clientInitialization(failIfDaemonOrTool, Config::new); + } + /** * Initializes this class as a client, which means that just an empty configuration will * be used. @@ -272,7 +289,7 @@ public class DatabaseDescriptor * {@link #toolInitialization()} has been performed before, an * {@link AssertionError} will be thrown. */ - public static void clientInitialization(boolean failIfDaemonOrTool) + public static void clientInitialization(boolean failIfDaemonOrTool, Supplier configSupplier) { if (!failIfDaemonOrTool && (daemonInitialized || toolInitialized)) { @@ -291,8 +308,9 @@ public class DatabaseDescriptor clientInitialized = true; setDefaultFailureDetector(); Config.setClientMode(true); - conf = new Config(); + conf = configSupplier.get(); diskOptimizationStrategy = new SpinningDiskOptimizationStrategy(); + applySSTableFormats(); } public static boolean isClientInitialized() @@ -379,6 +397,8 @@ public class DatabaseDescriptor private static void applyAll() throws ConfigurationException { //InetAddressAndPort cares that applySimpleConfig runs first + applySSTableFormats(); + applySimpleConfig(); applyPartitioner(); @@ -1330,6 +1350,74 @@ public class DatabaseDescriptor paritionerName = partitioner.getClass().getCanonicalName(); } + @VisibleForTesting + public static Map>> loadSSTableFormatFactories(List configuredFormats) + { + ImmutableMap.Builder>> sstableFormatFactories = ImmutableMap.builderWithExpectedSize(configuredFormats.size()); + Set names = new HashSet<>(configuredFormats.size()); + Set ids = new HashSet<>(configuredFormats.size()); + + for (ParameterizedClass formatConfig : configuredFormats) + { + assert formatConfig.parameters != null; + Map params = new HashMap<>(formatConfig.parameters); + + String name = params.get(Config.SSTABLE_FORMAT_NAME); + if (name == null) + throw new ConfigurationException("Missing 'name' parameter in sstable format configuration for " + formatConfig.class_name); + if (!name.matches("^[a-z]+$")) + throw new ConfigurationException("'name' parameter in sstable format configuration for " + formatConfig.class_name + " must be non-empty, lower-case letters only string"); + if (names.contains(name)) + throw new ConfigurationException("Name '" + name + "' of sstable format " + formatConfig.class_name + " is already defined for another sstable format"); + params.remove(Config.SSTABLE_FORMAT_NAME); + + String idString = params.get(Config.SSTABLE_FORMAT_ID); + if (idString == null) + throw new ConfigurationException("Missing 'id' parameter in sstable format configuration for " + formatConfig.class_name); + int id; + try + { + id = Integer.parseInt(idString); + } + catch (RuntimeException ex) + { + throw new ConfigurationException("'id' parameter in sstable format configuration for " + formatConfig.class_name + " must be an integer"); + } + if (id < 0 || id > 127) + throw new ConfigurationException("'id' parameter in sstable format configuration for " + formatConfig.class_name + " must be within bounds [0..127] range"); + if (ids.contains(id)) + throw new ConfigurationException("ID '" + id + "' of sstable format " + formatConfig.class_name + " is already defined for another sstable format"); + params.remove(Config.SSTABLE_FORMAT_ID); + + Supplier> factory = () -> { + Class> cls = FBUtilities.classForName(formatConfig.class_name, "sstable format"); + if (!SSTableFormat.class.isAssignableFrom(cls)) + throw new ConfigurationException(String.format("Class %s for sstable format %s does not implement %s", formatConfig.class_name, name, SSTableFormat.class.getName())); + + SSTableFormat sstableFormat = FBUtilities.instanceOrConstruct(cls.getName(), "sstable format"); + sstableFormat.setup(id, name, params); + return sstableFormat; + }; + sstableFormatFactories.put(name, factory); + names.add(name); + ids.add(id); + } + + return sstableFormatFactories.build(); + } + + private static void applySSTableFormats() + { + if (sstableFormatFactories != null) + logger.warn("Reinitializing SSTableFactories - this should happen only in tests"); + + sstableFormatFactories = loadSSTableFormatFactories(conf.sstable_formats); + + Iterable types = SSTableFormat.Type.values(); // make sure we know where those types get initialized + types.forEach(t -> t.info.allComponents()); // make sure to reach all supported components for a type so that we know all of them are registered + logger.info("Supported sstable formats are: {}", Lists.newArrayList(types).stream().map(f -> f.name + " -> " + f.info.getClass().getName() + " with singleton components: " + f.info.allComponents()).collect(Collectors.joining(", "))); + } + /** * Computes the sum of the 2 specified positive values returning {@code Long.MAX_VALUE} if the sum overflow. * @@ -3208,6 +3296,12 @@ public class DatabaseDescriptor return conf.file_cache_enabled; } + @VisibleForTesting + public static void setFileCacheEnabled(boolean enabled) + { + conf.file_cache_enabled = enabled; + } + public static int getFileCacheSizeInMiB() { if (conf.file_cache_size == null) @@ -4578,4 +4672,9 @@ public class DatabaseDescriptor { conf.client_request_size_metrics_enabled = enabled; } + + public static Map>> getSSTableFormatFactories() + { + return Objects.requireNonNull(sstableFormatFactories, "Forgot to initialize DatabaseDescriptor?"); + } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index d656c37380..edfe831a8c 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -88,17 +88,16 @@ import org.apache.cassandra.db.compaction.CompactionInfo; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.CompactionStrategyManager; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.filter.ClusteringIndexFilter; import org.apache.cassandra.db.filter.DataLimits; -import org.apache.cassandra.db.memtable.Flushing; -import org.apache.cassandra.db.memtable.Memtable; -import org.apache.cassandra.db.memtable.ShardBoundaries; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.Tracker; import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.memtable.Flushing; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.memtable.ShardBoundaries; import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.repair.CassandraTableRepairManager; @@ -120,17 +119,19 @@ import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableIdFactory; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; -import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.Sampler; import org.apache.cassandra.metrics.Sampler.Sample; import org.apache.cassandra.metrics.Sampler.SamplerType; @@ -186,7 +187,7 @@ import static org.apache.cassandra.utils.Throwables.maybeFail; import static org.apache.cassandra.utils.Throwables.merge; import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; -public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner +public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner, SSTable.Owner { private static final Logger logger = LoggerFactory.getLogger(ColumnFamilyStore.class); @@ -507,7 +508,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner if (data.loadsstables) { Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true); - sstables = SSTableReader.openAll(sstableFiles.list().entrySet(), metadata); + sstables = SSTableReader.openAll(this, sstableFiles.list().entrySet(), metadata); data.addInitialSSTablesWithoutUpdatingSize(sstables); } @@ -795,19 +796,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner } } - File dataFile = new File(desc.filenameFor(Component.DATA)); - if (components.contains(Component.DATA) && dataFile.length() > 0) - // everything appears to be in order... moving on. - continue; - - // missing the DATA file! all components are orphaned - logger.warn("Removing orphans for {}: {}", desc, components); - for (Component component : components) - { - File file = new File(desc.filenameFor(component)); - if (file.exists()) - FileUtils.deleteWithConfirm(desc.filenameFor(component)); - } + desc.getFormat().deleteOrphanedComponents(desc, components); } // cleanup incomplete saved caches @@ -889,7 +878,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner sstableIdGenerator.get(), descriptor.formatType); } - while (newDescriptor.fileFor(Component.DATA).exists()); + while (newDescriptor.fileFor(Components.DATA).exists()); return newDescriptor; } @@ -949,7 +938,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner name, sstableIdGenerator.get(), format); - assert !newDescriptor.fileFor(Component.DATA).exists(); + assert !newDescriptor.fileFor(Components.DATA).exists(); return newDescriptor; } @@ -1530,7 +1519,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner View view = data.getView(); List sortedByFirst = Lists.newArrayList(sstables); - Collections.sort(sortedByFirst, (o1, o2) -> o1.first.compareTo(o2.first)); + sortedByFirst.sort(SSTableReader.sstableComparator); List> bounds = new ArrayList<>(); DecoratedKey first = null, last = null; @@ -1547,21 +1536,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner { if (first == null) { - first = sstable.first; - last = sstable.last; + first = sstable.getFirst(); + last = sstable.getLast(); } else { - if (sstable.first.compareTo(last) <= 0) // we do overlap + if (sstable.getFirst().compareTo(last) <= 0) // we do overlap { - if (sstable.last.compareTo(last) > 0) - last = sstable.last; + if (sstable.getLast().compareTo(last) > 0) + last = sstable.getLast(); } else { bounds.add(AbstractBounds.bounds(first, true, last, true)); - first = sstable.first; - last = sstable.last; + first = sstable.getFirst(); + last = sstable.getLast(); } } } @@ -1669,13 +1658,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return CompactionManager.instance.performCleanup(ColumnFamilyStore.this, jobs); } - public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs) throws ExecutionException, InterruptedException + public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException { - return scrub(disableSnapshot, skipCorrupted, reinsertOverflowedTTL, false, checkData, jobs); + return scrub(disableSnapshot, false, options, jobs); } @VisibleForTesting - public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean skipCorrupted, boolean reinsertOverflowedTTL, boolean alwaysFail, boolean checkData, int jobs) throws ExecutionException, InterruptedException + public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean alwaysFail, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException { // skip snapshot creation during scrub, SEE JIRA 5891 if(!disableSnapshot) @@ -1687,7 +1676,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner try { - return CompactionManager.instance.performScrub(ColumnFamilyStore.this, skipCorrupted, checkData, reinsertOverflowedTTL, jobs); + return CompactionManager.instance.performScrub(ColumnFamilyStore.this, options, jobs); } catch(Throwable t) { @@ -1722,7 +1711,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return true; } - public CompactionManager.AllSSTableOpStatus verify(Verifier.Options options) throws ExecutionException, InterruptedException + public CompactionManager.AllSSTableOpStatus verify(IVerifier.Options options) throws ExecutionException, InterruptedException { return CompactionManager.instance.performVerify(ColumnFamilyStore.this, options); } @@ -1730,7 +1719,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner /** * Rewrites all SSTables according to specified parameters * - * @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link org.apache.cassandra.io.sstable.format.big.BigFormat#latestVersion}) + * @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link SSTableFormat#getLatestVersion()}) * @param skipIfNewerThanTimestamp - max timestamp (local creation time) for SSTable; SSTables created _after_ this timestamp will be excluded from compaction * @param skipIfCompressionMatches - if {@link true}, will rewrite only SSTables whose compression parameters are different from {@link TableMetadata#params#getCompressionParameters()} ()} * @param jobs number of jobs for parallel execution @@ -1973,7 +1962,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner for (SSTableReader sstr : select(View.select(SSTableSet.LIVE, dk)).sstables) { // check if the key actually exists in this sstable, without updating cache and stats - if (sstr.getPosition(dk, SSTableReader.Operator.EQ, false) != null) + if (sstr.getPosition(dk, SSTableReader.Operator.EQ, false) >= 0) mapped.add(mapper.apply(sstr)); } return mapped; @@ -2136,7 +2125,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner private List mapToDataFilenames(Collection sstables) { - return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Component.DATA)).collect(Collectors.toList()); + return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Components.DATA)).collect(Collectors.toList()); } private void writeSnapshotSchema(File schemaFile) @@ -2194,7 +2183,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner if (logger.isTraceEnabled()) logger.trace("using snapshot sstable {}", entries.getKey()); // open offline so we don't modify components or track hotness. - sstable = SSTableReader.open(entries.getKey(), entries.getValue(), metadata, true, true); + sstable = SSTableReader.open(this, entries.getKey(), entries.getValue(), metadata, true, true); refs.tryRef(sstable); // release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released sstable.selfRef().release(); @@ -2944,6 +2933,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner } + @Override public Double getCrcCheckChance() { return crcCheckChance.value(); @@ -3289,10 +3279,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner DecoratedKey last = null; for (SSTableReader sstable : sstables) { - if (first == null || first.compareTo(sstable.first) > 0) - first = sstable.first; - if (last == null || last.compareTo(sstable.last) < 0) - last = sstable.last; + if (first == null || first.compareTo(sstable.getFirst()) > 0) + first = sstable.getFirst(); + if (last == null || last.compareTo(sstable.getLast()) < 0) + last = sstable.getLast(); } DiskBoundaries diskBoundaries = getDiskBoundaries(); @@ -3476,4 +3466,16 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return null; return topPartitions.topTombstones().lastUpdate; } + + @Override + public OpOrder.Barrier newReadOrderingBarrier() + { + return readOrdering.newBarrier(); + } + + @Override + public TableMetrics getMetrics() + { + return metric; + } } diff --git a/src/java/org/apache/cassandra/db/DecoratedKey.java b/src/java/org/apache/cassandra/db/DecoratedKey.java index 914566e962..01ada14204 100644 --- a/src/java/org/apache/cassandra/db/DecoratedKey.java +++ b/src/java/org/apache/cassandra/db/DecoratedKey.java @@ -30,11 +30,12 @@ import org.apache.cassandra.dht.Token.KeyBound; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.IFilter.FilterKey; +import org.apache.cassandra.utils.MurmurHash; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; -import org.apache.cassandra.utils.IFilter.FilterKey; -import org.apache.cassandra.utils.MurmurHash; +import org.apache.cassandra.utils.memory.HeapCloner; /** * Represents a decorated key, handy for certain operations @@ -47,13 +48,7 @@ import org.apache.cassandra.utils.MurmurHash; */ public abstract class DecoratedKey implements PartitionPosition, FilterKey { - public static final Comparator comparator = new Comparator() - { - public int compare(DecoratedKey o1, DecoratedKey o2) - { - return o1.compareTo(o2); - } - }; + public static final Comparator comparator = DecoratedKey::compareTo; private final Token token; @@ -200,6 +195,17 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey public abstract ByteBuffer getKey(); public abstract int getKeyLength(); + /** + * If this key occupies only part of a larger buffer, allocate a new buffer that is only as large as necessary. + * Otherwise, it returns this key. + */ + public DecoratedKey retainable() + { + return ByteBufferUtil.canMinimize(getKey()) + ? new BufferDecoratedKey(getToken(), HeapCloner.instance.clone(getKey())) + : this; + } + public void filterHash(long[] dest) { ByteBuffer key = getKey(); @@ -240,4 +246,4 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey assert terminator == ByteSource.TERMINATOR : "Decorated key encoding must end in terminator."; return keyBytes; } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java index b169724e84..cf07e995ac 100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@ -17,46 +17,59 @@ */ package org.apache.cassandra.db; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ThreadLocalRandom; -import java.util.function.BiPredicate; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import com.google.common.annotations.VisibleForTesting; - import java.io.IOError; import java.io.IOException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.Spliterator; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.util.concurrent.RateLimiter; - import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.*; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.FSDiskFullWriteError; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.FSNoDiskAvailableForWriteError; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileStoreUtils; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.io.util.PathUtils; +import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.service.snapshot.TableSnapshot; @@ -275,7 +288,7 @@ public class Directories if (file.isDirectory()) return false; - Descriptor desc = SSTable.tryDescriptorFromFilename(file); + Descriptor desc = SSTable.tryDescriptorFromFile(file); return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name); }); for (File indexFile : indexFiles) @@ -322,7 +335,7 @@ public class Directories { File file = new File(dir, filename); if (file.exists()) - return Descriptor.fromFilename(file); + return Descriptor.fromFileWithComponent(file, false).left; } return null; } @@ -969,7 +982,7 @@ public class Directories { for (Component c : entry.getValue()) { - l.add(new File(entry.getKey().filenameFor(c))); + l.add(entry.getKey().fileFor(c)); } } return l; @@ -1310,7 +1323,7 @@ public class Directories public boolean isAcceptable(Path path) { File file = new File(path); - Descriptor desc = SSTable.tryDescriptorFromFilename(file); + Descriptor desc = SSTable.tryDescriptorFromFile(file); return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name) diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index b01a79fcff..164e03cf65 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -23,9 +23,6 @@ import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; -import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.filter.ClusteringIndexFilter; import org.apache.cassandra.db.filter.ColumnFilter; @@ -38,14 +35,17 @@ import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.db.rows.BaseRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.transform.RTBoundValidator; import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; +import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.index.Index; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.metrics.TableMetrics; diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java index 83b4e7bc21..87402130cf 100644 --- a/src/java/org/apache/cassandra/db/SSTableImporter.java +++ b/src/java/org/apache/cassandra/db/SSTableImporter.java @@ -28,19 +28,20 @@ import java.util.Set; import java.util.UUID; import com.google.common.annotations.VisibleForTesting; - -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Refs; @@ -185,7 +186,7 @@ public class SSTableImporter for (SSTableReader reader : newSSTables) { if (options.invalidateCaches && cfs.isRowCacheEnabled()) - invalidateCachesForSSTable(reader.descriptor); + invalidateCachesForSSTable(reader); } } @@ -209,8 +210,8 @@ public class SSTableImporter private static String formatMetadata(SSTableReader sstable) { return String.format("{[%s, %s], %d, %s, %d}", - sstable.first.getToken(), - sstable.last.getToken(), + sstable.getFirst().getToken(), + sstable.getLast().getToken(), sstable.getSSTableLevel(), sstable.isRepaired(), sstable.onDiskLength()); @@ -233,7 +234,7 @@ public class SSTableImporter SSTableReader sstable = null; try { - sstable = SSTableReader.open(descriptor, components, cfs.metadata); + sstable = SSTableReader.open(cfs, descriptor, components, cfs.metadata); targetDirectory = cfs.getDirectories().getLocationForDisk(cfs.diskBoundaryManager.getDiskBoundaries(cfs).getCorrectDiskForSSTable(sstable)); } finally @@ -241,7 +242,7 @@ public class SSTableImporter if (sstable != null) sstable.selfRef().release(); } - return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(new File(descriptor.baseFilename())) : targetDirectory; + return targetDirectory == null ? cfs.getDirectories().getWriteableLocationToLoadFile(descriptor.baseFile()) : targetDirectory; } /** @@ -304,11 +305,11 @@ public class SSTableImporter { for (MovedSSTable movedSSTable : movedSSTables) { - if (new File(movedSSTable.newDescriptor.filenameFor(Component.DATA)).exists()) + if (movedSSTable.newDescriptor.fileFor(Components.DATA).exists()) { - logger.debug("Moving sstable {} back to {}", movedSSTable.newDescriptor.filenameFor(Component.DATA) - , movedSSTable.oldDescriptor.filenameFor(Component.DATA)); - SSTableWriter.rename(movedSSTable.newDescriptor, movedSSTable.oldDescriptor, movedSSTable.components); + logger.debug("Moving sstable {} back to {}", movedSSTable.newDescriptor.fileFor(Components.DATA) + , movedSSTable.oldDescriptor.fileFor(Components.DATA)); + SSTable.rename(movedSSTable.newDescriptor, movedSSTable.oldDescriptor, movedSSTable.components); } } } @@ -324,11 +325,8 @@ public class SSTableImporter logger.debug("Removing copied SSTables which were left in data directories after failed SSTable import."); for (MovedSSTable movedSSTable : movedSSTables) { - if (new File(movedSSTable.newDescriptor.filenameFor(Component.DATA)).exists()) - { - // no logging here as for moveSSTablesBack case above as logging is done in delete method - SSTableWriter.delete(movedSSTable.newDescriptor, movedSSTable.components); - } + // no logging here as for moveSSTablesBack case above as logging is done in delete method + movedSSTable.newDescriptor.getFormat().delete(movedSSTable.newDescriptor); } } @@ -336,9 +334,9 @@ public class SSTableImporter * Iterates over all keys in the sstable index and invalidates the row cache */ @VisibleForTesting - void invalidateCachesForSSTable(Descriptor desc) + void invalidateCachesForSSTable(SSTableReader reader) { - try (KeyIterator iter = new KeyIterator(desc, cfs.metadata())) + try (KeyIterator iter = reader.keyIterator()) { while (iter.hasNext()) { @@ -346,6 +344,10 @@ public class SSTableImporter cfs.invalidateCachedPartition(decoratedKey); } } + catch (IOException ex) + { + throw new RuntimeException("Failed to import sstable " + reader.getFilename(), ex); + } } /** @@ -360,14 +362,15 @@ public class SSTableImporter SSTableReader reader = null; try { - reader = SSTableReader.open(descriptor, components, cfs.metadata); - Verifier.Options verifierOptions = Verifier.options() - .extendedVerification(extendedVerify) - .checkOwnsTokens(verifyTokens) - .quick(!verifySSTables) - .invokeDiskFailurePolicy(false) - .mutateRepairStatus(false).build(); - try (Verifier verifier = new Verifier(cfs, reader, false, verifierOptions)) + reader = SSTableReader.open(cfs, descriptor, components, cfs.metadata); + IVerifier.Options verifierOptions = IVerifier.options() + .extendedVerification(extendedVerify) + .checkOwnsTokens(verifyTokens) + .quick(!verifySSTables) + .invokeDiskFailurePolicy(false) + .mutateRepairStatus(false).build(); + + try (IVerifier verifier = reader.getVerifier(cfs, new OutputHandler.LogOutput(), false, verifierOptions)) { verifier.verify(); } @@ -389,7 +392,7 @@ public class SSTableImporter */ private void maybeMutateMetadata(Descriptor descriptor, Options options) throws IOException { - if (new File(descriptor.filenameFor(Component.STATS)).exists()) + if (descriptor.fileFor(Components.STATS).exists()) { if (options.resetLevel) { diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index cd390fa6db..d85e2fe628 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -19,7 +19,12 @@ package org.apache.cassandra.db; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -30,18 +35,37 @@ import org.apache.cassandra.cache.IRowCacheEntry; import org.apache.cassandra.cache.RowCacheKey; import org.apache.cassandra.cache.RowCacheSentinel; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.lifecycle.*; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.memtable.Memtable; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.partitions.CachedBTreePartition; +import org.apache.cassandra.db.partitions.CachedPartition; +import org.apache.cassandra.db.partitions.ImmutableBTreePartition; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; +import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator; import org.apache.cassandra.db.transform.RTBoundValidator; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.metrics.TableMetrics; @@ -49,7 +73,9 @@ import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.*; +import org.apache.cassandra.service.CacheService; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.btree.BTreeSet; @@ -538,20 +564,26 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar try { // Use a custom iterator instead of DataLimits to avoid stopping the original iterator - UnfilteredRowIterator toCacheIterator = new WrappingUnfilteredRowIterator(iter) + UnfilteredRowIterator toCacheIterator = new WrappingUnfilteredRowIterator() { private int rowsCounted = 0; + @Override + public UnfilteredRowIterator wrapped() + { + return iter; + } + @Override public boolean hasNext() { - return rowsCounted < rowsToCache && super.hasNext(); + return rowsCounted < rowsToCache && iter.hasNext(); } @Override public Unfiltered next() { - Unfiltered unfiltered = super.next(); + Unfiltered unfiltered = iter.next(); if (unfiltered.isRow()) { Row row = (Row) unfiltered; @@ -1307,7 +1339,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar private int mergedSSTables; @Override - public void onSSTableSelected(SSTableReader sstable, RowIndexEntry indexEntry, SelectionReason reason) + public void onSSTableSelected(SSTableReader sstable, SelectionReason reason) { sstable.incrementReadCount(); mergedSSTables++; diff --git a/src/java/org/apache/cassandra/db/StorageHook.java b/src/java/org/apache/cassandra/db/StorageHook.java index b2a243970e..7014ae4d8b 100644 --- a/src/java/org/apache/cassandra/db/StorageHook.java +++ b/src/java/org/apache/cassandra/db/StorageHook.java @@ -23,8 +23,8 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.FBUtilities; diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index 35aac1bb2a..5fbece5210 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -17,34 +17,39 @@ */ package org.apache.cassandra.db.compaction; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; - -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.index.Index; -import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTableMultiWriter; -import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTableMultiWriter; +import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.utils.TimeUUID; -import static org.apache.cassandra.io.sstable.Component.DATA; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** @@ -399,7 +404,7 @@ public abstract class AbstractCompactionStrategy // since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually. // if that happens we will end up in infinite compaction loop, so first we check enough if enough time has // elapsed since SSTable created. - if (currentTimeMillis() < sstable.getCreationTimeFor(DATA) + tombstoneCompactionInterval * 1000) + if (currentTimeMillis() < sstable.getDataCreationTime() + tombstoneCompactionInterval * 1000) return false; double droppableRatio = sstable.getEstimatedDroppableTombstoneRatio(gcBefore); @@ -423,16 +428,16 @@ public abstract class AbstractCompactionStrategy else { // what percentage of columns do we expect to compact outside of overlap? - if (sstable.getIndexSummarySize() < 2) + if (!sstable.isEstimationInformative()) { // we have too few samples to estimate correct percentage return false; } // first, calculate estimated keys that do not overlap long keys = sstable.estimatedKeys(); - Set> ranges = new HashSet>(overlaps.size()); + Set> ranges = new HashSet<>(overlaps.size()); for (SSTableReader overlap : overlaps) - ranges.add(new Range<>(overlap.first.getToken(), overlap.last.getToken())); + ranges.add(new Range<>(overlap.getFirst().getToken(), overlap.getLast().getToken())); long remainingKeys = keys - sstable.estimatedKeysForRanges(ranges); // next, calculate what percentage of columns we have within those keys long columns = sstable.getEstimatedCellPerPartitionCount().mean() * remainingKeys; @@ -560,7 +565,7 @@ public abstract class AbstractCompactionStrategy Collection indexes, LifecycleNewTracker lifecycleNewTracker) { - return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker); + return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker, cfs); } public boolean supportsEarlyOpen() diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java index de994a15cf..e289e71b6d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java @@ -17,7 +17,14 @@ */ package org.apache.cassandra.db.compaction; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.LongPredicate; import com.google.common.base.Predicates; @@ -27,7 +34,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.Config; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.AbstractCompactionController; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -251,7 +261,7 @@ public class CompactionController extends AbstractCompactionController for (SSTableReader sstable: filteredSSTables) { - if (sstable.maybePresent(key)) + if (sstable.mayContainAssumingKeyIsInRange(key)) { minTimestampSeen = Math.min(minTimestampSeen, sstable.getMinTimestamp()); hasTimestamp = true; @@ -316,8 +326,8 @@ public class CompactionController extends AbstractCompactionController reader.getMaxTimestamp() <= minTimestamp || tombstoneOnly && !reader.mayHaveTombstones()) return null; - RowIndexEntry position = reader.getPosition(key, SSTableReader.Operator.EQ); - if (position == null) + long position = reader.getPosition(key, SSTableReader.Operator.EQ); + if (position < 0) return null; FileDataInput dfile = openDataFiles.computeIfAbsent(reader, this::openDataFile); return reader.simpleIterator(dfile, key, position, tombstoneOnly); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java index 7e6f5f64a2..0bfc925a7d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionInfo.java @@ -57,7 +57,7 @@ public final class CompactionInfo private final ImmutableSet sstables; private final String targetDirectory; - public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection sstables, String targetDirectory) + public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection sstables, String targetDirectory) { this.tasktype = tasktype; this.completed = completed; @@ -74,7 +74,7 @@ public final class CompactionInfo this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory); } - public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection sstables) + public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection sstables) { this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null); } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java index 4d1b2ac132..5cde41e1bc 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java @@ -17,33 +17,51 @@ */ package org.apache.cassandra.db.compaction; -import java.util.*; -import java.util.function.LongPredicate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.LongPredicate; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Ordering; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; - -import org.apache.cassandra.db.transform.DuplicateRowChecker; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.AbstractCompactionController; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.PurgeFunction; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator; +import org.apache.cassandra.db.transform.DuplicateRowChecker; import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.CompactionTransaction; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; import org.apache.cassandra.utils.TimeUUID; @@ -385,8 +403,10 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte * The result produced by this iterator is such that when merged with tombSource it produces the same output * as the merge of dataSource and tombSource. */ - private static class GarbageSkippingUnfilteredRowIterator extends WrappingUnfilteredRowIterator + private static class GarbageSkippingUnfilteredRowIterator implements WrappingUnfilteredRowIterator { + private final UnfilteredRowIterator wrapped; + final UnfilteredRowIterator tombSource; final DeletionTime partitionLevelDeletion; final Row staticRow; @@ -413,7 +433,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte */ protected GarbageSkippingUnfilteredRowIterator(UnfilteredRowIterator dataSource, UnfilteredRowIterator tombSource, boolean cellLevelGC) { - super(dataSource); + this.wrapped = dataSource; this.tombSource = tombSource; this.cellLevelGC = cellLevelGC; metadata = dataSource.metadata(); @@ -433,6 +453,12 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte dataNext = advance(dataSource); } + @Override + public UnfilteredRowIterator wrapped() + { + return wrapped; + } + private static Unfiltered advance(UnfilteredRowIterator source) { return source.hasNext() ? source.next() : null; @@ -446,7 +472,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte public void close() { - super.close(); + wrapped.close(); tombSource.close(); } @@ -719,4 +745,4 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte { return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index b64f579d70..31221bfe39 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -18,7 +18,16 @@ package org.apache.cassandra.db.compaction; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -34,25 +43,33 @@ import javax.management.openmbean.TabularData; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.*; +import com.google.common.base.Predicates; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Collections2; +import com.google.common.collect.ConcurrentHashMultiset; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multiset; +import com.google.common.collect.Sets; import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; - -import org.apache.cassandra.concurrent.ExecutorFactory; -import org.apache.cassandra.concurrent.WrappedExecutorPlus; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.locator.RangesAtEndpoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.openhft.chronicle.core.util.ThrowingSupplier; import org.apache.cassandra.cache.AutoSavingCache; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.repair.NoSuchRepairSessionException; -import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.concurrent.ExecutorFactory; +import org.apache.cassandra.concurrent.WrappedExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.DiskBoundaries; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionInfo.Holder; import org.apache.cassandra.db.lifecycle.ILifecycleTransaction; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -62,27 +79,40 @@ import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.lifecycle.WrappedLifecycleTransaction; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.view.ViewBuilderTask; -import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.index.SecondaryIndexBuilder; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.io.sstable.IndexSummaryRedistribution; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.SSTableRewriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.metrics.CompactionMetrics; import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.repair.NoSuchRepairSessionException; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.MBeanWrapper; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.WrappedRunnable; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.cassandra.utils.concurrent.Refs; @@ -455,16 +485,7 @@ public class CompactionManager implements CompactionManagerMBean } } - public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData, - int jobs) - throws InterruptedException, ExecutionException - { - return performScrub(cfs, skipCorrupted, checkData, false, jobs); - } - - public AllSSTableOpStatus performScrub(final ColumnFamilyStore cfs, final boolean skipCorrupted, final boolean checkData, - final boolean reinsertOverflowedTTL, int jobs) - throws InterruptedException, ExecutionException + public AllSSTableOpStatus performScrub(ColumnFamilyStore cfs, IScrubber.Options options, int jobs) { return parallelAllSSTableOperation(cfs, new OneSSTableOperation() { @@ -477,12 +498,12 @@ public class CompactionManager implements CompactionManagerMBean @Override public void execute(LifecycleTransaction input) { - scrubOne(cfs, input, skipCorrupted, checkData, reinsertOverflowedTTL, active); + scrubOne(cfs, input, options, active); } }, jobs, OperationType.SCRUB); } - public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, Verifier.Options options) throws InterruptedException, ExecutionException + public AllSSTableOpStatus performVerify(ColumnFamilyStore cfs, IVerifier.Options options) throws InterruptedException, ExecutionException { assert !cfs.isIndex(); return parallelAllSSTableOperation(cfs, new OneSSTableOperation() @@ -513,7 +534,7 @@ public class CompactionManager implements CompactionManagerMBean return false; // Skip if SSTable creation time is past given timestamp - if (sstable.getCreationTimeFor(Component.DATA) > skipIfOlderThanTimestamp) + if (sstable.getDataCreationTime() > skipIfOlderThanTimestamp) return false; TableMetadata metadata = cfs.metadata.get(); @@ -598,8 +619,8 @@ public class CompactionManager implements CompactionManagerMBean { logger.debug("Skipping {} ([{}, {}]) for cleanup; all rows should be kept. Needs cleanup full ranges: {} Needs cleanup transient ranges: {} Repaired: {}", sstable, - sstable.first.getToken(), - sstable.last.getToken(), + sstable.getFirst().getToken(), + sstable.getLast().getToken(), needsCleanupFull, needsCleanupTransient, sstable.isRepaired()); @@ -862,7 +883,7 @@ public class CompactionManager implements CompactionManagerMBean List> normalizedRanges = Range.normalize(ranges.ranges()); for (SSTableReader sstable : sstables) { - Bounds bounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken()); + AbstractBounds bounds = sstable.getBounds(); if (!Iterables.any(normalizedRanges, r -> (r.contains(bounds.left) && r.contains(bounds.right)) || r.intersects(bounds))) { @@ -884,12 +905,12 @@ public class CompactionManager implements CompactionManagerMBean { SSTableReader sstable = sstableIterator.next(); - Bounds sstableBounds = new Bounds<>(sstable.first.getToken(), sstable.last.getToken()); + AbstractBounds sstableBounds = sstable.getBounds(); for (Range r : normalizedRanges) { // ranges are normalized - no wrap around - if first and last are contained we know that all tokens are contained in the range - if (r.contains(sstable.first.getToken()) && r.contains(sstable.last.getToken())) + if (r.contains(sstable.getFirst().getToken()) && r.contains(sstable.getLast().getToken())) { logger.info("{} SSTable {} fully contained in range {}, mutating repairedAt instead of anticompacting", PreviewKind.NONE.logPrefix(parentRepairSession), sstable, r); fullyContainedSSTables.add(sstable); @@ -1005,7 +1026,7 @@ public class CompactionManager implements CompactionManagerMBean { forceCompaction(cfStore, () -> sstablesInBounds(cfStore, ranges), - sstable -> new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges)); + sstable -> sstable.getBounds().intersects(ranges)); } /** @@ -1043,24 +1064,12 @@ public class CompactionManager implements CompactionManagerMBean public void forceCompactionForKey(ColumnFamilyStore cfStore, DecoratedKey key) { - forceCompaction(cfStore, () -> sstablesWithKey(cfStore, key), sstable -> sstable.maybePresent(key)); + forceCompaction(cfStore, () -> sstablesWithKey(cfStore, key), Predicates.alwaysTrue()); } public void forceCompactionForKeys(ColumnFamilyStore cfStore, Collection keys) { - com.google.common.base.Predicate predicate = sstable -> { - for (DecoratedKey key : keys) - { - if(sstable.maybePresent(key)) - { - return true; - } - } - - return false; - }; - - forceCompaction(cfStore, () -> sstablesWithKeys(cfStore, keys), predicate); + forceCompaction(cfStore, () -> sstablesWithKeys(cfStore, keys), Predicates.alwaysTrue()); } private static Collection sstablesWithKey(ColumnFamilyStore cfs, DecoratedKey key) @@ -1070,7 +1079,7 @@ public class CompactionManager implements CompactionManagerMBean key.getToken().maxKeyBound()); for (SSTableReader sstable : liveTables) { - if (sstable.maybePresent(key)) + if (sstable.mayContainAssumingKeyIsInRange(key)) sstables.add(sstable); } return sstables.isEmpty() ? Collections.emptyList() : sstables; @@ -1096,7 +1105,7 @@ public class CompactionManager implements CompactionManagerMBean for (String filename : filenames) { // extract keyspace and columnfamily name from filename - Descriptor desc = Descriptor.fromFilename(filename.trim()); + Descriptor desc = Descriptor.fromFileWithComponent(new File(filename.trim()), false).left; if (Schema.instance.getTableMetadataRef(desc) == null) { logger.warn("Schema does not exist for file {}. Skipping.", filename); @@ -1122,7 +1131,7 @@ public class CompactionManager implements CompactionManagerMBean for (String filename : filenames) { // extract keyspace and columnfamily name from filename - Descriptor desc = Descriptor.fromFilename(filename.trim()); + Descriptor desc = Descriptor.fromFileWithComponent(new File(filename.trim()), false).left; if (Schema.instance.getTableMetadataRef(desc) == null) { logger.warn("Schema does not exist for file {}. Skipping.", filename); @@ -1243,11 +1252,11 @@ public class CompactionManager implements CompactionManagerMBean } @VisibleForTesting - void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, ActiveCompactionsTracker activeCompactions) + void scrubOne(ColumnFamilyStore cfs, LifecycleTransaction modifier, IScrubber.Options options, ActiveCompactionsTracker activeCompactions) { CompactionInfo.Holder scrubInfo = null; - - try (Scrubber scrubber = new Scrubber(cfs, modifier, skipCorrupted, checkData, reinsertOverflowedTTL)) + SSTableFormat format = modifier.onlyOne().descriptor.getFormat(); + try (IScrubber scrubber = format.getScrubber(cfs, modifier, new OutputHandler.LogOutput(), options)) { scrubInfo = scrubber.getScrubInfo(); activeCompactions.beginCompaction(scrubInfo); @@ -1261,11 +1270,10 @@ public class CompactionManager implements CompactionManagerMBean } @VisibleForTesting - void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, Verifier.Options options, ActiveCompactionsTracker activeCompactions) + void verifyOne(ColumnFamilyStore cfs, SSTableReader sstable, IVerifier.Options options, ActiveCompactionsTracker activeCompactions) { CompactionInfo.Holder verifyInfo = null; - - try (Verifier verifier = new Verifier(cfs, sstable, false, options)) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, options)) { verifyInfo = verifier.getVerifyInfo(); activeCompactions.beginCompaction(verifyInfo); @@ -1296,7 +1304,7 @@ public class CompactionManager implements CompactionManagerMBean // see if there are any keys LTE the token for the start of the first range // (token range ownership is exclusive on the LHS.) Range firstRange = sortedRanges.get(0); - if (sstable.first.getToken().compareTo(firstRange.left) <= 0) + if (sstable.getFirst().getToken().compareTo(firstRange.left) <= 0) return true; // then, iterate over all owned ranges and see if the next key beyond the end of the owned @@ -1352,11 +1360,11 @@ public class CompactionManager implements CompactionManagerMBean SSTableReader sstable = txn.onlyOne(); // if ranges is empty and no index, entire sstable is discarded - if (!hasIndexes && !new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(allRanges)) + if (!hasIndexes && !sstable.getBounds().intersects(allRanges)) { txn.obsoleteOriginals(); txn.finish(); - logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.first.getToken(), sstable.last.getToken(), allRanges); + logger.info("SSTable {} ([{}, {}]) does not intersect the owned ranges ({}), dropping it", sstable, sstable.getFirst().getToken(), sstable.getLast().getToken(), allRanges); return; } @@ -1556,16 +1564,18 @@ public class CompactionManager implements CompactionManagerMBean { FileUtils.createDirectory(compactionFileLocation); - return SSTableWriter.create(cfs.metadata, - cfs.newSSTableDescriptor(compactionFileLocation), - expectedBloomFilterSize, - repairedAt, - pendingRepair, - isTransient, - sstable.getSSTableLevel(), - sstable.header, - cfs.indexManager.listIndexes(), - txn); + Descriptor descriptor = cfs.newSSTableDescriptor(compactionFileLocation); + return descriptor.getFormat().getWriterFactory().builder(descriptor) + .setKeyCount(expectedBloomFilterSize) + .setRepairedAt(repairedAt) + .setPendingRepair(pendingRepair) + .setTransientSSTable(isTransient) + .setTableMetadataRef(cfs.metadata) + .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel())) + .setSerializationHeader(sstable.header) + .addDefaultComponents() + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType()) + .build(txn, cfs); } public static SSTableWriter createWriterForAntiCompaction(ColumnFamilyStore cfs, @@ -1593,16 +1603,19 @@ public class CompactionManager implements CompactionManagerMBean break; } } - return SSTableWriter.create(cfs.newSSTableDescriptor(compactionFileLocation), - (long) expectedBloomFilterSize, - repairedAt, - pendingRepair, - isTransient, - cfs.metadata, - new MetadataCollector(sstables, cfs.metadata().comparator, minLevel), - SerializationHeader.make(cfs.metadata(), sstables), - cfs.indexManager.listIndexes(), - txn); + + Descriptor descriptor = cfs.newSSTableDescriptor(compactionFileLocation); + return descriptor.getFormat().getWriterFactory().builder(descriptor) + .setKeyCount(expectedBloomFilterSize) + .setRepairedAt(repairedAt) + .setPendingRepair(pendingRepair) + .setTransientSSTable(isTransient) + .setTableMetadataRef(cfs.metadata) + .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator, minLevel)) + .setSerializationHeader(SerializationHeader.make(cfs.metadata(), sstables)) + .addDefaultComponents() + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType()) + .build(txn, cfs); } /** @@ -1894,22 +1907,16 @@ public class CompactionManager implements CompactionManagerMBean return executor.submitIfRunning(runnable, "cache write"); } - public List runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution) throws IOException + public T runAsActiveCompaction(Holder activeCompactionInfo, ThrowingSupplier callable) throws E { - return runIndexSummaryRedistribution(redistribution, active); - } - - @VisibleForTesting - List runIndexSummaryRedistribution(IndexSummaryRedistribution redistribution, ActiveCompactionsTracker activeCompactions) throws IOException - { - activeCompactions.beginCompaction(redistribution); + active.beginCompaction(activeCompactionInfo); try { - return redistribution.redistributeSummaries(); + return callable.get(); } finally { - activeCompactions.finishCompaction(redistribution); + active.finishCompaction(activeCompactionInfo); } } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index bc0fc0dc81..72645f223d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -58,11 +58,11 @@ import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableMultiWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; @@ -252,8 +252,8 @@ public class CompactionStrategyManager implements INotificationConsumer .stream() .filter(s -> !compacting.contains(s) && !s.descriptor.version.isLatestVersion()) .sorted((o1, o2) -> { - File f1 = new File(o1.descriptor.filenameFor(Component.DATA)); - File f2 = new File(o2.descriptor.filenameFor(Component.DATA)); + File f1 = o1.descriptor.fileFor(Components.DATA); + File f2 = o2.descriptor.fileFor(Components.DATA); return Longs.compare(f1.lastModified(), f2.lastModified()); }).collect(Collectors.toList()); for (SSTableReader sstable : potentialUpgrade) diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 81ea5e5b9d..4185200653 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -241,7 +241,7 @@ public class CompactionTask extends AbstractCompactionTask StringBuilder newSSTableNames = new StringBuilder(); for (SSTableReader reader : newSStables) - newSSTableNames.append(reader.descriptor.baseFilename()).append(","); + newSSTableNames.append(reader.descriptor.baseFile()).append(","); long totalSourceRows = 0; for (int i = 0; i < mergedRowCounts.length; i++) totalSourceRows += mergedRowCounts[i] * (i + 1); diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index ecf0915cb1..4339181411 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -17,7 +17,16 @@ */ package org.apache.cassandra.db.compaction; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; @@ -27,18 +36,16 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; - -import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Pair; @@ -115,7 +122,7 @@ public class LeveledManifest long maxModificationTime = Long.MIN_VALUE; for (SSTableReader ssTableReader : level) { - long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA); + long modificationTime = ssTableReader.getDataCreationTime(); if (modificationTime >= maxModificationTime) { sstableWithMaxModificationTime = ssTableReader; diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java deleted file mode 100644 index 8646682b7e..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ /dev/null @@ -1,864 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import java.io.IOError; -import java.io.IOException; -import java.nio.ByteBuffer; - -import java.nio.file.Paths; -import java.util.*; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.*; -import org.apache.cassandra.utils.concurrent.Refs; -import org.apache.cassandra.utils.memory.HeapCloner; - -import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; - -public class Scrubber implements Closeable -{ - private final ColumnFamilyStore cfs; - private final SSTableReader sstable; - private final LifecycleTransaction transaction; - private final File destination; - private final boolean skipCorrupted; - private final boolean reinsertOverflowedTTLRows; - - private final boolean isCommutative; - private final boolean isIndex; - private final boolean checkData; - private final long expectedBloomFilterSize; - - private final ReadWriteLock fileAccessLock; - private final RandomAccessReader dataFile; - private final RandomAccessReader indexFile; - private final ScrubInfo scrubInfo; - private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; - - private int goodPartitions; - private int badPartitions; - private int emptyPartitions; - - private ByteBuffer currentIndexKey; - private ByteBuffer nextIndexKey; - private long currentPartitionPositionFromIndex; - private long nextPartitionPositionFromIndex; - - private NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics = new NegativeLocalDeletionInfoMetrics(); - - private final OutputHandler outputHandler; - - private static final Comparator partitionComparator = Comparator.comparing(Partition::partitionKey); - private final SortedSet outOfOrder = new TreeSet<>(partitionComparator); - - public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData) - { - this(cfs, transaction, skipCorrupted, checkData, false); - } - - public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean checkData, - boolean reinsertOverflowedTTLRows) - { - this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), checkData, reinsertOverflowedTTLRows); - } - - @SuppressWarnings("resource") - public Scrubber(ColumnFamilyStore cfs, - LifecycleTransaction transaction, - boolean skipCorrupted, - OutputHandler outputHandler, - boolean checkData, - boolean reinsertOverflowedTTLRows) - { - this.cfs = cfs; - this.transaction = transaction; - this.sstable = transaction.onlyOne(); - this.outputHandler = outputHandler; - this.skipCorrupted = skipCorrupted; - this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(), - sstable.descriptor.version, - sstable.header); - - List toScrub = Collections.singletonList(sstable); - - this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable)); - this.isCommutative = cfs.metadata().isCounter(); - - boolean hasIndexFile = (new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))).exists(); - this.isIndex = cfs.isIndex(); - if (!hasIndexFile) - { - // if there's any corruption in the -Data.db then partitions can't be skipped over. but it's worth a shot. - outputHandler.warn("Missing component: " + sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)); - } - this.checkData = checkData && !this.isIndex; //LocalByPartitionerType does not support validation - this.expectedBloomFilterSize = Math.max( - cfs.metadata().params.minIndexInterval, - hasIndexFile ? SSTableReader.getApproximateKeyCount(toScrub) : 0); - - this.fileAccessLock = new ReentrantReadWriteLock(); - // loop through each partition, deserializing to check for damage. - // We'll also loop through the index at the same time, using the position from the index to recover if the - // partition header (key or data size) is corrupt. (This means our position in the index file will be one - // partition "ahead" of the data file.) - this.dataFile = transaction.isOffline() - ? sstable.openDataReader() - : sstable.openDataReader(CompactionManager.instance.getRateLimiter()); - - this.indexFile = hasIndexFile - ? RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))) - : null; - - this.scrubInfo = new ScrubInfo(dataFile, sstable, fileAccessLock.readLock()); - - this.currentPartitionPositionFromIndex = 0; - this.nextPartitionPositionFromIndex = 0; - - if (reinsertOverflowedTTLRows) - outputHandler.output("Starting scrub with reinsert overflowed TTL option"); - } - - private UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename) - { - return checkData ? UnfilteredRowIterators.withValidation(iter, filename) : iter; - } - - private String keyString(DecoratedKey key) - { - if (key == null) - return "(unknown)"; - - try - { - return cfs.metadata().partitionKeyType.getString(key.getKey()); - } - catch (Exception e) - { - return String.format("(corrupted; hex value: %s)", ByteBufferUtil.bytesToHex(key.getKey())); - } - } - - public void scrub() - { - List finished = new ArrayList<>(); - outputHandler.output(String.format("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length()))); - try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge); - Refs refs = Refs.ref(Collections.singleton(sstable))) - { - nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null; - if (indexAvailable()) - { - // throw away variable so we don't have a side effect in the assert - long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex; - } - - StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)); - - DecoratedKey prevKey = null; - - while (!dataFile.isEOF()) - { - if (scrubInfo.isStopRequested()) - throw new CompactionInterruptedException(scrubInfo.getCompactionInfo()); - - long partitionStart = dataFile.getFilePointer(); - outputHandler.debug("Reading row at " + partitionStart); - - DecoratedKey key = null; - try - { - ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile); - if (!cfs.metadata.getLocal().isIndex()) - cfs.metadata.getLocal().partitionKeyType.validate(raw); - key = sstable.decorateKey(raw); - } - catch (Throwable th) - { - throwIfFatal(th); - // check for null key below - } - - updateIndexKey(); - - long dataStart = dataFile.getFilePointer(); - - long dataStartFromIndex = -1; - long dataSizeFromIndex = -1; - if (currentIndexKey != null) - { - dataStartFromIndex = currentPartitionPositionFromIndex + 2 + currentIndexKey.remaining(); - dataSizeFromIndex = nextPartitionPositionFromIndex - dataStartFromIndex; - } - - String keyName = key == null ? "(unreadable key)" : keyString(key); - outputHandler.debug(String.format("partition %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex))); - assert currentIndexKey != null || !indexAvailable(); - - try - { - if (key == null) - throw new IOError(new IOException("Unable to read partition key from data file")); - - if (currentIndexKey != null && !key.getKey().equals(currentIndexKey)) - { - throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)", - //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey)))); - "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey)))); - } - - if (indexFile != null && dataSizeFromIndex > dataFile.length()) - throw new IOError(new IOException("Impossible partition size (greater than file length): " + dataSizeFromIndex)); - - if (indexFile != null && dataStart != dataStartFromIndex) - outputHandler.warn(String.format("Data file partition position %d differs from index file row position %d", dataStart, dataStartFromIndex)); - - if (tryAppend(prevKey, key, writer)) - prevKey = key; - } - catch (Throwable th) - { - throwIfFatal(th); - outputHandler.warn(String.format("Error reading partition %s (stacktrace follows):", keyName), th); - - if (currentIndexKey != null - && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex)) - { - - outputHandler.output(String.format("Retrying from partition index; data is %s bytes starting at %s", - dataSizeFromIndex, dataStartFromIndex)); - key = sstable.decorateKey(currentIndexKey); - try - { - if (!cfs.metadata.getLocal().isIndex()) - cfs.metadata.getLocal().partitionKeyType.validate(key.getKey()); - dataFile.seek(dataStartFromIndex); - - if (tryAppend(prevKey, key, writer)) - prevKey = key; - } - catch (Throwable th2) - { - throwIfFatal(th2); - throwIfCannotContinue(key, th2); - - outputHandler.warn("Retry failed too. Skipping to next partition (retry's stacktrace follows)", th2); - badPartitions++; - if (!seekToNextPartition()) - break; - } - } - else - { - throwIfCannotContinue(key, th); - - outputHandler.warn("Partition starting at position " + dataStart + " is unreadable; skipping to next"); - badPartitions++; - if (currentIndexKey != null) - if (!seekToNextPartition()) - break; - } - } - } - - if (!outOfOrder.isEmpty()) - { - // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time - long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt; - SSTableReader newInOrderSstable; - try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)) - { - for (Partition partition : outOfOrder) - inOrderWriter.append(partition.unfilteredIterator()); - newInOrderSstable = inOrderWriter.finish(-1, sstable.maxDataAge, true); - } - transaction.update(newInOrderSstable, false); - finished.add(newInOrderSstable); - outputHandler.warn(String.format("%d out of order partition (or partitions with out of order rows) found while scrubbing %s; " + - "Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable)); - } - - // finish obsoletes the old sstable - finished.addAll(writer.setRepairedAt(badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt).finish()); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - finally - { - if (transaction.isOffline()) - finished.forEach(sstable -> sstable.selfRef().release()); - } - - if (!finished.isEmpty()) - { - outputHandler.output("Scrub of " + sstable + " complete: " + goodPartitions + " partitions in new sstable and " + emptyPartitions + " empty (tombstoned) partitions dropped"); - if (negativeLocalDeletionInfoMetrics.fixedRows > 0) - outputHandler.output("Fixed " + negativeLocalDeletionInfoMetrics.fixedRows + " rows with overflowed local deletion time."); - if (badPartitions > 0) - outputHandler.warn("Unable to recover " + badPartitions + " partitions that were skipped. You can attempt manual recovery from the pre-scrub snapshot. You can also run nodetool repair to transfer the data from a healthy replica, if any"); - } - else - { - if (badPartitions > 0) - outputHandler.warn("No valid partitions found while scrubbing " + sstable + "; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot"); - else - outputHandler.output("Scrub of " + sstable + " complete; looks like all " + emptyPartitions + " partitions were tombstoned"); - } - } - - @SuppressWarnings("resource") - private boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer) - { - // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects - // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added - // to the outOfOrder set that will be later written to a new SSTable. - OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), - cfs.metadata().comparator); - - try (UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath())) - { - if (prevKey != null && prevKey.compareTo(key) > 0) - { - saveOutOfOrderPartition(prevKey, key, iterator); - return false; - } - - if (writer.tryAppend(iterator) == null) - emptyPartitions++; - else - goodPartitions++; - } - - if (sstableIterator.hasRowsOutOfOrder()) - { - outputHandler.warn(String.format("Out of order rows found in partition: %s", keyString(key))); - outOfOrder.add(sstableIterator.getRowsOutOfOrder()); - } - - return true; - } - - /** - * Only wrap with {@link FixNegativeLocalDeletionTimeIterator} if {@link #reinsertOverflowedTTLRows} option - * is specified - */ - @SuppressWarnings("resource") - private UnfilteredRowIterator getIterator(DecoratedKey key) - { - RowMergingSSTableIterator rowMergingIterator = new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key), outputHandler); - return reinsertOverflowedTTLRows ? new FixNegativeLocalDeletionTimeIterator(rowMergingIterator, - outputHandler, - negativeLocalDeletionInfoMetrics) : rowMergingIterator; - } - - private void updateIndexKey() - { - currentIndexKey = nextIndexKey; - currentPartitionPositionFromIndex = nextPartitionPositionFromIndex; - try - { - nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile); - - nextPartitionPositionFromIndex = !indexAvailable() - ? dataFile.length() - : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - } - catch (Throwable th) - { - JVMStabilityInspector.inspectThrowable(th); - outputHandler.warn("Error reading index file", th); - nextIndexKey = null; - nextPartitionPositionFromIndex = dataFile.length(); - } - } - - private boolean indexAvailable() - { - return indexFile != null && !indexFile.isEOF(); - } - - private boolean seekToNextPartition() - { - while(nextPartitionPositionFromIndex < dataFile.length()) - { - try - { - dataFile.seek(nextPartitionPositionFromIndex); - return true; - } - catch (Throwable th) - { - throwIfFatal(th); - outputHandler.warn(String.format("Failed to seek to next partition position %d", nextPartitionPositionFromIndex), th); - badPartitions++; - } - - updateIndexKey(); - } - - return false; - } - - private void saveOutOfOrderPartition(DecoratedKey prevKey, DecoratedKey key, UnfilteredRowIterator iterator) - { - // TODO bitch if the row is too large? if it is there's not much we can do ... - outputHandler.warn(String.format("Out of order partition detected (%s found after %s)", - keyString(key), keyString(prevKey))); - outOfOrder.add(ImmutableBTreePartition.create(iterator)); - } - - private void throwIfFatal(Throwable th) - { - if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError)) - throw (Error) th; - } - - private void throwIfCannotContinue(DecoratedKey key, Throwable th) - { - if (isIndex) - { - outputHandler.warn(String.format("An error occurred while scrubbing the partition with key '%s' for an index table. " + - "Scrubbing will abort for this table and the index will be rebuilt.", keyString(key))); - throw new IOError(th); - } - - if (isCommutative && !skipCorrupted) - { - outputHandler.warn(String.format("An error occurred while scrubbing the partition with key '%s'. Skipping corrupt " + - "data in counter tables will result in undercounts for the affected " + - "counters (see CASSANDRA-2759 for more details), so by default the scrub will " + - "stop at this point. If you would like to skip the row anyway and continue " + - "scrubbing, re-run the scrub with the --skip-corrupted option.", - keyString(key))); - throw new IOError(th); - } - } - - public void close() - { - fileAccessLock.writeLock().lock(); - try - { - FileUtils.closeQuietly(dataFile); - FileUtils.closeQuietly(indexFile); - } - finally - { - fileAccessLock.writeLock().unlock(); - } - } - - public CompactionInfo.Holder getScrubInfo() - { - return scrubInfo; - } - - private static class ScrubInfo extends CompactionInfo.Holder - { - private final RandomAccessReader dataFile; - private final SSTableReader sstable; - private final TimeUUID scrubCompactionId; - private final Lock fileReadLock; - - public ScrubInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock) - { - this.dataFile = dataFile; - this.sstable = sstable; - this.fileReadLock = fileReadLock; - scrubCompactionId = nextTimeUUID(); - } - - public CompactionInfo getCompactionInfo() - { - fileReadLock.lock(); - try - { - return new CompactionInfo(sstable.metadata(), - OperationType.SCRUB, - dataFile.getFilePointer(), - dataFile.length(), - scrubCompactionId, - ImmutableSet.of(sstable), - Paths.get(sstable.getFilename()).getParent().toString()); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - finally - { - fileReadLock.unlock(); - } - } - - public boolean isGlobal() - { - return false; - } - } - - @VisibleForTesting - public ScrubResult scrubWithResult() - { - scrub(); - return new ScrubResult(this); - } - - public static final class ScrubResult - { - public final int goodPartitions; - public final int badPartitions; - public final int emptyPartitions; - - public ScrubResult(Scrubber scrubber) - { - this.goodPartitions = scrubber.goodPartitions; - this.badPartitions = scrubber.badPartitions; - this.emptyPartitions = scrubber.emptyPartitions; - } - } - - public class NegativeLocalDeletionInfoMetrics - { - public volatile int fixedRows = 0; - } - - /** - * During 2.x migration, under some circumstances rows might have gotten duplicated. - * Merging iterator merges rows with same clustering. - * - * For more details, refer to CASSANDRA-12144. - */ - private static class RowMergingSSTableIterator extends WrappingUnfilteredRowIterator - { - Unfiltered nextToOffer = null; - private final OutputHandler output; - - RowMergingSSTableIterator(UnfilteredRowIterator source, OutputHandler output) - { - super(source); - this.output = output; - } - - @Override - public boolean hasNext() - { - return nextToOffer != null || wrapped.hasNext(); - } - - @Override - public Unfiltered next() - { - Unfiltered next = nextToOffer != null ? nextToOffer : wrapped.next(); - - if (next.isRow()) - { - boolean logged = false; - while (wrapped.hasNext()) - { - Unfiltered peek = wrapped.next(); - if (!peek.isRow() || !next.clustering().equals(peek.clustering())) - { - nextToOffer = peek; // Offer peek in next call - return next; - } - - // Duplicate row, merge it. - next = Rows.merge((Row) next, (Row) peek); - - if (!logged) - { - String partitionKey = metadata().partitionKeyType.getString(partitionKey().getKey()); - output.warn("Duplicate row detected in " + metadata().keyspace + '.' + metadata().name + ": " + partitionKey + " " + next.clustering().toString(metadata())); - logged = true; - } - } - } - - nextToOffer = null; - return next; - } - } - - /** - * In some case like CASSANDRA-12127 the cells might have been stored in the wrong order. This decorator check the - * cells order and collect the out of order cells to correct the problem. - */ - private static final class OrderCheckerIterator extends AbstractIterator implements UnfilteredRowIterator - { - /** - * The decorated iterator. - */ - private final UnfilteredRowIterator iterator; - - private final ClusteringComparator comparator; - - private Unfiltered previous; - - /** - * The partition containing the rows which are out of order. - */ - private Partition rowsOutOfOrder; - - public OrderCheckerIterator(UnfilteredRowIterator iterator, ClusteringComparator comparator) - { - this.iterator = iterator; - this.comparator = comparator; - } - - public TableMetadata metadata() - { - return iterator.metadata(); - } - - public boolean isReverseOrder() - { - return iterator.isReverseOrder(); - } - - public RegularAndStaticColumns columns() - { - return iterator.columns(); - } - - public DecoratedKey partitionKey() - { - return iterator.partitionKey(); - } - - public Row staticRow() - { - return iterator.staticRow(); - } - - @Override - public boolean isEmpty() - { - return iterator.isEmpty(); - } - - public void close() - { - iterator.close(); - } - - public DeletionTime partitionLevelDeletion() - { - return iterator.partitionLevelDeletion(); - } - - public EncodingStats stats() - { - return iterator.stats(); - } - - public boolean hasRowsOutOfOrder() - { - return rowsOutOfOrder != null; - } - - public Partition getRowsOutOfOrder() - { - return rowsOutOfOrder; - } - - protected Unfiltered computeNext() - { - if (!iterator.hasNext()) - return endOfData(); - - Unfiltered next = iterator.next(); - - // If we detect that some rows are out of order we will store and sort the remaining ones to insert them - // in a separate SSTable. - if (previous != null && comparator.compare(next, previous) < 0) - { - rowsOutOfOrder = ImmutableBTreePartition.create(UnfilteredRowIterators.concat(next, iterator), false); - return endOfData(); - } - previous = next; - return next; - } - } - - /** - * This iterator converts negative {@link AbstractCell#localDeletionTime()} into {@link AbstractCell#MAX_DELETION_TIME} - * - * This is to recover entries with overflowed localExpirationTime due to CASSANDRA-14092 - */ - private static final class FixNegativeLocalDeletionTimeIterator extends AbstractIterator implements UnfilteredRowIterator - { - /** - * The decorated iterator. - */ - private final UnfilteredRowIterator iterator; - - private final OutputHandler outputHandler; - private final NegativeLocalDeletionInfoMetrics negativeLocalExpirationTimeMetrics; - - public FixNegativeLocalDeletionTimeIterator(UnfilteredRowIterator iterator, OutputHandler outputHandler, - NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics) - { - this.iterator = iterator; - this.outputHandler = outputHandler; - this.negativeLocalExpirationTimeMetrics = negativeLocalDeletionInfoMetrics; - } - - public TableMetadata metadata() - { - return iterator.metadata(); - } - - public boolean isReverseOrder() - { - return iterator.isReverseOrder(); - } - - public RegularAndStaticColumns columns() - { - return iterator.columns(); - } - - public DecoratedKey partitionKey() - { - return iterator.partitionKey(); - } - - public Row staticRow() - { - return iterator.staticRow(); - } - - @Override - public boolean isEmpty() - { - return iterator.isEmpty(); - } - - public void close() - { - iterator.close(); - } - - public DeletionTime partitionLevelDeletion() - { - return iterator.partitionLevelDeletion(); - } - - public EncodingStats stats() - { - return iterator.stats(); - } - - protected Unfiltered computeNext() - { - if (!iterator.hasNext()) - return endOfData(); - - Unfiltered next = iterator.next(); - if (!next.isRow()) - return next; - - if (hasNegativeLocalExpirationTime((Row) next)) - { - outputHandler.debug(String.format("Found row with negative local expiration time: %s", next.toString(metadata(), false))); - negativeLocalExpirationTimeMetrics.fixedRows++; - return fixNegativeLocalExpirationTime((Row) next); - } - - return next; - } - - private boolean hasNegativeLocalExpirationTime(Row next) - { - Row row = next; - if (row.primaryKeyLivenessInfo().isExpiring() && row.primaryKeyLivenessInfo().localExpirationTime() < 0) - { - return true; - } - - for (ColumnData cd : row) - { - if (cd.column().isSimple()) - { - Cell cell = (Cell)cd; - if (cell.isExpiring() && cell.localDeletionTime() < 0) - return true; - } - else - { - ComplexColumnData complexData = (ComplexColumnData)cd; - for (Cell cell : complexData) - { - if (cell.isExpiring() && cell.localDeletionTime() < 0) - return true; - } - } - } - - return false; - } - - private Unfiltered fixNegativeLocalExpirationTime(Row row) - { - LivenessInfo livenessInfo = row.primaryKeyLivenessInfo(); - if (livenessInfo.isExpiring() && livenessInfo.localExpirationTime() < 0) - livenessInfo = livenessInfo.withUpdatedTimestampAndLocalDeletionTime(livenessInfo.timestamp() + 1, AbstractCell.MAX_DELETION_TIME); - - return row.transformAndFilter(livenessInfo, row.deletion(), cd -> { - if (cd.column().isSimple()) - { - Cell cell = (Cell)cd; - return cell.isExpiring() && cell.localDeletionTime() < 0 - ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME) - : cell; - } - else - { - ComplexColumnData complexData = (ComplexColumnData)cd; - return complexData.transformAndFilter(cell -> cell.isExpiring() && cell.localDeletionTime() < 0 - ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME) - : cell); - } - }).clone(HeapCloner.instance); - } - } -} diff --git a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java index 2e1dffc522..1f73c4cd30 100644 --- a/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java +++ b/src/java/org/apache/cassandra/db/compaction/SingleSSTableLCSTask.java @@ -27,8 +27,8 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; @@ -82,7 +82,7 @@ public class SingleSSTableLCSTask extends AbstractCompactionTask catch (Throwable t) { transaction.abort(); - throw new CorruptSSTableException(t, sstable.descriptor.filenameFor(Component.DATA)); + throw new CorruptSSTableException(t, sstable.descriptor.fileFor(Components.DATA)); } cfs.getTracker().notifySSTableMetadataChanged(sstable, metadataBefore); } diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java index 132a99cc68..f18cc4995c 100644 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@ -17,7 +17,7 @@ */ package org.apache.cassandra.db.compaction; -import java.util.*; +import java.util.Arrays; import java.util.function.LongPredicate; import com.google.common.base.Throwables; @@ -25,9 +25,10 @@ import com.google.common.collect.Sets; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableRewriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -72,16 +73,19 @@ public class Upgrader { MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator()); sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel()); - return SSTableWriter.create(cfs.newSSTableDescriptor(directory), - estimatedRows, - metadata.repairedAt, - metadata.pendingRepair, - metadata.isTransient, - cfs.metadata, - sstableMetadataCollector, - SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)), - cfs.indexManager.listIndexes(), - transaction); + + Descriptor descriptor = cfs.newSSTableDescriptor(directory); + return descriptor.getFormat().getWriterFactory().builder(descriptor) + .setKeyCount(estimatedRows) + .setRepairedAt(metadata.repairedAt) + .setPendingRepair(metadata.pendingRepair) + .setTransientSSTable(metadata.isTransient) + .setTableMetadataRef(cfs.metadata) + .setMetadataCollector(sstableMetadataCollector) + .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable))) + .addDefaultComponents() + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), transaction.opType()) + .build(transaction, cfs); } public void upgrade(boolean keepOriginals) diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java deleted file mode 100644 index a67593bf55..0000000000 --- a/src/java/org/apache/cassandra/db/compaction/Verifier.java +++ /dev/null @@ -1,726 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.compaction; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableSet; - -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.Cell; -import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.db.rows.Unfiltered; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; - -import org.apache.cassandra.dht.LocalPartitioner; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.IndexSummary; -import org.apache.cassandra.io.sstable.KeyIterator; -import org.apache.cassandra.io.sstable.SSTableIdentityIterator; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; -import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; -import org.apache.cassandra.io.util.DataIntegrityMetadata; -import org.apache.cassandra.io.util.DataIntegrityMetadata.FileDigestValidator; -import org.apache.cassandra.io.util.FileInputStreamPlus; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.BloomFilterSerializer; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.IFilter; -import org.apache.cassandra.utils.OutputHandler; -import org.apache.cassandra.utils.TimeUUID; - -import java.io.Closeable; -import java.io.DataInputStream; -import java.io.IOError; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; -import java.util.function.LongPredicate; - -import org.apache.cassandra.io.util.File; - -import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; - -public class Verifier implements Closeable -{ - private final ColumnFamilyStore cfs; - private final SSTableReader sstable; - - private final CompactionController controller; - - private final ReadWriteLock fileAccessLock; - private final RandomAccessReader dataFile; - private final RandomAccessReader indexFile; - private final VerifyInfo verifyInfo; - private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; - private final Options options; - private final boolean isOffline; - /** - * Given a keyspace, return the set of local and pending token ranges. By default {@link StorageService#getLocalAndPendingRanges(String)} - * is expected, but for the standalone verifier case we can't use that, so this is here to allow the CLI to provide - * the token ranges. - */ - private final Function>> tokenLookup; - - private int goodRows; - - private final OutputHandler outputHandler; - private FileDigestValidator validator; - - public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, boolean isOffline, Options options) - { - this(cfs, sstable, new OutputHandler.LogOutput(), isOffline, options); - } - - public Verifier(ColumnFamilyStore cfs, SSTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options) - { - this.cfs = cfs; - this.sstable = sstable; - this.outputHandler = outputHandler; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(cfs.metadata(), sstable.descriptor.version, sstable.header); - - this.controller = new VerifyController(cfs); - - this.fileAccessLock = new ReentrantReadWriteLock(); - this.dataFile = isOffline - ? sstable.openDataReader() - : sstable.openDataReader(CompactionManager.instance.getRateLimiter()); - this.indexFile = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX))); - this.verifyInfo = new VerifyInfo(dataFile, sstable, fileAccessLock.readLock()); - this.options = options; - this.isOffline = isOffline; - this.tokenLookup = options.tokenLookup; - } - - public void verify() - { - boolean extended = options.extendedVerification; - long rowStart = 0; - - outputHandler.output(String.format("Verifying %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length()))); - if (options.checkVersion && !sstable.descriptor.version.isLatestVersion()) - { - String msg = String.format("%s is not the latest version, run upgradesstables", sstable); - outputHandler.output(msg); - // don't use markAndThrow here because we don't want a CorruptSSTableException for this. - throw new RuntimeException(msg); - } - - outputHandler.output(String.format("Deserializing sstable metadata for %s ", sstable)); - try - { - EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); - Map sstableMetadata = sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, types); - if (sstableMetadata.containsKey(MetadataType.VALIDATION) && - !((ValidationMetadata)sstableMetadata.get(MetadataType.VALIDATION)).partitioner.equals(sstable.getPartitioner().getClass().getCanonicalName())) - throw new IOException("Partitioner does not match validation metadata"); - } - catch (Throwable t) - { - outputHandler.warn(t); - markAndThrow(t, false); - } - - try - { - outputHandler.debug("Deserializing index for "+sstable); - deserializeIndex(sstable); - } - catch (Throwable t) - { - outputHandler.warn(t); - markAndThrow(t); - } - - try - { - outputHandler.debug("Deserializing index summary for "+sstable); - deserializeIndexSummary(sstable); - } - catch (Throwable t) - { - outputHandler.output("Index summary is corrupt - if it is removed it will get rebuilt on startup "+sstable.descriptor.filenameFor(Component.SUMMARY)); - outputHandler.warn(t); - markAndThrow(t, false); - } - - try - { - outputHandler.debug("Deserializing bloom filter for "+sstable); - deserializeBloomFilter(sstable); - - } - catch (Throwable t) - { - outputHandler.warn(t); - markAndThrow(t); - } - - if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner() instanceof LocalPartitioner)) - { - outputHandler.debug("Checking that all tokens are owned by the current node"); - try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata())) - { - List> ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace)); - if (ownedRanges.isEmpty()) - return; - RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges); - while (iter.hasNext()) - { - DecoratedKey key = iter.next(); - rangeOwnHelper.validate(key); - } - } - catch (Throwable t) - { - outputHandler.warn(t); - markAndThrow(t); - } - } - - if (options.quick) - return; - - // Verify will use the Digest files, which works for both compressed and uncompressed sstables - outputHandler.output(String.format("Checking computed hash of %s ", sstable)); - try - { - validator = null; - - if (new File(sstable.descriptor.filenameFor(Component.DIGEST)).exists()) - { - validator = DataIntegrityMetadata.fileDigestValidator(sstable.descriptor); - validator.validate(); - } - else - { - outputHandler.output("Data digest missing, assuming extended verification of disk values"); - extended = true; - } - } - catch (IOException e) - { - outputHandler.warn(e); - markAndThrow(e); - } - finally - { - FileUtils.closeQuietly(validator); - } - - if (!extended) - return; - - outputHandler.output("Extended Verify requested, proceeding to inspect values"); - - try - { - ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); - { - long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - if (firstRowPositionFromIndex != 0) - markAndThrow(new RuntimeException("firstRowPositionFromIndex != 0: "+firstRowPositionFromIndex)); - } - - List> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace)); - RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges); - DecoratedKey prevKey = null; - - while (!dataFile.isEOF()) - { - - if (verifyInfo.isStopRequested()) - throw new CompactionInterruptedException(verifyInfo.getCompactionInfo()); - - rowStart = dataFile.getFilePointer(); - outputHandler.debug("Reading row at " + rowStart); - - DecoratedKey key = null; - try - { - key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile)); - } - catch (Throwable th) - { - throwIfFatal(th); - // check for null key below - } - - if (options.checkOwnsTokens && ownedRanges.size() > 0 && !(cfs.getPartitioner() instanceof LocalPartitioner)) - { - try - { - rangeOwnHelper.validate(key); - } - catch (Throwable t) - { - outputHandler.warn(String.format("Key %s in sstable %s not owned by local ranges %s", key, sstable, ownedRanges), t); - markAndThrow(t); - } - } - - ByteBuffer currentIndexKey = nextIndexKey; - long nextRowPositionFromIndex = 0; - try - { - nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile); - nextRowPositionFromIndex = indexFile.isEOF() - ? dataFile.length() - : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - } - catch (Throwable th) - { - markAndThrow(th); - } - - long dataStart = dataFile.getFilePointer(); - long dataStartFromIndex = currentIndexKey == null - ? -1 - : rowStart + 2 + currentIndexKey.remaining(); - - long dataSize = nextRowPositionFromIndex - dataStartFromIndex; - // avoid an NPE if key is null - String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey()); - outputHandler.debug(String.format("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize))); - - assert currentIndexKey != null || indexFile.isEOF(); - - try - { - if (key == null || dataSize > dataFile.length()) - markAndThrow(new RuntimeException(String.format("key = %s, dataSize=%d, dataFile.length() = %d", key, dataSize, dataFile.length()))); - - try (UnfilteredRowIterator iterator = SSTableIdentityIterator.create(sstable, dataFile, key)) - { - Row first = null; - int duplicateRows = 0; - long minTimestamp = Long.MAX_VALUE; - long maxTimestamp = Long.MIN_VALUE; - while (iterator.hasNext()) - { - Unfiltered uf = iterator.next(); - if (uf.isRow()) - { - Row row = (Row) uf; - if (first != null && first.clustering().equals(row.clustering())) - { - duplicateRows++; - for (Cell cell : row.cells()) - { - maxTimestamp = Math.max(cell.timestamp(), maxTimestamp); - minTimestamp = Math.min(cell.timestamp(), minTimestamp); - } - } - else - { - if (duplicateRows > 0) - logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp); - duplicateRows = 0; - first = row; - maxTimestamp = Long.MIN_VALUE; - minTimestamp = Long.MAX_VALUE; - } - } - } - if (duplicateRows > 0) - logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp); - } - - if ( (prevKey != null && prevKey.compareTo(key) > 0) || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex ) - markAndThrow(new RuntimeException("Key out of order: previous = "+prevKey + " : current = " + key)); - - goodRows++; - prevKey = key; - - - outputHandler.debug(String.format("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex)); - dataFile.seek(nextRowPositionFromIndex); - } - catch (Throwable th) - { - markAndThrow(th); - } - } - } - catch (Throwable t) - { - Throwables.throwIfUnchecked(t); - throw new RuntimeException(t); - } - finally - { - controller.close(); - } - - outputHandler.output("Verify of " + sstable + " succeeded. All " + goodRows + " rows read successfully"); - } - - private void logDuplicates(DecoratedKey key, Row first, int duplicateRows, long minTimestamp, long maxTimestamp) - { - String keyString = sstable.metadata().partitionKeyType.getString(key.getKey()); - long firstMaxTs = Long.MIN_VALUE; - long firstMinTs = Long.MAX_VALUE; - for (Cell cell : first.cells()) - { - firstMaxTs = Math.max(firstMaxTs, cell.timestamp()); - firstMinTs = Math.min(firstMinTs, cell.timestamp()); - } - outputHandler.output(String.format("%d duplicate rows found for [%s %s] in %s.%s (%s), timestamps: [first row (%s, %s)], [duplicates (%s, %s, eq:%b)]", - duplicateRows, - keyString, first.clustering().toString(sstable.metadata()), - sstable.metadata().keyspace, - sstable.metadata().name, - sstable, - dateString(firstMinTs), dateString(firstMaxTs), - dateString(minTimestamp), dateString(maxTimestamp), minTimestamp == maxTimestamp)); - } - - private String dateString(long time) - { - return Instant.ofEpochMilli(TimeUnit.MICROSECONDS.toMillis(time)).toString(); - } - - /** - * Use the fact that check(..) is called with sorted tokens - we keep a pointer in to the normalized ranges - * and only bump the pointer if the key given is out of range. This is done to avoid calling .contains(..) many - * times for each key (with vnodes for example) - */ - @VisibleForTesting - public static class RangeOwnHelper - { - private final List> normalizedRanges; - private int rangeIndex = 0; - private DecoratedKey lastKey; - - public RangeOwnHelper(List> normalizedRanges) - { - this.normalizedRanges = normalizedRanges; - Range.assertNormalized(normalizedRanges); - } - - /** - * check if the given key is contained in any of the given ranges - * - * Must be called in sorted order - key should be increasing - * - * @param key the key - * @throws RuntimeException if the key is not contained - */ - public void validate(DecoratedKey key) - { - if (!check(key)) - throw new RuntimeException("Key " + key + " is not contained in the given ranges"); - } - - /** - * check if the given key is contained in any of the given ranges - * - * Must be called in sorted order - key should be increasing - * - * @param key the key - * @return boolean - */ - public boolean check(DecoratedKey key) - { - assert lastKey == null || key.compareTo(lastKey) > 0; - lastKey = key; - - if (normalizedRanges.isEmpty()) // handle tests etc where we don't have any ranges - return true; - - if (rangeIndex > normalizedRanges.size() - 1) - throw new IllegalStateException("RangeOwnHelper can only be used to find the first out-of-range-token"); - - while (!normalizedRanges.get(rangeIndex).contains(key.getToken())) - { - rangeIndex++; - if (rangeIndex > normalizedRanges.size() - 1) - return false; - } - - return true; - } - } - - private void deserializeIndex(SSTableReader sstable) throws IOException - { - try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)))) - { - long indexSize = primaryIndex.length(); - - while ((primaryIndex.getFilePointer()) != indexSize) - { - ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex); - RowIndexEntry.Serializer.skip(primaryIndex, sstable.descriptor.version); - } - } - } - - private void deserializeIndexSummary(SSTableReader sstable) throws IOException - { - File file = new File(sstable.descriptor.filenameFor(Component.SUMMARY)); - TableMetadata metadata = cfs.metadata(); - try (DataInputStream iStream = new DataInputStream(Files.newInputStream(file.toPath()))) - { - try (IndexSummary indexSummary = IndexSummary.serializer.deserialize(iStream, - cfs.getPartitioner(), - metadata.params.minIndexInterval, - metadata.params.maxIndexInterval)) - { - ByteBufferUtil.readWithLength(iStream); - ByteBufferUtil.readWithLength(iStream); - } - } - } - - private void deserializeBloomFilter(SSTableReader sstable) throws IOException - { - File bfPath = new File(sstable.descriptor.filenameFor(Component.FILTER)); - if (bfPath.exists()) - { - try (FileInputStreamPlus stream = bfPath.newInputStream(); - IFilter bf = BloomFilterSerializer.deserialize(stream, sstable.descriptor.version.hasOldBfFormat())) - { - } - } - } - - public void close() - { - fileAccessLock.writeLock().lock(); - try - { - FileUtils.closeQuietly(dataFile); - FileUtils.closeQuietly(indexFile); - } - finally - { - fileAccessLock.writeLock().unlock(); - } - } - - private void throwIfFatal(Throwable th) - { - if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError)) - throw (Error) th; - } - - private void markAndThrow(Throwable cause) - { - markAndThrow(cause, true); - } - - private void markAndThrow(Throwable cause, boolean mutateRepaired) - { - if (mutateRepaired && options.mutateRepairStatus) // if we are able to mutate repaired flag, an incremental repair should be enough - { - try - { - sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient()); - cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); - } - catch(IOException ioe) - { - outputHandler.output("Error mutating repairedAt for SSTable " + sstable.getFilename() + ", as part of markAndThrow"); - } - } - Exception e = new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), (mutateRepaired && options.mutateRepairStatus) ? "" : "a full "), cause); - if (options.invokeDiskFailurePolicy) - throw new CorruptSSTableException(e, sstable.getFilename()); - else - throw new RuntimeException(e); - } - - public CompactionInfo.Holder getVerifyInfo() - { - return verifyInfo; - } - - private static class VerifyInfo extends CompactionInfo.Holder - { - private final RandomAccessReader dataFile; - private final SSTableReader sstable; - private final TimeUUID verificationCompactionId; - private final Lock fileReadLock; - - public VerifyInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock) - { - this.dataFile = dataFile; - this.sstable = sstable; - this.fileReadLock = fileReadLock; - verificationCompactionId = nextTimeUUID(); - } - - public CompactionInfo getCompactionInfo() - { - fileReadLock.lock(); - try - { - return new CompactionInfo(sstable.metadata(), - OperationType.VERIFY, - dataFile.getFilePointer(), - dataFile.length(), - verificationCompactionId, - ImmutableSet.of(sstable)); - } - catch (Exception e) - { - throw new RuntimeException(); - } - finally - { - fileReadLock.unlock(); - } - } - - public boolean isGlobal() - { - return false; - } - } - - private static class VerifyController extends CompactionController - { - public VerifyController(ColumnFamilyStore cfs) - { - super(cfs, Integer.MAX_VALUE); - } - - @Override - public LongPredicate getPurgeEvaluator(DecoratedKey key) - { - return time -> false; - } - } - - public static Options.Builder options() - { - return new Options.Builder(); - } - - public static class Options - { - public final boolean invokeDiskFailurePolicy; - public final boolean extendedVerification; - public final boolean checkVersion; - public final boolean mutateRepairStatus; - public final boolean checkOwnsTokens; - public final boolean quick; - public final Function>> tokenLookup; - - private Options(boolean invokeDiskFailurePolicy, boolean extendedVerification, boolean checkVersion, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, Function>> tokenLookup) - { - this.invokeDiskFailurePolicy = invokeDiskFailurePolicy; - this.extendedVerification = extendedVerification; - this.checkVersion = checkVersion; - this.mutateRepairStatus = mutateRepairStatus; - this.checkOwnsTokens = checkOwnsTokens; - this.quick = quick; - this.tokenLookup = tokenLookup; - } - - @Override - public String toString() - { - return "Options{" + - "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy + - ", extendedVerification=" + extendedVerification + - ", checkVersion=" + checkVersion + - ", mutateRepairStatus=" + mutateRepairStatus + - ", checkOwnsTokens=" + checkOwnsTokens + - ", quick=" + quick + - '}'; - } - - public static class Builder - { - private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable - private boolean extendedVerification = false; - private boolean checkVersion = false; - private boolean mutateRepairStatus = false; // mutating repair status can be dangerous - private boolean checkOwnsTokens = false; - private boolean quick = false; - private Function>> tokenLookup = StorageService.instance::getLocalAndPendingRanges; - - public Builder invokeDiskFailurePolicy(boolean param) - { - this.invokeDiskFailurePolicy = param; - return this; - } - - public Builder extendedVerification(boolean param) - { - this.extendedVerification = param; - return this; - } - - public Builder checkVersion(boolean param) - { - this.checkVersion = param; - return this; - } - - public Builder mutateRepairStatus(boolean param) - { - this.mutateRepairStatus = param; - return this; - } - - public Builder checkOwnsTokens(boolean param) - { - this.checkOwnsTokens = param; - return this; - } - - public Builder quick(boolean param) - { - this.quick = param; - return this; - } - - public Builder tokenLookup(Function>> tokenLookup) - { - this.tokenLookup = tokenLookup; - return this; - } - - public Options build() - { - return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup); - } - - } - } -} diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java index c375c85650..451a78d394 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java @@ -31,16 +31,19 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionTask; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableRewriter; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Transactional; -import org.apache.cassandra.io.util.File; /** @@ -246,4 +249,24 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa { return cfs.getExpectedCompactedFileSize(nonExpiredSSTables, txn.opType()); } + + /** + * It is up to the caller to set the following fields: + * - {@link SSTableWriter.Builder#setKeyCount(long)}, + * - {@link SSTableWriter.Builder#setSerializationHeader(SerializationHeader)} and, + * - {@link SSTableWriter.Builder#setMetadataCollector(MetadataCollector)} + * + * @param descriptor + * @return + */ + protected SSTableWriter.Builder newWriterBuilder(Descriptor descriptor) + { + return descriptor.getFormat().getWriterFactory().builder(descriptor) + .setTableMetadataRef(cfs.metadata) + .setTransientSSTable(isTransient) + .setRepairedAt(minRepairedAt) + .setPendingRepair(pendingRepair) + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType()) + .addDefaultComponents(); + } } diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java index dfd771afdd..f9858473e2 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java @@ -26,8 +26,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -69,17 +70,15 @@ public class DefaultCompactionWriter extends CompactionAwareWriter { sstableDirectory = directory; + Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory)); + MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel); + SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables); + @SuppressWarnings("resource") - SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory)), - estimatedTotalKeys, - minRepairedAt, - pendingRepair, - isTransient, - cfs.metadata, - new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel), - SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), - cfs.indexManager.listIndexes(), - txn); + SSTableWriter writer = newWriterBuilder(descriptor).setMetadataCollector(collector) + .setSerializationHeader(header) + .setKeyCount(estimatedTotalKeys) + .build(txn, cfs); sstableWriter.switchWriter(writer); } diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java index 21f698ad9c..38e2d2d2c9 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java @@ -21,11 +21,12 @@ import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -69,7 +70,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter @SuppressWarnings("resource") public boolean realAppend(UnfilteredRowIterator partition) { - RowIndexEntry rie = sstableWriter.append(partition); + AbstractRowIndexEntry rie = sstableWriter.append(partition); partitionsWritten++; long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten(); if (totalWrittenInCurrentWriter > maxSSTableSize) @@ -91,16 +92,18 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter { sstableDirectory = location; averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1)); - sstableWriter.switchWriter(SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)), - keysPerSSTable, - minRepairedAt, - pendingRepair, - isTransient, - cfs.metadata, - new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel), - SerializationHeader.make(cfs.metadata(), txn.originals()), - cfs.indexManager.listIndexes(), - txn)); + + Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)); + MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel); + SerializationHeader serializationHeader = SerializationHeader.make(cfs.metadata(), txn.originals()); + + @SuppressWarnings("resource") + SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(keysPerSSTable) + .setSerializationHeader(serializationHeader) + .setMetadataCollector(collector) + .build(txn, cfs); + + sstableWriter.switchWriter(writer); partitionsWritten = 0; sstablesWritten = 0; } diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java index bd4aee0473..834bda9e23 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java @@ -21,11 +21,12 @@ import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -80,7 +81,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter protected boolean realAppend(UnfilteredRowIterator partition) { - RowIndexEntry rie = sstableWriter.append(partition); + AbstractRowIndexEntry rie = sstableWriter.append(partition); if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize) { switchCompactionLocation(sstableDirectory); @@ -92,18 +93,16 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter public void switchCompactionLocation(Directories.DataDirectory location) { sstableDirectory = location; - @SuppressWarnings("resource") - SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)), - estimatedTotalKeys / estimatedSSTables, - minRepairedAt, - pendingRepair, - isTransient, - cfs.metadata, - new MetadataCollector(allSSTables, cfs.metadata().comparator, level), - SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), - cfs.indexManager.listIndexes(), - txn); + Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)); + MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, level); + SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables); + + @SuppressWarnings("resource") + SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(estimatedTotalKeys / estimatedSSTables) + .setMetadataCollector(collector) + .setSerializationHeader(header) + .build(txn, cfs); sstableWriter.switchWriter(writer); } diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java index 5d3f1a6e0b..458d80a71a 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java @@ -25,10 +25,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -85,7 +86,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter @Override public boolean realAppend(UnfilteredRowIterator partition) { - RowIndexEntry rie = sstableWriter.append(partition); + AbstractRowIndexEntry rie = sstableWriter.append(partition); if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect { currentRatioIndex++; @@ -102,17 +103,15 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter { sstableDirectory = location; long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys); + Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location)); + MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, 0); + SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables); + @SuppressWarnings("resource") - SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location)), - currentPartitionsToWrite, - minRepairedAt, - pendingRepair, - isTransient, - cfs.metadata, - new MetadataCollector(allSSTables, cfs.metadata().comparator, 0), - SerializationHeader.make(cfs.metadata(), nonExpiredSSTables), - cfs.indexManager.listIndexes(), - txn); + SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(currentPartitionsToWrite) + .setMetadataCollector(collector) + .setSerializationHeader(header) + .build(txn, cfs); logger.trace("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite); sstableWriter.switchWriter(writer); } diff --git a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java index 8e0d5144b8..134beec116 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java +++ b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java @@ -17,16 +17,27 @@ */ package org.apache.cassandra.db.lifecycle; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; import com.google.common.base.Predicate; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.Throwables; -import static com.google.common.base.Predicates.*; +import static com.google.common.base.Predicates.and; +import static com.google.common.base.Predicates.equalTo; +import static com.google.common.base.Predicates.in; +import static com.google.common.base.Predicates.not; +import static com.google.common.base.Predicates.or; import static com.google.common.collect.Iterables.any; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.filter; @@ -50,7 +61,7 @@ class Helpers * really present, and that the items to add are not (unless we're also removing them) * @return a new identity map with the contents of the provided one modified */ - static Map replace(Map original, Set remove, Iterable add) + static Map replace(Map original, Set remove, Iterable add) { // ensure the ones being removed are the exact same ones present for (T reader : remove) diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java index d3f3a1e9c9..7cff57da25 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java @@ -18,33 +18,54 @@ package org.apache.cassandra.db.lifecycle; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Set; import java.util.function.BiPredicate; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; -import com.google.common.collect.*; - -import org.apache.cassandra.io.util.File; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReader.UniqueIdentifier; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Transactional; import static com.google.common.base.Functions.compose; -import static com.google.common.base.Predicates.*; +import static com.google.common.base.Predicates.in; import static com.google.common.collect.ImmutableSet.copyOf; -import static com.google.common.collect.Iterables.*; +import static com.google.common.collect.Iterables.concat; +import static com.google.common.collect.Iterables.getFirst; +import static com.google.common.collect.Iterables.transform; import static java.util.Collections.singleton; -import static org.apache.cassandra.db.lifecycle.Helpers.*; +import static org.apache.cassandra.db.lifecycle.Helpers.abortObsoletion; +import static org.apache.cassandra.db.lifecycle.Helpers.checkNotReplaced; +import static org.apache.cassandra.db.lifecycle.Helpers.concatUniq; +import static org.apache.cassandra.db.lifecycle.Helpers.emptySet; +import static org.apache.cassandra.db.lifecycle.Helpers.filterIn; +import static org.apache.cassandra.db.lifecycle.Helpers.filterOut; +import static org.apache.cassandra.db.lifecycle.Helpers.markObsolete; +import static org.apache.cassandra.db.lifecycle.Helpers.orIn; +import static org.apache.cassandra.db.lifecycle.Helpers.prepareForObsoletion; +import static org.apache.cassandra.db.lifecycle.Helpers.select; +import static org.apache.cassandra.db.lifecycle.Helpers.selectFirst; +import static org.apache.cassandra.db.lifecycle.Helpers.setReplaced; import static org.apache.cassandra.db.lifecycle.View.updateCompacting; import static org.apache.cassandra.db.lifecycle.View.updateLiveSet; import static org.apache.cassandra.utils.Throwables.maybeFail; @@ -160,12 +181,12 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im } @SuppressWarnings("resource") // log closed during postCleanup - LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable readers) + LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable readers) { this(tracker, new LogTransaction(operationType, tracker), readers); } - LifecycleTransaction(Tracker tracker, LogTransaction log, Iterable readers) + LifecycleTransaction(Tracker tracker, LogTransaction log, Iterable readers) { this.tracker = tracker; this.log = log; @@ -456,7 +477,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional im private List restoreUpdatedOriginals() { Iterable torestore = filterIn(originals, logged.update, logged.obsolete); - return ImmutableList.copyOf(transform(torestore, (reader) -> current(reader).cloneWithRestoredStart(reader.first))); + return ImmutableList.copyOf(transform(torestore, (reader) -> current(reader).cloneWithRestoredStart(reader.getFirst()))); } /** diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java index 11e3ffbde1..7a23a7a250 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java @@ -21,15 +21,22 @@ package org.apache.cassandra.db.lifecycle; import java.nio.file.Path; -import java.util.*; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; - -import org.apache.cassandra.io.util.File; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,8 +44,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LogRecord.Type; import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; @@ -62,7 +70,7 @@ final class LogFile implements AutoCloseable static String EXT = ".log"; static char SEP = '_'; // cc_txn_opname_id.log (where cc is one of the sstable versions defined in BigVersion) - static Pattern FILE_REGEX = Pattern.compile(String.format("^(.{2})_txn_(.*)_(.*)%s$", EXT)); + static Pattern FILE_REGEX = Pattern.compile(String.format("^(.{2}_)?txn_(.*)_(.*)%s$", EXT)); // A set of physical files on disk, each file is an identical replica private final LogReplicaSet replicas = new LogReplicaSet(); @@ -496,18 +504,14 @@ final class LogFile implements AutoCloseable private String getFileName() { - return StringUtils.join(BigFormat.latestVersion, - LogFile.SEP, - "txn", - LogFile.SEP, - type.fileName, - LogFile.SEP, - id.toString(), - LogFile.EXT); + return StringUtils.join(SSTableFormat.Type.current().info.getLatestVersion(), LogFile.SEP, // remove version and separator when downgrading to 4.x is becomes unsupported + "txn", LogFile.SEP, + type.fileName, LogFile.SEP, + id.toString(), LogFile.EXT); } public boolean isEmpty() { return records.isEmpty(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java index 45653c47a3..34fd0daf91 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LogRecord.java @@ -22,7 +22,16 @@ package org.apache.cassandra.db.lifecycle; import java.nio.file.Path; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; import java.util.function.BiPredicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -152,7 +161,7 @@ final class LogRecord public static LogRecord make(Type type, SSTable table) { - String absoluteTablePath = absolutePath(table.descriptor.baseFilename()); + String absoluteTablePath = absolutePath(table.descriptor.baseFile()); return make(type, getExistingFiles(absoluteTablePath), table.getAllFilePaths().size(), absoluteTablePath); } @@ -161,7 +170,7 @@ final class LogRecord // contains a mapping from sstable absolute path (everything up until the 'Data'/'Index'/etc part of the filename) to the sstable Map absolutePaths = new HashMap<>(); for (SSTableReader table : tables) - absolutePaths.put(absolutePath(table.descriptor.baseFilename()), table); + absolutePaths.put(absolutePath(table.descriptor.baseFile()), table); // maps sstable base file name to the actual files on disk Map> existingFiles = getExistingFiles(absolutePaths.keySet()); @@ -176,9 +185,9 @@ final class LogRecord return records; } - private static String absolutePath(String baseFilename) + private static String absolutePath(File baseFile) { - return FileUtils.getCanonicalPath(baseFilename + Component.separator); + return baseFile.withSuffix(String.valueOf(Component.separator)).canonicalPath(); } public LogRecord withExistingFiles(List existingFiles) diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java index c37134b7de..e64c57a9a7 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java @@ -22,18 +22,22 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.NoSuchFileException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.Runnables; - -import com.codahale.metrics.Counter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.Counter; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Directories; @@ -41,15 +45,17 @@ import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LogRecord.Type; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.RefCounted; import org.apache.cassandra.utils.concurrent.Transactional; @@ -387,18 +393,14 @@ class LogTransaction extends Transactional.AbstractTransactional implements Tran try { // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier - File datafile = new File(desc.filenameFor(Component.DATA)); if (logger.isTraceEnabled()) - logger.trace("Tidier running for old sstable {}", desc.baseFilename()); + logger.trace("Tidier running for old sstable {}", desc); - if (datafile.exists()) - delete(datafile); - else if (!wasNew) + if (!desc.fileFor(Components.DATA).exists() && !wasNew) logger.error("SSTableTidier ran with no existing data file for an sstable that was not new"); - // let the remainder be cleaned up by delete - SSTable.delete(desc, SSTable.discoverComponentsFor(desc)); + desc.getFormat().delete(desc); } catch (Throwable t) { diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java index 12c76af8c9..c54ee83bd7 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java +++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java @@ -17,7 +17,11 @@ */ package org.apache.cassandra.db.lifecycle; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicReference; @@ -25,23 +29,34 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.collect.*; - -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.Directories; -import org.apache.cassandra.db.memtable.Memtable; -import org.apache.cassandra.db.commitlog.CommitLogPosition; -import org.apache.cassandra.io.util.File; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.StorageMetrics; -import org.apache.cassandra.notifications.*; +import org.apache.cassandra.notifications.INotification; +import org.apache.cassandra.notifications.INotificationConsumer; +import org.apache.cassandra.notifications.InitialSSTableAddedNotification; +import org.apache.cassandra.notifications.MemtableDiscardedNotification; +import org.apache.cassandra.notifications.MemtableRenewedNotification; +import org.apache.cassandra.notifications.MemtableSwitchedNotification; +import org.apache.cassandra.notifications.SSTableAddedNotification; +import org.apache.cassandra.notifications.SSTableDeletingNotification; +import org.apache.cassandra.notifications.SSTableListChangedNotification; +import org.apache.cassandra.notifications.SSTableMetadataChanged; +import org.apache.cassandra.notifications.SSTableRepairStatusChanged; +import org.apache.cassandra.notifications.TruncationNotification; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -51,7 +66,11 @@ import static com.google.common.collect.ImmutableSet.copyOf; import static com.google.common.collect.Iterables.filter; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.apache.cassandra.db.lifecycle.Helpers.*; +import static org.apache.cassandra.db.lifecycle.Helpers.abortObsoletion; +import static org.apache.cassandra.db.lifecycle.Helpers.markObsolete; +import static org.apache.cassandra.db.lifecycle.Helpers.notIn; +import static org.apache.cassandra.db.lifecycle.Helpers.prepareForObsoletion; +import static org.apache.cassandra.db.lifecycle.Helpers.setupOnline; import static org.apache.cassandra.db.lifecycle.View.permitCompacting; import static org.apache.cassandra.db.lifecycle.View.updateCompacting; import static org.apache.cassandra.db.lifecycle.View.updateLiveSet; @@ -99,7 +118,7 @@ public class Tracker /** * @return a Transaction over the provided sstables if we are able to mark the given @param sstables as compacted, before anyone else */ - public LifecycleTransaction tryModify(Iterable sstables, OperationType operationType) + public LifecycleTransaction tryModify(Iterable sstables, OperationType operationType) { if (Iterables.isEmpty(sstables)) return new LifecycleTransaction(this, operationType, sstables); diff --git a/src/java/org/apache/cassandra/db/lifecycle/View.java b/src/java/org/apache/cassandra/db/lifecycle/View.java index 958bc0d235..b238d24d58 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/View.java +++ b/src/java/org/apache/cassandra/db/lifecycle/View.java @@ -17,17 +17,22 @@ */ package org.apache.cassandra.db.lifecycle; -import java.util.*; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Functions; import com.google.common.base.Predicate; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.Interval; @@ -136,13 +141,30 @@ public class View case NONCOMPACTING: return filter(sstables, (s) -> !compacting.contains(s)); case CANONICAL: + // When early open is not in play, the LIVE and CANONICAL sets are the same. + // However, when we do have early-open sstables, we will have some unfinished sources in the live set. + // For these sources we need to extract the originals, in their non-moved-start versions, from the + // compacting set. + // This creates a problem when the compaction completes, as then both: + // - the source is in the compacting set + // - the result is in the live set + // This currently causes the CANONICAL set to return both source and result when early-open is disabled, + // and is otherwise worked around by opening early the last sstable in the result set (which pushes it + // in the compacting set with EARLY openReason) and the !compacting.contains(sstable) check in the + // second loop below. + // Unfortunately there does not appear to be a way to avoid this workaround. Filtering the compacting + // set through having an early-open version in live does not work because sources are fully removed from + // the live set when they are completely exhausted. + + // Add the compacting versions first because they will be the canonical versions of compaction sources. Set canonicalSSTables = new HashSet<>(sstables.size() + compacting.size()); for (SSTableReader sstable : compacting) if (sstable.openReason != SSTableReader.OpenReason.EARLY) canonicalSSTables.add(sstable); - // reason for checking if compacting contains the sstable is that if compacting has an EARLY version - // of a NORMAL sstable, we still have the canonical version of that sstable in sstables. - // note that the EARLY version is equal, but not == since it is a different instance of the same sstable. + // Add anything that is not compacting, removing any compaction result where we still have the + // compaction sources. + // note that the EARLY version is equal to the original, i.e. the set itself can guarantee early-open + // versions of sstables in compacting won't be added, but we also want to remove the results. for (SSTableReader sstable : sstables) if (!compacting.contains(sstable) && sstable.openReason != SSTableReader.OpenReason.EARLY) canonicalSSTables.add(sstable); @@ -241,7 +263,7 @@ public class View // METHODS TO CONSTRUCT FUNCTIONS FOR MODIFYING A VIEW: // return a function to un/mark the provided readers compacting in a view - static Function updateCompacting(final Set unmark, final Iterable mark) + static Function updateCompacting(final Set unmark, final Iterable mark) { if (unmark.isEmpty() && Iterables.isEmpty(mark)) return Functions.identity(); @@ -259,7 +281,7 @@ public class View // construct a predicate to reject views that do not permit us to mark these readers compacting; // i.e. one of them is either already compacting, has been compacted, or has been replaced - static Predicate permitCompacting(final Iterable readers) + static Predicate permitCompacting(final Iterable readers) { return new Predicate() { @@ -353,4 +375,4 @@ public class View } }; } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java index 42d2e7be18..036d742895 100644 --- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java @@ -52,7 +52,7 @@ import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.IncludingExcludingBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.index.transactions.UpdateTransaction; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.concurrent.OpOrder; diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java index 3c8dfb3104..e6fecec99e 100644 --- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java @@ -51,7 +51,7 @@ import org.apache.cassandra.dht.IncludingExcludingBounds; import org.apache.cassandra.dht.Murmur3Partitioner.LongToken; import org.apache.cassandra.dht.Range; import org.apache.cassandra.index.transactions.UpdateTransaction; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ObjectSizes; diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java index ff8d765706..f1401f76d8 100644 --- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java @@ -63,7 +63,7 @@ import org.apache.cassandra.dht.IncludingExcludingBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.compress.BufferType; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.metrics.TrieMemtableMetricsView; import org.apache.cassandra.schema.TableMetadata; diff --git a/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java index cbbac64471..24740845f3 100644 --- a/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java +++ b/src/java/org/apache/cassandra/db/rows/ThrottledUnfilteredIterator.java @@ -21,13 +21,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.CloseableIterator; -import com.google.common.annotations.VisibleForTesting; - /** * A utility class to split the given {@link#UnfilteredRowIterator} into smaller chunks each * having at most {@link #throttle} + 1 unfiltereds. @@ -83,7 +83,7 @@ public class ThrottledUnfilteredIterator extends AbstractIterator maybeGetLowerBoundFromKeyCache() { - RowIndexEntry rowIndexEntry = sstable.getCachedPosition(partitionKey(), false); - if (rowIndexEntry == null || !rowIndexEntry.indexOnHeap()) - return null; + if (sstable instanceof KeyCacheSupport) + return ((KeyCacheSupport) sstable).getLowerBoundPrefixFromCache(partitionKey(), isReverseOrder); - try (RowIndexEntry.IndexInfoRetriever onHeapRetriever = rowIndexEntry.openWithIndex(null)) - { - IndexInfo columns = onHeapRetriever.columnsIndex(isReverseOrder() ? rowIndexEntry.columnsIndexCount() - 1 : 0); - ClusteringBound bound = isReverseOrder() ? columns.lastName.asEndBound() : columns.firstName.asStartBound(); - assertBoundSize(bound); - return bound.artificialLowerBound(isReverseOrder()); - } - catch (IOException e) - { - throw new RuntimeException("should never occur", e); - } + return null; } /** @@ -253,16 +241,16 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt final StatsMetadata m = sstable.getSSTableMetadata(); ClusteringBound bound = m.coveredClustering.open(isReverseOrder); - assertBoundSize(bound); + assertBoundSize(bound, sstable); return bound.artificialLowerBound(isReverseOrder); } - private void assertBoundSize(ClusteringPrefix lowerBound) + public static void assertBoundSize(ClusteringPrefix lowerBound, SSTable sstable) { - assert lowerBound.size() <= metadata().comparator.size() : + assert lowerBound.size() <= sstable.metadata().comparator.size() : String.format("Unexpected number of clustering values %d, expected %d or fewer for %s", lowerBound.size(), - metadata().comparator.size(), + sstable.metadata().comparator.size(), sstable.getFilename()); } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java index 2eb5d8fde7..95cf0804e5 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java @@ -17,12 +17,17 @@ */ package org.apache.cassandra.db.rows; -import java.util.*; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Digest; +import org.apache.cassandra.db.EmptyIterators; +import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.transform.FilteredRows; import org.apache.cassandra.db.transform.MoreRows; @@ -263,16 +268,22 @@ public abstract class UnfilteredRowIterators /** * Returns an iterator that concatenate the specified atom with the iterator. */ - public static UnfilteredRowIterator concat(final Unfiltered first, final UnfilteredRowIterator rest) + public static UnfilteredRowIterator concat(final Unfiltered first, final UnfilteredRowIterator wrapped) { - return new WrappingUnfilteredRowIterator(rest) + return new WrappingUnfilteredRowIterator() { private boolean hasReturnedFirst; + @Override + public UnfilteredRowIterator wrapped() + { + return wrapped; + } + @Override public boolean hasNext() { - return hasReturnedFirst ? super.hasNext() : true; + return hasReturnedFirst ? wrapped.hasNext() : true; } @Override @@ -283,7 +294,7 @@ public abstract class UnfilteredRowIterators hasReturnedFirst = true; return first; } - return super.next(); + return wrapped.next(); } }; } @@ -604,4 +615,4 @@ public abstract class UnfilteredRowIterators } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java index b984522f62..62c22eda0c 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSource.java @@ -23,7 +23,7 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; /** * Common data access interface for sstables and memtables. diff --git a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java index 02e1cec411..e387136d45 100644 --- a/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java +++ b/src/java/org/apache/cassandra/db/rows/WrappingUnfilteredRowIterator.java @@ -17,76 +17,72 @@ */ package org.apache.cassandra.db.rows; -import com.google.common.collect.UnmodifiableIterator; - +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; /** * Abstract class to make writing unfiltered iterators that wrap another iterator * easier. By default, the wrapping iterator simply delegate every call to - * the wrapped iterator so concrete implementations will have to override - * some of the methods. + * the wrapped iterator so concrete implementations will have to override some methods. *

* Note that if most of what you want to do is modifying/filtering the returned - * {@code Unfiltered}, {@link org.apache.cassandra.db.transform.Transformation#merge(UnfilteredRowIterator,Transformation)} can be a simpler option. + * {@code Unfiltered}, {@link org.apache.cassandra.db.transform.Transformation#apply(UnfilteredRowIterator, Transformation)} + * can be a simpler option. */ -public abstract class WrappingUnfilteredRowIterator extends UnmodifiableIterator implements UnfilteredRowIterator +public interface WrappingUnfilteredRowIterator extends UnfilteredRowIterator { - protected final UnfilteredRowIterator wrapped; + UnfilteredRowIterator wrapped(); - protected WrappingUnfilteredRowIterator(UnfilteredRowIterator wrapped) + default TableMetadata metadata() { - this.wrapped = wrapped; + return wrapped().metadata(); } - public TableMetadata metadata() + default RegularAndStaticColumns columns() { - return wrapped.metadata(); + return wrapped().columns(); } - public RegularAndStaticColumns columns() + default boolean isReverseOrder() { - return wrapped.columns(); + return wrapped().isReverseOrder(); } - public boolean isReverseOrder() + default DecoratedKey partitionKey() { - return wrapped.isReverseOrder(); + return wrapped().partitionKey(); } - public DecoratedKey partitionKey() + default DeletionTime partitionLevelDeletion() { - return wrapped.partitionKey(); + return wrapped().partitionLevelDeletion(); } - public DeletionTime partitionLevelDeletion() + default Row staticRow() { - return wrapped.partitionLevelDeletion(); + return wrapped().staticRow(); } - public Row staticRow() + default EncodingStats stats() { - return wrapped.staticRow(); + return wrapped().stats(); } - public EncodingStats stats() + default boolean hasNext() { - return wrapped.stats(); + return wrapped().hasNext(); } - public boolean hasNext() + default Unfiltered next() { - return wrapped.hasNext(); + return wrapped().next(); } - public Unfiltered next() + default void close() { - return wrapped.next(); + wrapped().close(); } - - public void close() - { - wrapped.close(); - } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java index 41fd9b1651..806a74a35c 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamWriter.java @@ -27,12 +27,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.io.compress.CompressionMetadata; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.streaming.ProgressInfo; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.utils.FBUtilities; /** @@ -71,7 +71,7 @@ public class CassandraCompressedStreamWriter extends CassandraStreamWriter int sectionIdx = 0; // stream each of the required sections of the file - String filename = sstable.descriptor.filenameFor(Component.DATA); + String filename = sstable.descriptor.fileFor(Components.DATA).toString(); for (Section section : sections) { // length of the section to stream diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java index 515c85dea6..bafe53979f 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java @@ -25,17 +25,20 @@ import java.util.function.UnaryOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; +import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IOOptions; import org.apache.cassandra.io.sstable.SSTableMultiWriter; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.big.BigTableZeroCopyWriter; +import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.ProgressInfo; import org.apache.cassandra.streaming.StreamReceiver; @@ -60,9 +63,6 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader public CassandraEntireSSTableStreamReader(StreamMessageHeader messageHeader, CassandraStreamHeader streamHeader, StreamSession session) { - if (streamHeader.format != SSTableFormat.Type.BIG) - throw new AssertionError("Unsupported SSTable format " + streamHeader.format); - if (session.getPendingRepair() != null) { // we should only ever be streaming pending repair sstables if the session has a pending repair id @@ -84,7 +84,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader */ @SuppressWarnings("resource") // input needs to remain open, streams on top of it can't be closed @Override - public SSTableMultiWriter read(DataInputPlus in) throws Throwable + public SSTableMultiWriter read(DataInputPlus in) throws IOException { ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(tableId); if (cfs == null) @@ -103,7 +103,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader prettyPrintMemory(totalSize), cfs.metadata()); - BigTableZeroCopyWriter writer = null; + SSTableZeroCopyWriter writer = null; try { @@ -122,7 +122,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader prettyPrintMemory(totalSize)); writer.writeComponent(component.type, in, length); - session.progress(writer.descriptor.filenameFor(component), ProgressInfo.Direction.IN, length, length, length); + session.progress(writer.descriptor.fileFor(component).toString(), ProgressInfo.Direction.IN, length, length, length); bytesRead += length; logger.debug("[Stream #{}] Finished receiving {} component from {}, componentSize = {}, readBytes = {}, totalSize = {}", @@ -145,7 +145,11 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader { logger.error("[Stream {}] Error while reading sstable from stream for table = {}", session.planId(), cfs.metadata(), e); if (writer != null) - e = writer.abort(e); + { + Throwable e2 = writer.abort(null); + if (e2 != null) + e.addSuppressed(e2); + } throw e; } } @@ -165,7 +169,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader } @SuppressWarnings("resource") - protected BigTableZeroCopyWriter createWriter(ColumnFamilyStore cfs, long totalSize, Collection components) throws IOException + protected SSTableZeroCopyWriter createWriter(ColumnFamilyStore cfs, long totalSize, Collection components) throws IOException { File dataDir = getDataDir(cfs, totalSize); @@ -176,8 +180,24 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version, header.format); - logger.debug("[Table #{}] {} Components to write: {}", cfs.metadata(), desc.filenameFor(Component.DATA), components); + IOOptions ioOptions = new IOOptions(DatabaseDescriptor.getDiskOptimizationStrategy(), + DatabaseDescriptor.getDiskAccessMode(), + DatabaseDescriptor.getIndexAccessMode(), + DatabaseDescriptor.getDiskOptimizationEstimatePercentile(), + SequentialWriterOption.newBuilder() + .trickleFsync(false) + .bufferSize(2 << 20) + .bufferType(BufferType.OFF_HEAP) + .build(), + DatabaseDescriptor.getFlushCompression()); - return new BigTableZeroCopyWriter(desc, cfs.metadata, lifecycleNewTracker, components); + logger.debug("[Table #{}] {} Components to write: {}", cfs.metadata(), desc, components); + return desc.getFormat() + .getWriterFactory() + .builder(desc) + .setComponents(components) + .setTableMetadataRef(cfs.metadata) + .setIOOptions(ioOptions) + .createZeroCopyWriter(lifecycleNewTracker, cfs); } } diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java index 3d679a515e..54fec4990f 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java @@ -27,9 +27,9 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.streaming.ProgressInfo; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import static org.apache.cassandra.streaming.StreamManager.StreamRateLimiter; import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory; @@ -93,7 +93,7 @@ public class CassandraEntireSSTableStreamWriter long bytesWritten = out.writeFileToChannel(channel, limiter); progress += bytesWritten; - session.progress(sstable.descriptor.filenameFor(component), ProgressInfo.Direction.OUT, bytesWritten, bytesWritten, length); + session.progress(sstable.descriptor.fileFor(component).toString(), ProgressInfo.Direction.OUT, bytesWritten, bytesWritten, length); logger.debug("[Stream #{}] Finished streaming {}.{} gen {} component {} to {}, xfered = {}, length = {}, totalSize = {}", session.planId(), diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java index c9e10cf6a4..1ba5d13d02 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java @@ -182,7 +182,7 @@ public class CassandraStreamHeader if (header.isEntireSSTable) { - ComponentManifest.serializer.serialize(header.componentManifest, out, version); + ComponentManifest.serializers.get(header.format).serialize(header.componentManifest, out, version); ByteBufferUtil.writeWithVIntLength(header.firstKey.getKey(), out); } } @@ -201,8 +201,9 @@ public class CassandraStreamHeader @VisibleForTesting public CassandraStreamHeader deserialize(DataInputPlus in, int version, Function partitionerMapper) throws IOException { - Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF()); - SSTableFormat.Type format = SSTableFormat.Type.validate(in.readUTF()); + String sstableVersionString = in.readUTF(); + SSTableFormat.Type format = SSTableFormat.Type.getByName(in.readUTF()); + Version sstableVersion = format.info.getVersion(sstableVersionString); long estimatedKeys = in.readLong(); int count = in.readInt(); @@ -221,7 +222,7 @@ public class CassandraStreamHeader if (isEntireSSTable) { - manifest = ComponentManifest.serializer.deserialize(in, version); + manifest = ComponentManifest.serializers.get(format).deserialize(in, version); ByteBuffer keyBuf = ByteBufferUtil.readWithVIntLength(in); IPartitioner partitioner = partitionerMapper.apply(tableId); if (partitioner == null) @@ -267,7 +268,7 @@ public class CassandraStreamHeader if (header.isEntireSSTable) { - size += ComponentManifest.serializer.serializedSize(header.componentManifest, version); + size += ComponentManifest.serializers.get(header.format).serializedSize(header.componentManifest, version); size += ByteBufferUtil.serializedSizeWithVIntLength(header.firstKey.getKey()); } return size; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index 04268f024c..a659c73e7c 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -39,9 +39,9 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.exceptions.UnknownColumnException; +import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.SSTableSimpleIterator; -import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java index 9d9ea3c1fc..3e98c7dbda 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamWriter.java @@ -21,23 +21,21 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collection; -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; import org.apache.cassandra.io.compress.BufferType; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.ChannelProxy; -import org.apache.cassandra.io.util.DataIntegrityMetadata; import org.apache.cassandra.io.util.DataIntegrityMetadata.ChecksumValidator; import org.apache.cassandra.streaming.ProgressInfo; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamManager.StreamRateLimiter; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.async.StreamCompressionSerializer; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.memory.BufferPools; @@ -84,9 +82,7 @@ public class CassandraStreamWriter sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize); try(ChannelProxy proxy = sstable.getDataChannel().newChannel(); - ChecksumValidator validator = new File(sstable.descriptor.filenameFor(Component.CRC)).exists() - ? DataIntegrityMetadata.checksumValidator(sstable.descriptor) - : null) + ChecksumValidator validator = sstable.maybeGetChecksumValidator()) { int bufferSize = validator == null ? DEFAULT_CHUNK_SIZE: validator.chunkSize; @@ -94,7 +90,7 @@ public class CassandraStreamWriter long progress = 0L; // stream each of the required sections of the file - String filename = sstable.descriptor.filenameFor(Component.DATA); + String filename = sstable.descriptor.fileFor(Components.DATA).toString(); for (SSTableReader.PartitionPositionBounds section : sections) { long start = validator == null ? section.lowerPosition : validator.chunkStart(section.lowerPosition); diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java index c8c08aa850..164dd6ba5c 100644 --- a/src/java/org/apache/cassandra/db/streaming/ComponentContext.java +++ b/src/java/org/apache/cassandra/db/streaming/ComponentContext.java @@ -18,32 +18,23 @@ package org.apache.cassandra.db.streaming; -import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.util.HashMap; +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.util.FileUtils; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Mutable SSTable components and their hardlinks to avoid concurrent sstable component modification - * during entire-sstable-streaming. - */ import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; public class ComponentContext implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(ComponentContext.class); - private static final Set MUTABLE_COMPONENTS = ImmutableSet.of(Component.STATS, Component.SUMMARY); - private final Map hardLinks; private final ComponentManifest manifest; @@ -57,13 +48,13 @@ public class ComponentContext implements AutoCloseable { Map hardLinks = new HashMap<>(1); - for (Component component : MUTABLE_COMPONENTS) + for (Component component : descriptor.getFormat().mutableComponents()) { - File file = new File(descriptor.filenameFor(component)); + File file = descriptor.fileFor(component); if (!file.exists()) continue; - File hardlink = new File(descriptor.tmpFilenameForStreaming(component)); + File hardlink = descriptor.tmpFileForStreaming(component); FileUtils.createHardLink(file, hardlink); hardLinks.put(component, hardlink); } @@ -81,9 +72,9 @@ public class ComponentContext implements AutoCloseable */ public FileChannel channel(Descriptor descriptor, Component component, long size) throws IOException { - String toTransfer = hardLinks.containsKey(component) ? hardLinks.get(component).path() : descriptor.filenameFor(component); + File toTransfer = hardLinks.containsKey(component) ? hardLinks.get(component) : descriptor.fileFor(component); @SuppressWarnings("resource") // file channel will be closed by Caller - FileChannel channel = new File(toTransfer).newReadChannel(); + FileChannel channel = toTransfer.newReadChannel(); assert size == channel.size() : String.format("Entire sstable streaming expects %s file size to be %s but got %s.", component, size, channel.size()); diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java index 41b2026c1d..36eb668e86 100644 --- a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java +++ b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java @@ -19,26 +19,30 @@ package org.apache.cassandra.db.streaming; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.File; +/** + * SSTable components and their sizes to be tranfered via entire-sstable-streaming + */ public final class ComponentManifest implements Iterable { - private static final List STREAM_COMPONENTS = ImmutableList.of(Component.DATA, Component.PRIMARY_INDEX, Component.STATS, - Component.COMPRESSION_INFO, Component.FILTER, Component.SUMMARY, - Component.DIGEST, Component.CRC); - private final LinkedHashMap components; public ComponentManifest(Map components) @@ -49,11 +53,11 @@ public final class ComponentManifest implements Iterable @VisibleForTesting public static ComponentManifest create(Descriptor descriptor) { - LinkedHashMap components = new LinkedHashMap<>(STREAM_COMPONENTS.size()); + LinkedHashMap components = new LinkedHashMap<>(descriptor.getFormat().streamingComponents().size()); - for (Component component : STREAM_COMPONENTS) + for (Component component : descriptor.getFormat().streamingComponents()) { - File file = new File(descriptor.filenameFor(component)); + File file = descriptor.fileFor(component); if (!file.exists()) continue; @@ -111,45 +115,58 @@ public final class ComponentManifest implements Iterable '}'; } - public static final IVersionedSerializer serializer = new IVersionedSerializer() + public static final Map> serializers; + + static { - public void serialize(ComponentManifest manifest, DataOutputPlus out, int version) throws IOException + ImmutableMap.Builder> b = ImmutableMap.builder(); + for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) { - out.writeUnsignedVInt32(manifest.components.size()); - for (Map.Entry entry : manifest.components.entrySet()) + IVersionedSerializer serializer = new IVersionedSerializer() { - out.writeUTF(entry.getKey().name); - out.writeUnsignedVInt(entry.getValue()); - } + + public void serialize(ComponentManifest manifest, DataOutputPlus out, int version) throws IOException + { + out.writeUnsignedVInt32(manifest.components.size()); + for (Map.Entry entry : manifest.components.entrySet()) + { + out.writeUTF(entry.getKey().name); + out.writeUnsignedVInt(entry.getValue()); + } + } + + public ComponentManifest deserialize(DataInputPlus in, int version) throws IOException + { + int size = in.readUnsignedVInt32(); + + LinkedHashMap components = new LinkedHashMap<>(size); + + for (int i = 0; i < size; i++) + { + Component component = Component.parse(in.readUTF(), formatType); + long length = in.readUnsignedVInt(); + components.put(component, length); + } + + return new ComponentManifest(components); + } + + public long serializedSize(ComponentManifest manifest, int version) + { + long size = TypeSizes.sizeofUnsignedVInt(manifest.components.size()); + for (Map.Entry entry : manifest.components.entrySet()) + { + size += TypeSizes.sizeof(entry.getKey().name); + size += TypeSizes.sizeofUnsignedVInt(entry.getValue()); + } + return size; + } + }; + + b.put(formatType, serializer); } - - public ComponentManifest deserialize(DataInputPlus in, int version) throws IOException - { - int size = in.readUnsignedVInt32(); - - LinkedHashMap components = new LinkedHashMap<>(size); - - for (int i = 0; i < size; i++) - { - Component component = Component.parse(in.readUTF()); - long length = in.readUnsignedVInt(); - components.put(component, length); - } - - return new ComponentManifest(components); - } - - public long serializedSize(ComponentManifest manifest, int version) - { - long size = TypeSizes.sizeofUnsignedVInt(manifest.components.size()); - for (Map.Entry entry : manifest.components.entrySet()) - { - size += TypeSizes.sizeof(entry.getKey().name); - size += TypeSizes.sizeofUnsignedVInt(entry.getValue()); - } - return size; - } - }; + serializers = b.build(); + } @Override public Iterator iterator() diff --git a/src/java/org/apache/cassandra/db/transform/BasePartitions.java b/src/java/org/apache/cassandra/db/transform/BasePartitions.java index 464ae6f88b..79e89529e2 100644 --- a/src/java/org/apache/cassandra/db/transform/BasePartitions.java +++ b/src/java/org/apache/cassandra/db/transform/BasePartitions.java @@ -20,8 +20,6 @@ */ package org.apache.cassandra.db.transform; -import java.util.Collections; - import org.apache.cassandra.db.partitions.BasePartitionIterator; import org.apache.cassandra.db.rows.BaseRowIterator; import org.apache.cassandra.utils.Throwables; @@ -112,7 +110,7 @@ implements BasePartitionIterator catch (Throwable t) { if (next != null) - Throwables.close(t, Collections.singleton(next)); + Throwables.close(t, next); throw t; } } diff --git a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java index 82d5e8a3dc..9b154e9241 100644 --- a/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java +++ b/src/java/org/apache/cassandra/hadoop/cql3/CqlBulkRecordWriter.java @@ -23,11 +23,17 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.*; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import com.google.common.net.HostAndPort; -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +46,7 @@ import org.apache.cassandra.hadoop.ConfigHelper; import org.apache.cassandra.hadoop.HadoopCompat; import org.apache.cassandra.io.sstable.CQLSSTableWriter; import org.apache.cassandra.io.sstable.SSTableLoader; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.TableMetadataRef; @@ -321,6 +328,6 @@ public class CqlBulkRecordWriter extends RecordWriter> public void output(String msg) {} public void debug(String msg) {} public void warn(String msg) {} - public void warn(String msg, Throwable th) {} + public void warn(Throwable th, String msg) {} } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java index 9f51b16806..f2307b90ba 100644 --- a/src/java/org/apache/cassandra/index/Index.java +++ b/src/java/org/apache/cassandra/index/Index.java @@ -26,13 +26,19 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.function.BiFunction; -import org.apache.cassandra.db.memtable.Memtable; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.WriteContext; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; @@ -42,8 +48,9 @@ import org.apache.cassandra.index.internal.CollatedViewIndexBuilder; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ReducingKeyIterator; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; /** diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java index 1e86bc6d8d..38588d3283 100644 --- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java @@ -17,16 +17,30 @@ */ package org.apache.cassandra.index.sasi; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.function.BiFunction; import com.googlecode.concurrenttrees.common.Iterables; - -import org.apache.cassandra.config.*; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.CassandraWriteContext; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.WriteContext; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.filter.RowFilter; @@ -49,9 +63,15 @@ import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter; import org.apache.cassandra.index.sasi.plan.QueryPlan; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.notifications.*; +import org.apache.cassandra.notifications.INotification; +import org.apache.cassandra.notifications.INotificationConsumer; +import org.apache.cassandra.notifications.MemtableDiscardedNotification; +import org.apache.cassandra.notifications.MemtableRenewedNotification; +import org.apache.cassandra.notifications.MemtableSwitchedNotification; +import org.apache.cassandra.notifications.SSTableAddedNotification; +import org.apache.cassandra.notifications.SSTableListChangedNotification; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Schema; diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java index a99b9f6e3a..555bce1b9a 100644 --- a/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java +++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java @@ -21,13 +21,13 @@ package org.apache.cassandra.index.sasi; import java.io.IOException; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.SortedMap; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.compaction.CompactionInfo; import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.OperationType; @@ -36,11 +36,12 @@ import org.apache.cassandra.index.SecondaryIndexBuilder; import org.apache.cassandra.index.sasi.conf.ColumnIndex; import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter; import org.apache.cassandra.io.FSReadError; -import org.apache.cassandra.io.sstable.KeyIterator; -import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.KeyReader; import org.apache.cassandra.io.sstable.SSTableIdentityIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.TimeUUID; @@ -57,22 +58,23 @@ class SASIIndexBuilder extends SecondaryIndexBuilder private final SortedMap> sstables; private long bytesProcessed = 0; - private final long totalSizeInBytes; + private final long totalBytesToProcess; public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap> sstables) { - long totalIndexBytes = 0; + long totalBytesToProcess = 0; for (SSTableReader sstable : sstables.keySet()) - totalIndexBytes += getPrimaryIndexLength(sstable); + totalBytesToProcess += sstable.uncompressedLength(); this.cfs = cfs; this.sstables = sstables; - this.totalSizeInBytes = totalIndexBytes; + this.totalBytesToProcess = totalBytesToProcess; } public void build() { AbstractType keyValidator = cfs.metadata().partitionKeyType; + long processedBytesInFinishedSSTables = 0; for (Map.Entry> e : sstables.entrySet()) { SSTableReader sstable = e.getKey(); @@ -83,46 +85,45 @@ class SASIIndexBuilder extends SecondaryIndexBuilder PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION); targetDirectory = indexWriter.getDescriptor().directory.path(); - long previousKeyPosition = 0; - try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata())) + try (KeyReader keys = sstable.keyReader()) { - while (keys.hasNext()) + while (!keys.isExhausted()) { if (isStopRequested()) throw new CompactionInterruptedException(getCompactionInfo()); - final DecoratedKey key = keys.next(); - final long keyPosition = keys.getKeyPosition(); + final DecoratedKey key = sstable.decorateKey(keys.key()); + final long keyPosition = keys.keyPositionForSecondaryIndex(); - indexWriter.startPartition(key, keyPosition); + indexWriter.startPartition(key, keys.dataPosition(), keyPosition); - try + dataFile.seek(keys.dataPosition()); + ByteBufferUtil.readWithShortLength(dataFile); // key + + try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key)) { - RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ); - dataFile.seek(indexEntry.position); - ByteBufferUtil.readWithShortLength(dataFile); // key - - try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key)) + // if the row has statics attached, it has to be indexed separately + if (cfs.metadata().hasStaticColumns()) { - // if the row has statics attached, it has to be indexed separately - if (cfs.metadata().hasStaticColumns()) - indexWriter.nextUnfilteredCluster(partition.staticRow()); - - while (partition.hasNext()) - indexWriter.nextUnfilteredCluster(partition.next()); + indexWriter.nextUnfilteredCluster(partition.staticRow()); } - } - catch (IOException ex) - { - throw new FSReadError(ex, sstable.getFilename()); + + while (partition.hasNext()) + indexWriter.nextUnfilteredCluster(partition.next()); } - bytesProcessed += keyPosition - previousKeyPosition; - previousKeyPosition = keyPosition; + keys.advance(); + long dataPosition = keys.isExhausted() ? sstable.uncompressedLength() : keys.dataPosition(); + bytesProcessed = processedBytesInFinishedSSTables + dataPosition; } completeSSTable(indexWriter, sstable, indexes.values()); } + catch (IOException ex) + { + throw new FSReadError(ex, sstable.getFilename()); + } + processedBytesInFinishedSSTables += sstable.uncompressedLength(); } } } @@ -132,25 +133,19 @@ class SASIIndexBuilder extends SecondaryIndexBuilder return new CompactionInfo(cfs.metadata(), OperationType.INDEX_BUILD, bytesProcessed, - totalSizeInBytes, + totalBytesToProcess, compactionId, sstables.keySet(), targetDirectory); } - private long getPrimaryIndexLength(SSTable sstable) - { - File primaryIndex = new File(sstable.getIndexFilename()); - return primaryIndex.exists() ? primaryIndex.length() : 0; - } - private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection indexes) { indexWriter.complete(); for (ColumnIndex index : indexes) { - File tmpIndex = new File(sstable.descriptor.filenameFor(index.getComponent())); + File tmpIndex = sstable.descriptor.fileFor(index.getComponent()); if (!tmpIndex.exists()) // no data was inserted into the index for given sstable continue; diff --git a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java index d756737856..de9c0c2b4e 100644 --- a/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/SSTableIndex.java @@ -22,6 +22,9 @@ import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.base.Function; +import org.apache.commons.lang3.builder.HashCodeBuilder; + import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.index.sasi.conf.ColumnIndex; @@ -36,10 +39,6 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.concurrent.Ref; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import com.google.common.base.Function; - public class SSTableIndex { private final ColumnIndex columnIndex; @@ -176,7 +175,7 @@ public class SSTableIndex { try { - return sstable.keyAt(offset); + return sstable.keyAtPositionFromSecondaryIndex(offset); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java index 81b776de62..7d8287891e 100644 --- a/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/conf/ColumnIndex.java @@ -28,13 +28,12 @@ import java.util.concurrent.atomic.AtomicReference; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer; @@ -47,7 +46,9 @@ import org.apache.cassandra.index.sasi.plan.Expression.Op; import org.apache.cassandra.index.sasi.utils.RangeIterator; import org.apache.cassandra.index.sasi.utils.RangeUnionIterator; import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.utils.FBUtilities; @@ -78,7 +79,7 @@ public class ColumnIndex this.mode = IndexMode.getMode(column, config); this.memtable = new AtomicReference<>(new IndexMemtable(this)); this.tracker = new DataTracker(keyValidator, this); - this.component = new Component(Component.Type.SECONDARY_INDEX, String.format(FILE_NAME_FORMAT, getIndexName())); + this.component = Components.Types.SECONDARY_INDEX.createComponent(String.format(FILE_NAME_FORMAT, getIndexName())); this.isTokenized = getAnalyzer().isTokenizing(); } diff --git a/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java b/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java index bf2293f487..9b3e9ab8ff 100644 --- a/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java +++ b/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java @@ -24,6 +24,9 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.index.sasi.SSTableIndex; import org.apache.cassandra.index.sasi.conf.view.View; @@ -31,9 +34,6 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** a pared-down version of DataTracker and DT.View. need one for each index of each column family */ public class DataTracker { @@ -145,7 +145,7 @@ public class DataTracker if (sstable.isMarkedCompacted()) continue; - File indexFile = new File(sstable.descriptor.filenameFor(columnIndex.getComponent())); + File indexFile = sstable.descriptor.fileFor(columnIndex.getComponent()); if (!indexFile.exists()) continue; diff --git a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java index e438079eef..88b272a769 100644 --- a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java +++ b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java @@ -17,21 +17,33 @@ */ package org.apache.cassandra.index.sasi.disk; -import java.io.*; +import java.io.Closeable; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import java.util.stream.Collectors; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.PeekingIterator; + import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.index.sasi.Term; import org.apache.cassandra.index.sasi.plan.Expression; import org.apache.cassandra.index.sasi.plan.Expression.Op; -import org.apache.cassandra.index.sasi.utils.MappedBuffer; -import org.apache.cassandra.index.sasi.utils.RangeUnionIterator; import org.apache.cassandra.index.sasi.utils.AbstractIterator; +import org.apache.cassandra.index.sasi.utils.MappedBuffer; import org.apache.cassandra.index.sasi.utils.RangeIterator; -import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.index.sasi.utils.RangeUnionIterator; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.File; @@ -40,11 +52,6 @@ import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import com.google.common.collect.PeekingIterator; - import static org.apache.cassandra.index.sasi.disk.OnDiskBlock.SearchResult; public class OnDiskIndex implements Iterable, Closeable @@ -144,7 +151,7 @@ public class OnDiskIndex implements Iterable, Closeable FileChannel channel = index.newReadChannel(); indexSize = channel.size(); - indexFile = new MappedBuffer(new ChannelProxy(indexPath, channel)); + indexFile = new MappedBuffer(new ChannelProxy(index, channel)); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java index fb5e9b9e7f..f89ff62e28 100644 --- a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java +++ b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java @@ -21,35 +21,37 @@ import java.nio.ByteBuffer; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import org.apache.cassandra.io.util.File; import org.apache.cassandra.concurrent.ExecutorPlus; -import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sasi.conf.ColumnIndex; import org.apache.cassandra.index.sasi.utils.CombinedTermIterator; import org.apache.cassandra.index.sasi.utils.TypeUtil; -import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.CountDownLatch; import org.apache.cassandra.utils.concurrent.ImmediateFuture; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; @@ -98,15 +100,24 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver indexes.put(entry.getKey(), newIndex(entry.getValue())); } + @Override public void begin() {} - public void startPartition(DecoratedKey key, long curPosition) + @Override + public void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI) { currentKey = key; - currentKeyPosition = curPosition; + currentKeyPosition = KeyPositionForSASI; } + @Override + public void staticRow(Row staticRow) + { + nextUnfilteredCluster(staticRow); + } + + @Override public void nextUnfilteredCluster(Unfiltered unfiltered) { if (!unfiltered.isRow()) @@ -126,6 +137,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver }); } + @Override public void complete() { if (isComplete) @@ -168,7 +180,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver protected class Index { @VisibleForTesting - protected final String outputFile; + protected final File outputFile; private final ColumnIndex columnIndex; private final AbstractAnalyzer analyzer; @@ -183,7 +195,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver public Index(ColumnIndex columnIndex) { this.columnIndex = columnIndex; - this.outputFile = descriptor.filenameFor(columnIndex.getComponent()); + this.outputFile = descriptor.fileFor(columnIndex.getComponent()); this.analyzer = columnIndex.getAnalyzer(); this.segments = new HashSet<>(); this.maxMemorySize = maxMemorySize(columnIndex); @@ -244,15 +256,14 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver final OnDiskIndexBuilder builder = currentBuilder; currentBuilder = newIndexBuilder(); - final String segmentFile = filename(isFinal); + final File segmentFile = file(isFinal); return () -> { long start = nanoTime(); try { - File index = new File(segmentFile); - return builder.finish(index) ? new OnDiskIndex(index, columnIndex.getValidator(), null) : null; + return builder.finish(segmentFile) ? new OnDiskIndex(segmentFile, columnIndex.getValidator(), null) : null; } catch (Exception | FSError e) { @@ -310,13 +321,13 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver OnDiskIndexBuilder builder = newIndexBuilder(); builder.finish(Pair.create(combinedMin, combinedMax), - new File(outputFile), + outputFile, new CombinedTermIterator(parts)); } catch (Exception | FSError e) { logger.error("Failed to flush index {}.", outputFile, e); - FileUtils.delete(outputFile); + outputFile.tryDelete(); } finally { @@ -330,7 +341,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver if (part != null) FileUtils.closeQuietly(part); - FileUtils.delete(outputFile + "_" + segment); + outputFile.withSuffix("_" + segment).tryDelete(); } latch.decrement(); @@ -348,9 +359,9 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver return new OnDiskIndexBuilder(keyValidator, columnIndex.getValidator(), columnIndex.getMode().mode); } - public String filename(boolean isFinal) + public File file(boolean isFinal) { - return outputFile + (isFinal ? "" : "_" + segmentNumber++); + return isFinal ? outputFile : outputFile.withSuffix("_" + segmentNumber++); } } diff --git a/src/java/org/apache/cassandra/io/IGenericSerializer.java b/src/java/org/apache/cassandra/io/IGenericSerializer.java new file mode 100644 index 0000000000..5be21cc889 --- /dev/null +++ b/src/java/org/apache/cassandra/io/IGenericSerializer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public interface IGenericSerializer +{ + void serialize(T t, O out) throws IOException; + + T deserialize(I in) throws IOException; + + long serializedSize(T t); +} diff --git a/src/java/org/apache/cassandra/io/ISerializer.java b/src/java/org/apache/cassandra/io/ISerializer.java index 637a1c7aa0..da9049df8d 100644 --- a/src/java/org/apache/cassandra/io/ISerializer.java +++ b/src/java/org/apache/cassandra/io/ISerializer.java @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -public interface ISerializer +public interface ISerializer extends IGenericSerializer { /** * Serialize the specified type into the specified DataOutput instance. @@ -32,7 +32,8 @@ public interface ISerializer * @param out DataOutput into which serialization needs to happen. * @throws java.io.IOException */ - public void serialize(T t, DataOutputPlus out) throws IOException; + @Override + void serialize(T t, DataOutputPlus out) throws IOException; /** * Deserialize from the specified DataInput instance. @@ -40,11 +41,13 @@ public interface ISerializer * @throws IOException * @return the type that was deserialized */ - public T deserialize(DataInputPlus in) throws IOException; + @Override + T deserialize(DataInputPlus in) throws IOException; - public long serializedSize(T t); + @Override + long serializedSize(T t); - public default void skip(DataInputPlus in) throws IOException + default void skip(DataInputPlus in) throws IOException { deserialize(in); } diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java index 024e4ef2b7..7d54e09601 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java @@ -29,7 +29,12 @@ import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.util.ChecksumWriter; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -66,14 +71,14 @@ public class CompressedSequentialWriter extends SequentialWriter * Create CompressedSequentialWriter without digest file. * * @param file File to write - * @param offsetsPath File name to write compression metadata + * @param offsetsFile File to write compression metadata * @param digestFile File to write digest * @param option Write option (buffer size and type will be set the same as compression params) * @param parameters Compression mparameters * @param sstableMetadataCollector Metadata collector */ public CompressedSequentialWriter(File file, - String offsetsPath, + File offsetsFile, File digestFile, SequentialWriterOption option, CompressionParams parameters, @@ -95,7 +100,7 @@ public class CompressedSequentialWriter extends SequentialWriter maxCompressedLength = parameters.maxCompressedLength(); /* Index File (-CompressionInfo.db component) and it's header */ - metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsPath); + metadataWriter = CompressionMetadata.Writer.open(parameters, offsetsFile); this.sstableMetadataCollector = sstableMetadataCollector; crcMetadata = new ChecksumWriter(new DataOutputStream(Channels.newOutputStream(channel))); @@ -397,4 +402,4 @@ public class CompressedSequentialWriter extends SequentialWriter this.nextChunkIndex = nextChunkIndex; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java index 5ec0a04952..d2fab3031d 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java +++ b/src/java/org/apache/cassandra/io/compress/CompressionMetadata.java @@ -17,14 +17,11 @@ */ package org.apache.cassandra.io.compress; -import java.nio.file.NoSuchFileException; -import java.io.DataInput; import java.io.DataOutput; import java.io.EOFException; - -import org.apache.cassandra.io.util.*; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -39,18 +36,25 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.util.Memory; +import org.apache.cassandra.io.util.SafeMemory; import org.apache.cassandra.schema.CompressionParams; -import org.apache.cassandra.utils.concurrent.Transactional; import org.apache.cassandra.utils.concurrent.Ref; +import org.apache.cassandra.utils.concurrent.Transactional; +import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable; /** * Holds metadata about compressed file + * TODO extract interface ICompressionMetadata which will just provide non-resource properties */ -public class CompressionMetadata +public class CompressionMetadata extends WrappedSharedCloseable { // dataLength can represent either the true length of the file // or some shorter value, in the case we want to impose a shorter limit on readers @@ -59,42 +63,18 @@ public class CompressionMetadata public final long compressedFileLength; private final Memory chunkOffsets; private final long chunkOffsetsSize; - public final String indexFilePath; + public final File chunksIndexFile; public final CompressionParams parameters; - /** - * Create metadata about given compressed file including uncompressed data length, chunk size - * and list of the chunk offsets of the compressed data. - * - * This is an expensive operation! Don't create more than one for each - * sstable. - * - * @param dataFilePath Path to the compressed file - * - * @return metadata about given compressed file. - */ - public static CompressionMetadata create(String dataFilePath) - { - return createWithLength(dataFilePath, new File(dataFilePath).length()); - } - - public static CompressionMetadata createWithLength(String dataFilePath, long compressedLength) - { - return new CompressionMetadata(Descriptor.fromFilename(dataFilePath), compressedLength); - } - @VisibleForTesting - public CompressionMetadata(Descriptor desc, long compressedLength) + @SuppressWarnings("resource") + public static CompressionMetadata open(File chunksIndexFile, long compressedLength, boolean hasMaxCompressedSize) { - this(desc.filenameFor(Component.COMPRESSION_INFO), compressedLength, desc.version.hasMaxCompressedLength()); - } + CompressionParams parameters; + long dataLength; + Memory chunkOffsets; - @VisibleForTesting - public CompressionMetadata(String indexFilePath, long compressedLength, boolean hasMaxCompressedSize) - { - this.indexFilePath = indexFilePath; - - try (FileInputStreamPlus stream = new File(indexFilePath).newInputStream()) + try (FileInputStreamPlus stream = chunksIndexFile.newInputStream()) { String compressorName = stream.readUTF(); int optionCount = stream.readInt(); @@ -119,7 +99,6 @@ public class CompressionMetadata } dataLength = stream.readLong(); - compressedFileLength = compressedLength; chunkOffsets = readChunkOffsets(stream); } catch (FileNotFoundException | NoSuchFileException e) @@ -128,22 +107,39 @@ public class CompressionMetadata } catch (IOException e) { - throw new CorruptSSTableException(e, indexFilePath); + throw new CorruptSSTableException(e, chunksIndexFile); } - this.chunkOffsetsSize = chunkOffsets.size(); + return new CompressionMetadata(chunksIndexFile, parameters, chunkOffsets, chunkOffsets.size(), dataLength, compressedLength); } // do not call this constructor directly, unless used in testing @VisibleForTesting - public CompressionMetadata(String filePath, CompressionParams parameters, Memory offsets, long offsetsSize, long dataLength, long compressedLength) + public CompressionMetadata(File chunksIndexFile, + CompressionParams parameters, + Memory chunkOffsets, + long chunkOffsetsSize, + long dataLength, + long compressedFileLength) { - this.indexFilePath = filePath; + super(chunkOffsets); + this.chunksIndexFile = chunksIndexFile; this.parameters = parameters; this.dataLength = dataLength; - this.compressedFileLength = compressedLength; - this.chunkOffsets = offsets; - this.chunkOffsetsSize = offsetsSize; + this.compressedFileLength = compressedFileLength; + this.chunkOffsets = chunkOffsets; + this.chunkOffsetsSize = chunkOffsetsSize; + } + + private CompressionMetadata(CompressionMetadata copy) + { + super(copy); + this.chunksIndexFile = copy.chunksIndexFile; + this.parameters = copy.parameters; + this.dataLength = copy.dataLength; + this.compressedFileLength = copy.compressedFileLength; + this.chunkOffsets = copy.chunkOffsets; + this.chunkOffsetsSize = copy.chunkOffsetsSize; } public ICompressor compressor() @@ -170,11 +166,19 @@ public class CompressionMetadata return chunkOffsets.size(); } + @Override public void addTo(Ref.IdentityCollection identities) { + super.addTo(identities); identities.add(chunkOffsets); } + @Override + public CompressionMetadata sharedCopy() + { + return new CompressionMetadata(this); + } + /** * Read offsets of the individual chunks from the given input. * @@ -182,7 +186,7 @@ public class CompressionMetadata * * @return collection of the chunk offsets. */ - private Memory readChunkOffsets(DataInput input) + private static Memory readChunkOffsets(FileInputStreamPlus input) { final int chunkCount; try @@ -193,7 +197,7 @@ public class CompressionMetadata } catch (IOException e) { - throw new FSReadError(e, indexFilePath); + throw new FSReadError(e, input.file); } @SuppressWarnings("resource") @@ -217,10 +221,10 @@ public class CompressionMetadata if (e instanceof EOFException) { String msg = String.format("Corrupted Index File %s: read %d but expected %d chunks.", - indexFilePath, i, chunkCount); - throw new CorruptSSTableException(new IOException(msg, e), indexFilePath); + input.file.path(), i, chunkCount); + throw new CorruptSSTableException(new IOException(msg, e), input.file); } - throw new FSReadError(e, indexFilePath); + throw new FSReadError(e, input.file); } } @@ -236,11 +240,11 @@ public class CompressionMetadata long idx = 8 * (position / parameters.chunkLength()); if (idx >= chunkOffsetsSize) - throw new CorruptSSTableException(new EOFException(), indexFilePath); + throw new CorruptSSTableException(new EOFException(), chunksIndexFile); if (idx < 0) throw new CorruptSSTableException(new IllegalArgumentException(String.format("Invalid negative chunk index %d with position %d", idx, position)), - indexFilePath); + chunksIndexFile); long chunkOffset = chunkOffsets.getLong(idx); long nextChunkOffset = (idx + 8 == chunkOffsetsSize) @@ -250,6 +254,28 @@ public class CompressionMetadata return new Chunk(chunkOffset, (int) (nextChunkOffset - chunkOffset - 4)); // "4" bytes reserved for checksum } + public long getDataOffsetForChunkOffset(long chunkOffset) + { + long l = 0; + long h = (chunkOffsetsSize >> 3) - 1; + long idx, offset; + + while (l <= h) + { + idx = (l + h) >>> 1; + offset = chunkOffsets.getLong(idx << 3); + + if (offset < chunkOffset) + l = idx + 1; + else if (offset > chunkOffset) + h = idx - 1; + else + return idx * parameters.chunkLength(); + } + + throw new IllegalArgumentException("No chunk with offset " + chunkOffset); + } + /** * @param sections Collection of sections in uncompressed file. Should not contain sections that overlap each other. * @return Total chunk size in bytes for given sections including checksum. @@ -314,16 +340,11 @@ public class CompressionMetadata return offsets.toArray(new Chunk[offsets.size()]); } - public void close() - { - chunkOffsets.close(); - } - public static class Writer extends Transactional.AbstractTransactional implements Transactional { // path to the file private final CompressionParams parameters; - private final String filePath; + private final File file; private int maxCount = 100; private SafeMemory offsets = new SafeMemory(maxCount * 8L); private int count = 0; @@ -331,15 +352,15 @@ public class CompressionMetadata // provided by user when setDescriptor private long dataLength, chunkCount; - private Writer(CompressionParams parameters, String path) + private Writer(CompressionParams parameters, File file) { this.parameters = parameters; - filePath = path; + this.file = file; } - public static Writer open(CompressionParams parameters, String path) + public static Writer open(CompressionParams parameters, File file) { - return new Writer(parameters, path); + return new Writer(parameters, file); } public void addOffset(long offset) @@ -374,7 +395,7 @@ public class CompressionMetadata } catch (IOException e) { - throw new FSWriteError(e, filePath); + throw new FSWriteError(e, file); } } @@ -386,6 +407,7 @@ public class CompressionMetadata return this; } + @Override public void doPrepare() { assert chunkCount == count; @@ -400,7 +422,7 @@ public class CompressionMetadata } // flush the data to disk - try (FileOutputStreamPlus out = new FileOutputStreamPlus(filePath)) + try (FileOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE)) { writeHeader(out, dataLength, count); for (int i = 0; i < count; i++) @@ -415,7 +437,7 @@ public class CompressionMetadata } catch (IOException e) { - throw new FSWriteError(e, filePath); + throw new FSWriteError(e, file); } } @@ -434,7 +456,7 @@ public class CompressionMetadata if (tCount < this.count) compressedLength = tOffsets.getLong(tCount * 8L); - return new CompressionMetadata(filePath, parameters, tOffsets, tCount * 8L, dataLength, compressedLength); + return new CompressionMetadata(file, parameters, tOffsets, tCount * 8L, dataLength, compressedLength); } /** @@ -460,16 +482,19 @@ public class CompressionMetadata count = chunkIndex; } + @Override protected Throwable doPostCleanup(Throwable failed) { return offsets.close(failed); } + @Override protected Throwable doCommit(Throwable accumulate) { return accumulate; } + @Override protected Throwable doAbort(Throwable accumulate) { return accumulate; @@ -494,6 +519,7 @@ public class CompressionMetadata this.length = length; } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -503,6 +529,7 @@ public class CompressionMetadata return length == chunk.length && offset == chunk.offset; } + @Override public int hashCode() { int result = (int) (offset ^ (offset >>> 32)); @@ -510,6 +537,7 @@ public class CompressionMetadata return result; } + @Override public String toString() { return String.format("Chunk", offset, length); @@ -518,17 +546,20 @@ public class CompressionMetadata static class ChunkSerializer implements IVersionedSerializer { + @Override public void serialize(Chunk chunk, DataOutputPlus out, int version) throws IOException { out.writeLong(chunk.offset); out.writeInt(chunk.length); } + @Override public Chunk deserialize(DataInputPlus in, int version) throws IOException { return new Chunk(in.readLong(), in.readInt()); } + @Override public long serializedSize(Chunk chunk, int version) { long size = TypeSizes.sizeof(chunk.offset); diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java b/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java new file mode 100644 index 0000000000..15b1161cf9 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/AbstractMetricsProviders.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.cassandra.io.sstable.format.SSTableReader; + +public abstract class AbstractMetricsProviders implements MetricsProviders +{ + protected final GaugeProvider newGaugeProvider(String name, Function, T> combiner) + { + return new SimpleGaugeProvider<>(this::map, name, combiner); + } + + protected final GaugeProvider newGaugeProvider(String name, T neutralValue, Function extractor, BiFunction combiner) + { + return new SimpleGaugeProvider<>(this::map, name, readers -> { + T total = neutralValue; + for (R reader : readers) + total = combiner.apply(total, extractor.apply(reader)); + return total; + }); + } + + protected abstract R map(SSTableReader r); +} diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java new file mode 100644 index 0000000000..b52b562eb6 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/AbstractRowIndexEntry.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.io.IOException; + +import org.apache.cassandra.cache.IMeasurableMemory; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * The base RowIndexEntry is not stored on disk, only specifies a position in the data file + */ +public abstract class AbstractRowIndexEntry implements IMeasurableMemory +{ + public final long position; + + public AbstractRowIndexEntry(long position) + { + this.position = position; + } + + /** + * Row position in a data file + */ + public long getPosition() + { + return position; + } + + /** + * @return true if this index entry contains the row-level tombstone and column summary. Otherwise, + * caller should fetch these from the row header. + */ + public boolean isIndexed() + { + return columnsIndexCount() > 1; + } + + public DeletionTime deletionTime() + { + throw new UnsupportedOperationException(); + } + + public int columnsIndexCount() + { + return 0; + } + + public abstract SSTableFormat getSSTableFormat(); + + /** + * Serialize this entry for key cache + * + * @param out the output stream for serialized entry + */ + public abstract void serializeForCache(DataOutputPlus out) throws IOException; +} diff --git a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java similarity index 60% rename from src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java rename to src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java index 8a266a8e9a..46c212979d 100644 --- a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java @@ -15,20 +15,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.columniterator; +package org.apache.cassandra.io.sstable; +import java.io.Closeable; import java.io.IOException; -import java.util.Comparator; import java.util.Iterator; import java.util.NoSuchElementException; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.BufferClusteringBound; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.UnfilteredDeserializer; +import org.apache.cassandra.db.UnfilteredValidation; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.IndexInfo; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredSerializer; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.DataPosition; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.schema.TableMetadata; @@ -37,7 +51,7 @@ import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.utils.vint.VIntCoding.VIntOutOfRangeException; -public abstract class AbstractSSTableIterator implements UnfilteredRowIterator +public abstract class AbstractSSTableIterator implements UnfilteredRowIterator { protected final SSTableReader sstable; // We could use sstable.metadata(), but that can change during execution so it's good hygiene to grab an immutable instance @@ -62,7 +76,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator protected AbstractSSTableIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, - RowIndexEntry indexEntry, + RIE indexEntry, Slices slices, ColumnFilter columnFilter, FileHandle ifile) @@ -179,9 +193,9 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator } } - protected abstract Reader createReaderInternal(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile); + protected abstract Reader createReaderInternal(RIE indexEntry, FileDataInput file, boolean shouldCloseFile); - private Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile) + private Reader createReader(RIE indexEntry, FileDataInput file, boolean shouldCloseFile) { return slices.isEmpty() ? new NoRowsReader(file, shouldCloseFile) : createReaderInternal(indexEntry, file, shouldCloseFile); @@ -258,7 +272,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator e.addSuppressed(suppressed); } sstable.markSuspect(); - throw new CorruptSSTableException(e, reader.file.getPath()); + throw new CorruptSSTableException(e, reader.toString()); } } @@ -289,21 +303,28 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator catch (IOException e) { sstable.markSuspect(); - throw new CorruptSSTableException(e, reader.file.getPath()); + throw new CorruptSSTableException(e, reader.toString()); } } - protected abstract class Reader implements Iterator + public interface Reader extends Iterator, Closeable { + + void setForSlice(Slice slice) throws IOException; + + void seekToPosition(long columnOffset) throws IOException; + } + + public abstract class AbstractReader implements Reader { private final boolean shouldCloseFile; public FileDataInput file; - protected UnfilteredDeserializer deserializer; + public UnfilteredDeserializer deserializer; // Records the currently open range tombstone (if any) - protected DeletionTime openMarker = null; + public DeletionTime openMarker; - protected Reader(FileDataInput file, boolean shouldCloseFile) + protected AbstractReader(FileDataInput file, boolean shouldCloseFile) { this.file = file; this.shouldCloseFile = shouldCloseFile; @@ -318,7 +339,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator deserializer = UnfilteredDeserializer.create(metadata, file, sstable.header, helper); } - protected void seekToPosition(long position) throws IOException + public void seekToPosition(long position) throws IOException { // This may be the first time we're actually looking into the file if (file == null) @@ -329,6 +350,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator else { file.seek(position); + deserializer.clearState(); } } @@ -355,7 +377,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator e.addSuppressed(suppressed); } sstable.markSuspect(); - throw new CorruptSSTableException(e, reader.file.getPath()); + throw new CorruptSSTableException(e, toString()); } } @@ -376,7 +398,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator e.addSuppressed(suppressed); } sstable.markSuspect(); - throw new CorruptSSTableException(e, reader.file.getPath()); + throw new CorruptSSTableException(e, toString()); } } @@ -386,229 +408,178 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator protected abstract boolean hasNextInternal() throws IOException; protected abstract Unfiltered nextInternal() throws IOException; + @Override public void close() throws IOException { if (shouldCloseFile && file != null) file.close(); } + + @Override + public String toString() + { + return file != null ? file.toString() : "null"; + } } - // Reader for when we have Slices.NONE but need to read static row or partition level deletion - private class NoRowsReader extends AbstractSSTableIterator.Reader + protected class ForwardReader extends AbstractReader { - private NoRowsReader(FileDataInput file, boolean shouldCloseFile) + // The start of the current slice. This will be null as soon as we know we've passed that bound. + protected ClusteringBound start; + // The end of the current slice. Will never be null. + protected ClusteringBound end = BufferClusteringBound.TOP; + + protected Unfiltered next; // the next element to return: this is computed by hasNextInternal(). + + protected boolean sliceDone; // set to true once we know we have no more result for the slice. This is in particular + // used by the indexed reader when we know we can't have results based on the index. + + public ForwardReader(FileDataInput file, boolean shouldCloseFile) { super(file, shouldCloseFile); } public void setForSlice(Slice slice) throws IOException { - return; + start = slice.start().isBottom() ? null : slice.start(); + end = slice.end(); + + sliceDone = false; + next = null; + } + + // Skip all data that comes before the currently set slice. + // Return what should be returned at the end of this, or null if nothing should. + private Unfiltered handlePreSliceData() throws IOException + { + assert deserializer != null; + + // Note that the following comparison is not strict. The reason is that the only cases + // where it can be == is if the "next" is a RT start marker (either a '[' of a ')[' boundary), + // and if we had a strict inequality and an open RT marker before this, we would issue + // the open marker first, and then return then next later, which would send in the + // stream both '[' (or '(') and then ')[' for the same clustering value, which is wrong. + // By using a non-strict inequality, we avoid that problem (if we do get ')[' for the same + // clustering value than the slice, we'll simply record it in 'openMarker'). + while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0) + { + if (deserializer.nextIsRow()) + deserializer.skipNext(); + else + updateOpenMarker((RangeTombstoneMarker)deserializer.readNext()); + } + + ClusteringBound sliceStart = start; + start = null; + + // We've reached the beginning of our queried slice. If we have an open marker + // we should return that first. + if (openMarker != null) + return new RangeTombstoneBoundMarker(sliceStart, openMarker); + + return null; + } + + // Compute the next element to return, assuming we're in the middle to the slice + // and the next element is either in the slice, or just after it. Returns null + // if we're done with the slice. + protected Unfiltered computeNext() throws IOException + { + assert deserializer != null; + + while (true) + { + // We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below. + // We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those + // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound + // (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because + // it's fundamentally excluded. And if the bound is a end (for a range tombstone), it means it's exactly + // our slice end, but in that case we will properly close the range tombstone anyway as part of our "close + // an open marker" code in hasNextInterna + if (!deserializer.hasNext() || deserializer.compareNextTo(end) >= 0) + return null; + + Unfiltered next = deserializer.readNext(); + UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable); + // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne. + if (next.isEmpty()) + continue; + + if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + updateOpenMarker((RangeTombstoneMarker) next); + return next; + } } protected boolean hasNextInternal() throws IOException { + if (next != null) + return true; + + if (sliceDone) + return false; + + if (start != null) + { + Unfiltered unfiltered = handlePreSliceData(); + if (unfiltered != null) + { + next = unfiltered; + return true; + } + } + + next = computeNext(); + if (next != null) + return true; + + // for current slice, no data read from deserialization + sliceDone = true; + // If we have an open marker, we should not close it, there could be more slices + if (openMarker != null) + { + next = new RangeTombstoneBoundMarker(end, openMarker); + return true; + } return false; } + protected Unfiltered nextInternal() throws IOException + { + if (!hasNextInternal()) + throw new NoSuchElementException(); + + Unfiltered toReturn = next; + next = null; + return toReturn; + } + } + + + // Reader for when we have Slices.NONE but need to read static row or partition level deletion + private class NoRowsReader extends AbstractReader + { + private NoRowsReader(FileDataInput file, boolean shouldCloseFile) + { + super(file, shouldCloseFile); + } + + @Override + public void setForSlice(Slice slice) + { + // no-op + } + + @Override + public boolean hasNextInternal() + { + return false; + } + + @Override protected Unfiltered nextInternal() throws IOException { throw new NoSuchElementException(); } } - - // Used by indexed readers to store where they are of the index. - public static class IndexState implements AutoCloseable - { - private final Reader reader; - private final ClusteringComparator comparator; - - private final RowIndexEntry indexEntry; - private final RowIndexEntry.IndexInfoRetriever indexInfoRetriever; - private final boolean reversed; - - private int currentIndexIdx; - - // Marks the beginning of the block corresponding to currentIndexIdx. - private DataPosition mark; - - public IndexState(Reader reader, ClusteringComparator comparator, RowIndexEntry indexEntry, boolean reversed, FileHandle indexFile) - { - this.reader = reader; - this.comparator = comparator; - this.indexEntry = indexEntry; - this.indexInfoRetriever = indexEntry.openWithIndex(indexFile); - this.reversed = reversed; - this.currentIndexIdx = reversed ? indexEntry.columnsIndexCount() : -1; - } - - public boolean isDone() - { - return reversed ? currentIndexIdx < 0 : currentIndexIdx >= indexEntry.columnsIndexCount(); - } - - // Sets the reader to the beginning of blockIdx. - public void setToBlock(int blockIdx) throws IOException - { - if (blockIdx >= 0 && blockIdx < indexEntry.columnsIndexCount()) - { - reader.seekToPosition(columnOffset(blockIdx)); - mark = reader.file.mark(); - reader.deserializer.clearState(); - } - - currentIndexIdx = blockIdx; - reader.openMarker = blockIdx > 0 ? index(blockIdx - 1).endOpenMarker : null; - } - - private long columnOffset(int i) throws IOException - { - return indexEntry.position + index(i).offset; - } - - public int blocksCount() - { - return indexEntry.columnsIndexCount(); - } - - // Update the block idx based on the current reader position if we're past the current block. - // This only makes sense for forward iteration (for reverse ones, when we reach the end of a block we - // should seek to the previous one, not update the index state and continue). - public void updateBlock() throws IOException - { - assert !reversed; - - // If we get here with currentBlockIdx < 0, it means setToBlock() has never been called, so it means - // we're about to read from the beginning of the partition, but haven't "prepared" the IndexState yet. - // Do so by setting us on the first block. - if (currentIndexIdx < 0) - { - setToBlock(0); - return; - } - - while (currentIndexIdx + 1 < indexEntry.columnsIndexCount() && isPastCurrentBlock()) - { - reader.openMarker = currentIndex().endOpenMarker; - ++currentIndexIdx; - - // We have to set the mark, and we have to set it at the beginning of the block. So if we're not at the beginning of the block, this forces us to a weird seek dance. - // This can only happen when reading old file however. - long startOfBlock = columnOffset(currentIndexIdx); - long currentFilePointer = reader.file.getFilePointer(); - if (startOfBlock == currentFilePointer) - { - mark = reader.file.mark(); - } - else - { - reader.seekToPosition(startOfBlock); - mark = reader.file.mark(); - reader.seekToPosition(currentFilePointer); - } - } - } - - // Check if we've crossed an index boundary (based on the mark on the beginning of the index block). - public boolean isPastCurrentBlock() throws IOException - { - assert reader.deserializer != null; - return reader.file.bytesPastMark(mark) >= currentIndex().width; - } - - public int currentBlockIdx() - { - return currentIndexIdx; - } - - public IndexInfo currentIndex() throws IOException - { - return index(currentIndexIdx); - } - - public IndexInfo index(int i) throws IOException - { - return indexInfoRetriever.columnsIndex(i); - } - - // Finds the index of the first block containing the provided bound, starting at the provided index. - // Will be -1 if the bound is before any block, and blocksCount() if it is after every block. - public int findBlockIndex(ClusteringBound bound, int fromIdx) throws IOException - { - if (bound.isBottom()) - return -1; - if (bound.isTop()) - return blocksCount(); - - return indexFor(bound, fromIdx); - } - - public int indexFor(ClusteringPrefix name, int lastIndex) throws IOException - { - IndexInfo target = new IndexInfo(name, name, 0, 0, null); - /* - Take the example from the unit test, and say your index looks like this: - [0..5][10..15][20..25] - and you look for the slice [13..17]. - - When doing forward slice, we are doing a binary search comparing 13 (the start of the query) - to the lastName part of the index slot. You'll end up with the "first" slot, going from left to right, - that may contain the start. - - When doing a reverse slice, we do the same thing, only using as a start column the end of the query, - i.e. 17 in this example, compared to the firstName part of the index slots. bsearch will give us the - first slot where firstName > start ([20..25] here), so we subtract an extra one to get the slot just before. - */ - int startIdx = 0; - int endIdx = indexEntry.columnsIndexCount() - 1; - - if (reversed) - { - if (lastIndex < endIdx) - { - endIdx = lastIndex; - } - } - else - { - if (lastIndex > 0) - { - startIdx = lastIndex; - } - } - - int index = binarySearch(target, comparator.indexComparator(reversed), startIdx, endIdx); - return (index < 0 ? -index - (reversed ? 2 : 1) : index); - } - - private int binarySearch(IndexInfo key, Comparator c, int low, int high) throws IOException - { - while (low <= high) - { - int mid = (low + high) >>> 1; - IndexInfo midVal = index(mid); - int cmp = c.compare(midVal, key); - - if (cmp < 0) - low = mid + 1; - else if (cmp > 0) - high = mid - 1; - else - return mid; - } - return -(low + 1); - } - - @Override - public String toString() - { - return String.format("IndexState(indexSize=%d, currentBlock=%d, reversed=%b)", indexEntry.columnsIndexCount(), currentIndexIdx, reversed); - } - - @Override - public void close() throws IOException - { - indexInfoRetriever.close(); - } - } } diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index 49ceb764de..0288ebc1d5 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -18,8 +18,8 @@ package org.apache.cassandra.io.sstable; -import java.io.IOException; import java.io.Closeable; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; @@ -27,7 +27,9 @@ import java.util.Collections; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.io.sstable.format.SSTableFormat; @@ -64,7 +66,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable this.makeRangeAware = makeRangeAware; } - protected SSTableTxnWriter createWriter() throws IOException + protected SSTableTxnWriter createWriter(SSTable.Owner owner) throws IOException { SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS); @@ -79,7 +81,8 @@ abstract class AbstractSSTableSimpleWriter implements Closeable false, 0, header, - Collections.emptySet()); + Collections.emptySet(), + owner); } private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) throws IOException @@ -95,7 +98,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable try (Stream existingPaths = Files.list(directory.toPath())) { Stream existingIds = existingPaths.map(File::new) - .map(SSTable::tryDescriptorFromFilename) + .map(SSTable::tryDescriptorFromFile) .filter(d -> d != null && d.cfname.equals(columnFamily)) .map(d -> d.id); diff --git a/src/java/org/apache/cassandra/io/sstable/Component.java b/src/java/org/apache/cassandra/io/sstable/Component.java index d87d8b7d4f..13c63859df 100644 --- a/src/java/org/apache/cassandra/io/sstable/Component.java +++ b/src/java/org/apache/cassandra/io/sstable/Component.java @@ -17,11 +17,18 @@ */ package org.apache.cassandra.io.sstable; -import java.util.EnumSet; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.regex.Pattern; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; + +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types; /** * SSTables are made up of multiple components in separate files. Components are @@ -32,84 +39,149 @@ public class Component { public static final char separator = '-'; - final static EnumSet TYPES = EnumSet.allOf(Type.class); - /** * WARNING: Be careful while changing the names or string representation of the enum * members. Streaming code depends on the names during streaming (Ref: CASSANDRA-14556). */ - public enum Type + public final static class Type { - // the base data for an sstable: the remaining components can be regenerated - // based on the data component - DATA("Data.db"), - // index of the row keys with pointers to their positions in the data file - PRIMARY_INDEX("Index.db"), - // serialized bloom filter for the row keys in the sstable - FILTER("Filter.db"), - // file to hold information about uncompressed data length, chunk offsets etc. - COMPRESSION_INFO("CompressionInfo.db"), - // statistical metadata about the content of the sstable - STATS("Statistics.db"), - // holds CRC32 checksum of the data file - DIGEST("Digest.crc32"), - // holds the CRC32 for chunks in an a uncompressed file. - CRC("CRC.db"), - // holds SSTable Index Summary (sampling of Index component) - SUMMARY("Summary.db"), - // table of contents, stores the list of all components for the sstable - TOC("TOC.txt"), - // built-in secondary index (may be multiple per sstable) - SECONDARY_INDEX("SI_.*.db"), - // custom component, used by e.g. custom compaction strategy - CUSTOM(null); + private final static CopyOnWriteArrayList typesCollector = new CopyOnWriteArrayList<>(); - final String repr; + public static final List all = Collections.unmodifiableList(typesCollector); - Type(String repr) + public final int id; + public final String name; + public final String repr; + private final Component singleton; + + @SuppressWarnings("rawtypes") + public final Class formatClass; + + /** + * Creates a new non-singleton type and registers it a global type registry - see {@link #registerType(Type)}. + * + * @param name type name, must be unique for this and all parent formats + * @param repr the regular expression to be used to recognize a name represents this type + * @param formatClass format class for which this type is defined for + */ + public static Type create(String name, String repr, Class> formatClass) { + return new Type(name, repr, false, formatClass); + } + + /** + * Creates a new singleton type and registers it in a global type registry - see {@link #registerType(Type)}. + * + * @param name type name, must be unique for this and all parent formats + * @param repr the regular expression to be used to recognize a name represents this type + * @param formatClass format class for which this type is defined for + */ + public static Type createSingleton(String name, String repr, Class> formatClass) + { + return new Type(name, repr, true, formatClass); + } + + private Type(String name, String repr, boolean isSingleton, Class> formatClass) + { + this.name = Objects.requireNonNull(name); this.repr = repr; + this.id = typesCollector.size(); + this.formatClass = formatClass == null ? SSTableFormat.class : formatClass; + this.singleton = isSingleton ? new Component(this) : null; + + registerType(this); + } + + /** + * If you have two formats registered, they may both define a type say `INDEX`. It is allowed even though + * they have the same name because they are not in the same branch. Though, we cannot let a custom type + * define a type `TOC` which is declared on the top level. + * So, e.g. given we have `TOC@SSTableFormat`, and `BigFormat` tries to define `TOC@BigFormat`, we should + * forbid that; but, given we have `INDEX@BigFormat`, we should allow to define `INDEX@TrieFormat` as those + * types are be distinguishable via format type. + * + * @param type a type to be registered + */ + private static void registerType(Type type) + { + synchronized (typesCollector) + { + if (typesCollector.stream().anyMatch(t -> (Objects.equals(t.name, type.name) || Objects.equals(t.repr, type.repr)) && (t.formatClass.isAssignableFrom(type.formatClass)))) + throw new AssertionError("Type named " + type.name + " is already registered"); + + typesCollector.add(type); + } } @VisibleForTesting - public static Type fromRepresentation(String repr) + public static Type fromRepresentation(String repr, SSTableFormat.Type formatType) { - for (Type type : TYPES) + for (Type type : Type.all) { - if (type.repr != null && Pattern.matches(type.repr, repr)) + if (type.repr != null && Pattern.matches(type.repr, repr) && type.formatClass.isAssignableFrom(formatType.info.getClass())) return type; } - return CUSTOM; + return Types.CUSTOM; + } + + public static Component createComponent(String repr, SSTableFormat.Type formatType) + { + Type type = fromRepresentation(repr, formatType); + if (type.singleton != null) + return type.singleton; + else + return new Component(type, repr); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Type type = (Type) o; + return id == type.id; + } + + @Override + public int hashCode() + { + return id; + } + + @Override + public String toString() + { + return name; + } + + public Component getSingleton() + { + return Objects.requireNonNull(singleton); + } + + public Component createComponent(String repr) + { + Preconditions.checkArgument(singleton == null); + return new Component(this, repr); } } - // singleton components for types that don't need ids - public final static Component DATA = new Component(Type.DATA); - public final static Component PRIMARY_INDEX = new Component(Type.PRIMARY_INDEX); - public final static Component FILTER = new Component(Type.FILTER); - public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO); - public final static Component STATS = new Component(Type.STATS); - public final static Component DIGEST = new Component(Type.DIGEST); - public final static Component CRC = new Component(Type.CRC); - public final static Component SUMMARY = new Component(Type.SUMMARY); - public final static Component TOC = new Component(Type.TOC); - public final Type type; public final String name; public final int hashCode; - public Component(Type type) + private Component(Type type) { this(type, type.repr); - assert type != Type.CUSTOM; } - public Component(Type type, String name) + private Component(Type type, String name) { assert name != null : "Component name cannot be null"; + this.type = type; this.name = name; - this.hashCode = Objects.hashCode(type, name); + this.hashCode = Objects.hash(type, name); } /** @@ -127,26 +199,24 @@ public class Component * @return the component corresponding to {@code name}. Note that this always return a component as an unrecognized * name is parsed into a CUSTOM component. */ - public static Component parse(String name) + public static Component parse(String name, SSTableFormat.Type formatType) { - Type type = Type.fromRepresentation(name); + return Type.createComponent(name, formatType); + } - // Build (or retrieve singleton for) the component object - switch (type) - { - case DATA: return Component.DATA; - case PRIMARY_INDEX: return Component.PRIMARY_INDEX; - case FILTER: return Component.FILTER; - case COMPRESSION_INFO: return Component.COMPRESSION_INFO; - case STATS: return Component.STATS; - case DIGEST: return Component.DIGEST; - case CRC: return Component.CRC; - case SUMMARY: return Component.SUMMARY; - case TOC: return Component.TOC; - case SECONDARY_INDEX: return new Component(Type.SECONDARY_INDEX, name); - case CUSTOM: return new Component(Type.CUSTOM, name); - default: throw new AssertionError(); - } + public static Iterable getSingletonsFor(SSTableFormat format) + { + return Iterables.transform(Iterables.filter(Type.all, t -> t.singleton != null && t.formatClass.isAssignableFrom(format.getClass())), t -> t.singleton); + } + + public static Iterable getSingletonsFor(Class> formatClass) + { + return Iterables.transform(Iterables.filter(Type.all, t -> t.singleton != null && t.formatClass.isAssignableFrom(formatClass)), t -> t.singleton); + } + + public boolean isValidFor(Descriptor descriptor) + { + return type.formatClass.isAssignableFrom(descriptor.formatType.info.getClass()); } @Override @@ -162,8 +232,8 @@ public class Component return true; if (!(o instanceof Component)) return false; - Component that = (Component)o; - return this.type == that.type && this.name.equals(that.name); + Component that = (Component) o; + return this.hashCode == that.hashCode && this.type == that.type && this.name.equals(that.name); } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java index 23668f966a..b7dcdcd0d1 100644 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@ -17,14 +17,17 @@ */ package org.apache.cassandra.io.sstable; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.base.Splitter; - +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +39,8 @@ import org.apache.cassandra.io.sstable.metadata.MetadataSerializer; import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.Pair; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.cassandra.io.sstable.Component.separator; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -84,6 +89,8 @@ public class Descriptor public final SSTableId id; public final SSTableFormat.Type formatType; private final int hashCode; + private final String prefix; + private final File baseFile; /** * A descriptor that assumes CURRENT_VERSION. @@ -110,7 +117,13 @@ public class Descriptor public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType) { - assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass()); + checkNotNull(version); + checkNotNull(directory); + checkNotNull(ksname); + checkNotNull(cfname); + checkNotNull(formatType); + checkArgument(version.getSSTableFormat() == formatType.info); + this.version = version; this.directory = directory.toCanonical(); this.ksname = ksname; @@ -118,41 +131,53 @@ public class Descriptor this.id = id; this.formatType = formatType; + StringBuilder buf = new StringBuilder(); + appendFileName(buf); + this.prefix = buf.toString(); + this.baseFile = new File(directory.toPath().resolve(prefix)); + // directory is unnecessary for hashCode, and for simulator consistency we do not include it hashCode = Objects.hashCode(version, id, ksname, cfname, formatType); } - public String tmpFilenameFor(Component component) + private String tmpFilenameFor(Component component) { - return filenameFor(component) + TMP_EXT; + return fileFor(component) + TMP_EXT; } - /** - * @return a unique temporary file name for given component during entire-sstable-streaming. - */ - public String tmpFilenameForStreaming(Component component) + public File tmpFileFor(Component component) + { + return new File(directory.toPath().resolve(tmpFilenameFor(component))); + } + + private String tmpFilenameForStreaming(Component component) { // Use UUID to handle concurrent streamings on the same sstable. // TMP_EXT allows temp file to be removed by {@link ColumnFamilyStore#scrubDataDirectories} return String.format("%s.%s%s", filenameFor(component), nextTimeUUID(), TMP_EXT); } - public String filenameFor(Component component) + /** + * @return a unique temporary file name for given component during entire-sstable-streaming. + */ + public File tmpFileForStreaming(Component component) { - return baseFilename() + separator + component.name(); + return new File(directory.toPath().resolve(tmpFilenameForStreaming(component))); + } + + private String filenameFor(Component component) + { + return prefix + separator + component.name(); } public File fileFor(Component component) { - return new File(filenameFor(component)); + return new File(directory.toPath().resolve(filenameFor(component))); } - public String baseFilename() + public File baseFile() { - StringBuilder buff = new StringBuilder(); - buff.append(directory).append(File.pathSeparator()); - appendFileName(buff); - return buff.toString(); + return baseFile; } private void appendFileName(StringBuilder buff) @@ -175,7 +200,7 @@ public class Descriptor return buff.toString(); } - public SSTableFormat getFormat() + public SSTableFormat getFormat() { return formatType.info; } @@ -193,29 +218,28 @@ public class Descriptor return ret; } + /** + * Returns the set of components consisting of the provided mandatory components and those optional components + * for which the corresponding file exists. + */ + public Set getComponents(Set mandatory, Set optional) + { + ImmutableSet.Builder builder = ImmutableSet.builder(); + builder.addAll(mandatory); + for (Component component : optional) + { + if (fileFor(component).exists()) + builder.add(component); + } + return builder.build(); + } + public static boolean isValidFile(File file) { String filename = file.name(); return filename.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(filename).matches(); } - /** - * Parse a sstable filename into a Descriptor. - *

- * This is a shortcut for {@code fromFilename(new File(filename))}. - * - * @param filename the filename to a sstable component. - * @return the descriptor for the parsed file. - * - * @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could - * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported - * versions. - */ - public static Descriptor fromFilename(String filename) - { - return fromFilenameWithComponent(new File(filename), false).left; - } - /** * Parse a sstable filename into a Descriptor. *

@@ -233,9 +257,9 @@ public class Descriptor * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported * versions. */ - public static Descriptor fromFilename(File file) + public static Descriptor fromFile(File file) { - return fromFilenameWithComponent(file).left; + return fromFileWithComponent(file).left; } /** @@ -249,12 +273,12 @@ public class Descriptor * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported * versions. */ - public static Pair fromFilenameWithComponent(File file) + public static Pair fromFileWithComponent(File file) { - return fromFilenameWithComponent(file, true); + return fromFileWithComponent(file, true); } - public static Pair fromFilenameWithComponent(File file, boolean validateDirs) + public static Pair fromFileWithComponent(File file, boolean validateDirs) { // We need to extract the keyspace and table names from the parent directories, so make sure we deal with the // absolute path. @@ -307,11 +331,11 @@ public class Descriptor * mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported * versions. */ - public static Pair fromFilenameWithComponent(File file, String keyspace, String table) + public static Pair fromFileWithComponent(File file, String keyspace, String table) { if (null == keyspace || null == table) { - return fromFilenameWithComponent(file); + return fromFileWithComponent(file); } SSTableInfo info = validateAndExtractInfo(file); @@ -354,14 +378,14 @@ public class Descriptor SSTableFormat.Type format; try { - format = SSTableFormat.Type.validate(formatString); + format = SSTableFormat.Type.getByName(formatString); } catch (RuntimeException e) { throw invalidSSTable(name, "unknown 'format' part (%s)", formatString); } - Component component = Component.parse(tokens.get(3)); + Component component = Component.parse(tokens.get(3), format); Version version = format.info.getVersion(versionString); if (!version.isCompatible()) @@ -412,10 +436,21 @@ public class Descriptor return version.isCompatible(); } + public Set discoverComponents() + { + Set components = Sets.newHashSetWithExpectedSize(Component.Type.all.size()); + for (Component component : Component.getSingletonsFor(formatType.info)) + { + if (fileFor(component).exists()) + components.add(component); + } + return components; + } + @Override public String toString() { - return baseFilename(); + return baseFile().absolutePath(); } @Override @@ -426,6 +461,8 @@ public class Descriptor if (!(o instanceof Descriptor)) return false; Descriptor that = (Descriptor)o; + if (this.hashCode != that.hashCode) + return false; return that.directory.equals(this.directory) && that.id.equals(this.id) && that.ksname.equals(this.ksname) diff --git a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java new file mode 100644 index 0000000000..8976ed4130 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.TableMetadata; + +public class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator implements ISSTableScanner +{ + private final SSTableReader sstable; + + public EmptySSTableScanner(SSTableReader sstable) + { + this.sstable = sstable; + } + + public long getBytesScanned() + { + return 0; + } + + public long getLengthInBytes() + { + return 0; + } + + public long getCompressedLengthInBytes() + { + return 0; + } + + public Set getBackingSSTables() + { + return ImmutableSet.of(sstable); + } + + public long getCurrentPosition() + { + return 0; + } + + public TableMetadata metadata() + { + return sstable.metadata(); + } + + public boolean hasNext() + { + return false; + } + + public UnfilteredRowIterator next() + { + return null; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java b/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java new file mode 100644 index 0000000000..d0680be2a2 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/GaugeProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import com.codahale.metrics.Gauge; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; + +public abstract class GaugeProvider +{ + public final String name; + + public GaugeProvider(String name) + { + this.name = name; + } + + public abstract Gauge getTableGauge(ColumnFamilyStore cfs); + + public abstract Gauge getKeyspaceGauge(Keyspace keyspace); + + public abstract Gauge getGlobalGauge(); +} diff --git a/src/java/org/apache/cassandra/io/sstable/IOOptions.java b/src/java/org/apache/cassandra/io/sstable/IOOptions.java new file mode 100644 index 0000000000..1f395a77ea --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/IOOptions.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.util.DiskOptimizationStrategy; +import org.apache.cassandra.io.util.SequentialWriterOption; + +public class IOOptions +{ + public static IOOptions fromDatabaseDescriptor() + { + return new IOOptions(DatabaseDescriptor.getDiskOptimizationStrategy(), + DatabaseDescriptor.getDiskAccessMode(), + DatabaseDescriptor.getIndexAccessMode(), + DatabaseDescriptor.getDiskOptimizationEstimatePercentile(), + SequentialWriterOption.newBuilder() + .trickleFsync(DatabaseDescriptor.getTrickleFsync()) + .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024) + .build(), + DatabaseDescriptor.getFlushCompression()); + } + + public final DiskOptimizationStrategy diskOptimizationStrategy; + public final Config.DiskAccessMode defaultDiskAccessMode; + public final Config.DiskAccessMode indexDiskAccessMode; + public final double diskOptimizationEstimatePercentile; + public final SequentialWriterOption writerOptions; + public final Config.FlushCompression flushCompression; + + public IOOptions(DiskOptimizationStrategy diskOptimizationStrategy, + Config.DiskAccessMode defaultDiskAccessMode, + Config.DiskAccessMode indexDiskAccessMode, + double diskOptimizationEstimatePercentile, + SequentialWriterOption writerOptions, + Config.FlushCompression flushCompression) + { + this.diskOptimizationStrategy = diskOptimizationStrategy; + this.defaultDiskAccessMode = defaultDiskAccessMode; + this.indexDiskAccessMode = indexDiskAccessMode; + this.diskOptimizationEstimatePercentile = diskOptimizationEstimatePercentile; + this.writerOptions = writerOptions; + this.flushCompression = flushCompression; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/IScrubber.java b/src/java/org/apache/cassandra/io/sstable/IScrubber.java new file mode 100644 index 0000000000..50c6eb35fa --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/IScrubber.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.StringJoiner; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.utils.Closeable; + +public interface IScrubber extends Closeable +{ + void scrub(); + + void close(); + + CompactionInfo.Holder getScrubInfo(); + + @VisibleForTesting + ScrubResult scrubWithResult(); + + static Options.Builder options() + { + return new Options.Builder(); + } + + final class ScrubResult + { + public final int goodPartitions; + public final int badPartitions; + public final int emptyPartitions; + + public ScrubResult(int goodPartitions, int badPartitions, int emptyPartitions) + { + this.goodPartitions = goodPartitions; + this.badPartitions = badPartitions; + this.emptyPartitions = emptyPartitions; + } + } + + class Options + { + public final boolean checkData; + public final boolean reinsertOverflowedTTLRows; + public final boolean skipCorrupted; + + private Options(boolean checkData, boolean reinsertOverflowedTTLRows, boolean skipCorrupted) + { + this.checkData = checkData; + this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows; + this.skipCorrupted = skipCorrupted; + } + + @Override + public String toString() + { + return new StringJoiner(", ", Options.class.getSimpleName() + "[", "]") + .add("checkData=" + checkData) + .add("reinsertOverflowedTTLRows=" + reinsertOverflowedTTLRows) + .add("skipCorrupted=" + skipCorrupted) + .toString(); + } + + public static class Builder + { + private boolean checkData = false; + private boolean reinsertOverflowedTTLRows = false; + private boolean skipCorrupted = false; + + public Builder checkData() + { + this.checkData = true; + return this; + } + + public Builder checkData(boolean checkData) + { + this.checkData = checkData; + return this; + } + + public Builder reinsertOverflowedTTLRows() + { + this.reinsertOverflowedTTLRows = true; + return this; + } + + public Builder reinsertOverflowedTTLRows(boolean reinsertOverflowedTTLRows) + { + this.reinsertOverflowedTTLRows = reinsertOverflowedTTLRows; + return this; + } + + public Builder skipCorrupted() + { + this.skipCorrupted = true; + return this; + } + + public Builder skipCorrupted(boolean skipCorrupted) + { + this.skipCorrupted = skipCorrupted; + return this; + } + + public Options build() + { + return new Options(checkData, reinsertOverflowedTTLRows, skipCorrupted); + } + } + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/IVerifier.java b/src/java/org/apache/cassandra/io/sstable/IVerifier.java new file mode 100644 index 0000000000..62ec0659af --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/IVerifier.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.io.Closeable; +import java.util.Collection; +import java.util.function.Function; + +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.service.StorageService; + +public interface IVerifier extends Closeable +{ + static Options.Builder options() + { + return new Options.Builder(); + } + + void verify(); + + @Override + void close(); + + CompactionInfo.Holder getVerifyInfo(); + + class Options + { + public final boolean invokeDiskFailurePolicy; + + /** + * Force extended verification - unless it is enabled, extended verificiation will be done only + * if there is no digest present. Setting it along with quick makes no sense. + */ + public final boolean extendedVerification; + + public final boolean checkVersion; + public final boolean mutateRepairStatus; + public final boolean checkOwnsTokens; + + /** + * Quick check which does not include sstable data verification. + */ + public final boolean quick; + + public final Function>> tokenLookup; + + private Options(boolean invokeDiskFailurePolicy, + boolean extendedVerification, + boolean checkVersion, + boolean mutateRepairStatus, + boolean checkOwnsTokens, + boolean quick, + Function>> tokenLookup) + { + this.invokeDiskFailurePolicy = invokeDiskFailurePolicy; + this.extendedVerification = extendedVerification; + this.checkVersion = checkVersion; + this.mutateRepairStatus = mutateRepairStatus; + this.checkOwnsTokens = checkOwnsTokens; + this.quick = quick; + this.tokenLookup = tokenLookup; + } + + @Override + public String toString() + { + return "Options{" + + "invokeDiskFailurePolicy=" + invokeDiskFailurePolicy + + ", extendedVerification=" + extendedVerification + + ", checkVersion=" + checkVersion + + ", mutateRepairStatus=" + mutateRepairStatus + + ", checkOwnsTokens=" + checkOwnsTokens + + ", quick=" + quick + + '}'; + } + + public static class Builder + { + private boolean invokeDiskFailurePolicy = false; // invoking disk failure policy can stop the node if we find a corrupt stable + private boolean extendedVerification = false; + private boolean checkVersion = false; + private boolean mutateRepairStatus = false; // mutating repair status can be dangerous + private boolean checkOwnsTokens = false; + private boolean quick = false; + private Function>> tokenLookup = StorageService.instance::getLocalAndPendingRanges; + + public Builder invokeDiskFailurePolicy(boolean param) + { + this.invokeDiskFailurePolicy = param; + return this; + } + + public Builder extendedVerification(boolean param) + { + this.extendedVerification = param; + return this; + } + + public Builder checkVersion(boolean param) + { + this.checkVersion = param; + return this; + } + + public Builder mutateRepairStatus(boolean param) + { + this.mutateRepairStatus = param; + return this; + } + + public Builder checkOwnsTokens(boolean param) + { + this.checkOwnsTokens = param; + return this; + } + + public Builder quick(boolean param) + { + this.quick = param; + return this; + } + + public Builder tokenLookup(Function>> tokenLookup) + { + this.tokenLookup = tokenLookup; + return this; + } + + public Options build() + { + return new Options(invokeDiskFailurePolicy, extendedVerification, checkVersion, mutateRepairStatus, checkOwnsTokens, quick, tokenLookup); + } + } + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java index 8090f55cf9..dbce416e5f 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java @@ -18,17 +18,21 @@ package org.apache.cassandra.io.sstable; -import org.apache.cassandra.db.*; +import java.io.IOException; +import java.util.List; + +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.ObjectSizes; -import java.io.IOException; -import java.util.List; - /** * {@code IndexInfo} is embedded in the indexed version of {@link RowIndexEntry}. * Each instance roughly covers a range of {@link org.apache.cassandra.config.Config#column_index_size column_index_size} KiB diff --git a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java index ceacf87cb7..dbe501f36e 100644 --- a/src/java/org/apache/cassandra/io/sstable/KeyIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/KeyIterator.java @@ -19,103 +19,48 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.AbstractIterator; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CloseableIterator; public class KeyIterator extends AbstractIterator implements CloseableIterator { - private final static class In - { - private final File path; - private volatile RandomAccessReader in; - - public In(File path) - { - this.path = path; - } - - private void maybeInit() - { - if (in != null) - return; - - synchronized (this) - { - if (in == null) - { - in = RandomAccessReader.open(path); - } - } - } - - public DataInputPlus get() - { - maybeInit(); - return in; - } - - public boolean isEOF() - { - maybeInit(); - return in.isEOF(); - } - - public void close() - { - if (in != null) - in.close(); - } - - public long getFilePointer() - { - maybeInit(); - return in.getFilePointer(); - } - - public long length() - { - maybeInit(); - return in.length(); - } - } - - private final Descriptor desc; - private final In in; private final IPartitioner partitioner; + private final KeyReader it; private final ReadWriteLock fileAccessLock; + private final long totalBytes; - private long keyPosition; + private boolean initialized = false; - public KeyIterator(Descriptor desc, TableMetadata metadata) + public KeyIterator(KeyReader it, IPartitioner partitioner, long totalBytes, ReadWriteLock fileAccessLock) { - this.desc = desc; - in = new In(new File(desc.filenameFor(Component.PRIMARY_INDEX))); - partitioner = metadata.partitioner; - fileAccessLock = new ReentrantReadWriteLock(); + this.it = it; + this.partitioner = partitioner; + this.totalBytes = totalBytes; + this.fileAccessLock = fileAccessLock; } protected DecoratedKey computeNext() { - fileAccessLock.readLock().lock(); + if (fileAccessLock != null) + fileAccessLock.readLock().lock(); try { - if (in.isEOF()) - return endOfData(); - - keyPosition = in.getFilePointer(); - DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get())); - RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry - return key; + if (!initialized) + { + initialized = true; + return it.isExhausted() + ? endOfData() + : partitioner.decorateKey(it.key()); + } + else + { + return it.advance() + ? partitioner.decorateKey(it.key()) + : endOfData(); + } } catch (IOException e) { @@ -123,45 +68,44 @@ public class KeyIterator extends AbstractIterator implements Close } finally { - fileAccessLock.readLock().unlock(); + if (fileAccessLock != null) + fileAccessLock.readLock().unlock(); } } public void close() { - fileAccessLock.writeLock().lock(); + if (fileAccessLock != null) + fileAccessLock.writeLock().lock(); try { - in.close(); + it.close(); } finally { - fileAccessLock.writeLock().unlock(); + if (fileAccessLock != null) + fileAccessLock.writeLock().unlock(); } } public long getBytesRead() { - fileAccessLock.readLock().lock(); + if (fileAccessLock != null) + fileAccessLock.readLock().lock(); try { - return in.getFilePointer(); + return it.isExhausted() ? totalBytes : it.dataPosition(); } finally { - fileAccessLock.readLock().unlock(); + if (fileAccessLock != null) + fileAccessLock.readLock().unlock(); } } public long getTotalBytes() { - // length is final in the referenced object. - // no need to acquire the lock - return in.length(); + return totalBytes; } - public long getKeyPosition() - { - return keyPosition; - } } diff --git a/src/java/org/apache/cassandra/io/sstable/KeyReader.java b/src/java/org/apache/cassandra/io/sstable/KeyReader.java new file mode 100644 index 0000000000..88ee14512d --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/KeyReader.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Reads keys from an SSTable. + *

+ * It is specific to SSTable format how the keys are read but in general the assumption is that it will read all the + * keys in the order as they are placed in data file. + *

+ * After creating it, it should be at the first key. Unless the SSTable is empty, {@link #key()}, + * {@link #keyPositionForSecondaryIndex()} and {@link #dataPosition()} should return approriate values. If there is + * no data, {@link #isExhausted()} returns {@code true}. In order to move to the next key, {@link #advance()} should be + * called. It returns {@code true} if the reader moved to the next key; otherwise, there is no more data to read and + * the reader is exhausted. When the reader is exhausted, return values of {@link #key()}, + * {@link #keyPositionForSecondaryIndex()} and {@link #dataPosition()} are undefined. + */ +public interface KeyReader extends Closeable +{ + /** + * Current key + */ + ByteBuffer key(); + + /** + * Position in the component preferred for reading keys. This is specific to SSTable implementation + */ + long keyPositionForSecondaryIndex(); + + /** + * Position in the data file where the associated content resides + */ + long dataPosition(); + + /** + * Moves the iterator forward. Returns false if we reach EOF and there nothing more to read + */ + boolean advance() throws IOException; + + /** + * Returns true if we reach EOF + */ + boolean isExhausted(); + + /** + * Resets the iterator to the initial position + */ + void reset() throws IOException; + + /** + * Closes the iterator quietly + */ + @Override + void close(); +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java b/src/java/org/apache/cassandra/io/sstable/MetricsProviders.java similarity index 75% rename from src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java rename to src/java/org/apache/cassandra/io/sstable/MetricsProviders.java index 20f3f8f426..cc52b634c4 100644 --- a/src/java/org/apache/cassandra/db/filter/RowIndexEntryReadSizeTooLargeException.java +++ b/src/java/org/apache/cassandra/io/sstable/MetricsProviders.java @@ -16,14 +16,9 @@ * limitations under the License. */ -package org.apache.cassandra.db.filter; +package org.apache.cassandra.io.sstable; -import org.apache.cassandra.db.RejectException; - -public class RowIndexEntryReadSizeTooLargeException extends RejectException +public interface MetricsProviders { - public RowIndexEntryReadSizeTooLargeException(String message) - { - super(message); - } + Iterable> getGaugeProviders(); } diff --git a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java similarity index 97% rename from src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java rename to src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java index e9b15b1bbf..6c75d38c74 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable.format; +package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.ArrayList; @@ -30,8 +30,8 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTableMultiWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; diff --git a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java index 826b91d652..1cd780e076 100644 --- a/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/ReducingKeyIterator.java @@ -17,15 +17,18 @@ */ package org.apache.cassandra.io.sstable; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.CloseableIterator; import org.apache.cassandra.utils.IMergeIterator; import org.apache.cassandra.utils.MergeIterator; +import org.apache.cassandra.utils.Throwables; /** * Caller must acquire and release references to the sstables used here. @@ -39,7 +42,17 @@ public class ReducingKeyIterator implements CloseableIterator { iters = new ArrayList<>(sstables.size()); for (SSTableReader sstable : sstables) - iters.add(new KeyIterator(sstable.descriptor, sstable.metadata())); + { + try + { + iters.add(sstable.keyIterator()); + } + catch (IOException ex) + { + iters.forEach(FileUtils::closeQuietly); + throw new RuntimeException("Failed to create a key iterator for sstable " + sstable.getFilename()); + } + } } private void maybeInit() @@ -78,7 +91,17 @@ public class ReducingKeyIterator implements CloseableIterator public void close() { if (mi != null) + { mi.close(); + } + else + { + // if merging iterator was not initialized before this reducing iterator is closed, we need to close the + // underlying iterators manually + Throwable err = Throwables.close(null, iters); + if (err != null) + throw Throwables.unchecked(err); + } } public long getTotalBytes() @@ -121,4 +144,4 @@ public class ReducingKeyIterator implements CloseableIterator { throw new UnsupportedOperationException(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index 3c4f5cd0a9..25ac7d9477 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -17,16 +17,15 @@ */ package org.apache.cassandra.io.sstable; - -import java.io.FileNotFoundException; -import java.io.IOError; -import java.io.IOException; -import java.io.PrintWriter; +import java.lang.ref.WeakReference; import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; +import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -34,43 +33,35 @@ import com.google.common.base.Predicates; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.util.DiskOptimizationStrategy; -import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.TOCComponent; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.memory.HeapCloner; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.concurrent.SharedCloseable; -import static org.apache.cassandra.io.util.File.WriteMode.APPEND; +import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; /** - * This class is built on top of the SequenceFile. It stores - * data on disk in sorted fashion. However the sorting is upto - * the application. This class expects keys to be handed to it - * in sorted order. - * - * A separate index file is maintained as well, containing the - * SSTable keys and the offset into the SSTable at which they are found. - * Every 1/indexInterval key is read into memory when the SSTable is opened. - * - * Finally, a bloom filter file is also kept for the keys in each SSTable. + * This class represents an abstract sstable on disk whose keys and corresponding partitions are stored in + * a {@link SSTableFormat.Components#DATA} file in order as imposed by {@link DecoratedKey#comparator}. */ public abstract class SSTable { @@ -78,70 +69,101 @@ public abstract class SSTable public static final int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100; public static final int TOMBSTONE_HISTOGRAM_SPOOL_SIZE = 100000; - public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = Integer.valueOf(System.getProperty("cassandra.streaminghistogram.roundseconds", "60")); + public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.getInt(); public final Descriptor descriptor; protected final Set components; public final boolean compression; - public DecoratedKey first; - public DecoratedKey last; - - protected final DiskOptimizationStrategy optimizationStrategy; protected final TableMetadataRef metadata; - protected SSTable(Descriptor descriptor, Set components, TableMetadataRef metadata, DiskOptimizationStrategy optimizationStrategy) - { - // In almost all cases, metadata shouldn't be null, but allowing null allows to create a mostly functional SSTable without - // full schema definition. SSTableLoader use that ability - assert descriptor != null; - assert components != null; + public final ChunkCache chunkCache; + public final IOOptions ioOptions; - this.descriptor = descriptor; - Set dataComponents = new HashSet<>(components); - this.compression = dataComponents.contains(Component.COMPRESSION_INFO); - this.components = new CopyOnWriteArraySet<>(dataComponents); - this.metadata = metadata; - this.optimizationStrategy = Objects.requireNonNull(optimizationStrategy); + @Nullable + private final WeakReference owner; + + public SSTable(Builder builder, Owner owner) + { + this.owner = new WeakReference<>(owner); + checkNotNull(builder.descriptor); + checkNotNull(builder.getComponents()); + + this.descriptor = builder.descriptor; + this.ioOptions = builder.getIOOptions(); + this.components = new CopyOnWriteArraySet<>(builder.getComponents()); + this.compression = components.contains(Components.COMPRESSION_INFO); + this.metadata = builder.getTableMetadataRef(); + this.chunkCache = builder.getChunkCache(); } + public final Optional owner() + { + if (owner == null) + return Optional.empty(); + return Optional.ofNullable(owner.get()); + } + + public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set components) + { + components.stream() + .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .filter(c -> !c.equals(Components.DATA)) + .forEach(c -> tmpdesc.fileFor(c).move(newdesc.fileFor(c))); + + // do -Data last because -Data present should mean the sstable was completely renamed before crash + tmpdesc.fileFor(Components.DATA).move(newdesc.fileFor(Components.DATA)); + + // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader + components.stream() + .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .forEach(c -> tmpdesc.fileFor(c).tryMove(newdesc.fileFor(c))); + } + + public static void copy(Descriptor tmpdesc, Descriptor newdesc, Set components) + { + components.stream() + .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .filter(c -> !c.equals(Components.DATA)) + .forEach(c -> FileUtils.copyWithConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c))); + + // do -Data last because -Data present should mean the sstable was completely copied before crash + FileUtils.copyWithConfirm(tmpdesc.fileFor(Components.DATA), newdesc.fileFor(Components.DATA)); + + // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader + components.stream() + .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .forEach(c -> FileUtils.copyWithOutConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c))); + } + + public static void hardlink(Descriptor tmpdesc, Descriptor newdesc, Set components) + { + components.stream() + .filter(c -> !newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .filter(c -> !c.equals(Components.DATA)) + .forEach(c -> FileUtils.createHardLinkWithConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c))); + + // do -Data last because -Data present should mean the sstable was completely copied before crash + FileUtils.createHardLinkWithConfirm(tmpdesc.fileFor(Components.DATA), newdesc.fileFor(Components.DATA)); + + // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader + components.stream() + .filter(c -> newdesc.getFormat().generatedOnLoadComponents().contains(c)) + .forEach(c -> FileUtils.createHardLinkWithoutConfirm(tmpdesc.fileFor(c), newdesc.fileFor(c))); + } + + public abstract DecoratedKey getFirst(); + + public abstract DecoratedKey getLast(); + + public abstract AbstractBounds getBounds(); + @VisibleForTesting public Set getComponents() { return ImmutableSet.copyOf(components); } - /** - * We use a ReferenceQueue to manage deleting files that have been compacted - * and for which no more SSTable references exist. But this is not guaranteed - * to run for each such file because of the semantics of the JVM gc. So, - * we write a marker to `compactedFilename` when a file is compacted; - * if such a marker exists on startup, the file should be removed. - * - * This method will also remove SSTables that are marked as temporary. - * - * @return true if the file was deleted - */ - public static boolean delete(Descriptor desc, Set components) - { - logger.info("Deleting sstable: {}", desc); - // remove the DATA component first if it exists - if (components.contains(Component.DATA)) - FileUtils.deleteWithConfirm(desc.filenameFor(Component.DATA)); - for (Component component : components) - { - if (component.equals(Component.DATA) || component.equals(Component.SUMMARY)) - continue; - - FileUtils.deleteWithConfirm(desc.filenameFor(component)); - } - - if (components.contains(Component.SUMMARY)) - FileUtils.delete(desc.filenameFor(Component.SUMMARY)); - - return true; - } - public TableMetadata metadata() { return metadata.get(); @@ -157,25 +179,9 @@ public abstract class SSTable return getPartitioner().decorateKey(key); } - /** - * If the given @param key occupies only part of a larger buffer, allocate a new buffer that is only - * as large as necessary. - */ - public static DecoratedKey getMinimalKey(DecoratedKey key) - { - return key.getKey().position() > 0 || key.getKey().hasRemaining() || !key.getKey().hasArray() - ? new BufferDecoratedKey(key.getToken(), HeapCloner.instance.clone(key.getKey())) - : key; - } - public String getFilename() { - return descriptor.filenameFor(Component.DATA); - } - - public String getIndexFilename() - { - return descriptor.filenameFor(Component.PRIMARY_INDEX); + return descriptor.fileFor(Components.DATA).absolutePath(); } public String getColumnFamilyName() @@ -192,10 +198,31 @@ public abstract class SSTable { List ret = new ArrayList<>(components.size()); for (Component component : components) - ret.add(descriptor.filenameFor(component)); + ret.add(descriptor.fileFor(component).absolutePath()); return ret; } + /** + * The method sets fields for this sstable representation on the provided {@link Builder}. The method is intended + * to be called from the overloaded {@code unbuildTo} method in subclasses. + * + * @param builder the builder on which the fields should be set + * @param sharedCopy whether the {@link SharedCloseable} resources should be passed as shared copies or directly; + * note that the method will overwrite the fields representing {@link SharedCloseable} only if + * they are not set in the builder yet (the relevant fields in the builder are {@code null}). + * Although {@link SSTable} does not keep any references to resources, the parameters is added + * for the possible future fields and for consistency with the overloaded implementations in + * subclasses + * @return the same instance of builder as provided + */ + protected final > B unbuildTo(B builder, boolean sharedCopy) + { + return builder.setTableMetadataRef(metadata) + .setComponents(components) + .setChunkCache(chunkCache) + .setIOOptions(ioOptions); + } + /** * Parse a sstable filename into both a {@link Descriptor} and {@code Component} object. * @@ -208,7 +235,7 @@ public abstract class SSTable { try { - return Descriptor.fromFilenameWithComponent(file); + return Descriptor.fromFileWithComponent(file); } catch (Throwable e) { @@ -230,7 +257,7 @@ public abstract class SSTable { try { - return Descriptor.fromFilenameWithComponent(file, keyspace, table); + return Descriptor.fromFileWithComponent(file, keyspace, table); } catch (Throwable e) { @@ -248,11 +275,11 @@ public abstract class SSTable * @return the {@code Descriptor} corresponding to {@code file} if it corresponds to a valid and supported sstable * filename, {@code null} otherwise. */ - public static Descriptor tryDescriptorFromFilename(File file) + public static Descriptor tryDescriptorFromFile(File file) { try { - return Descriptor.fromFilename(file); + return Descriptor.fromFile(file); } catch (Throwable e) { @@ -260,140 +287,10 @@ public abstract class SSTable } } - /** - * Discovers existing components for the descriptor. Slow: only intended for use outside the critical path. - */ - public static Set componentsFor(final Descriptor desc) - { - try - { - try - { - return readTOC(desc); - } - catch (FileNotFoundException | NoSuchFileException e) - { - Set components = discoverComponentsFor(desc); - if (components.isEmpty()) - return components; // sstable doesn't exist yet - - if (!components.contains(Component.TOC)) - components.add(Component.TOC); - appendTOC(desc, components); - return components; - } - } - catch (IOException e) - { - throw new IOError(e); - } - } - - public static Set discoverComponentsFor(Descriptor desc) - { - Set knownTypes = Sets.difference(Component.TYPES, Collections.singleton(Component.Type.CUSTOM)); - Set components = Sets.newHashSetWithExpectedSize(knownTypes.size()); - for (Component.Type componentType : knownTypes) - { - Component component = new Component(componentType); - if (new File(desc.filenameFor(component)).exists()) - components.add(component); - } - return components; - } - - /** @return An estimate of the number of keys contained in the given index file. */ - public static long estimateRowsFromIndex(RandomAccessReader ifile, Descriptor descriptor) throws IOException - { - // collect sizes for the first 10000 keys, or first 10 mebibytes of data - final int SAMPLES_CAP = 10000, BYTES_CAP = (int)Math.min(10000000, ifile.length()); - int keys = 0; - while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) - { - ByteBufferUtil.skipShortLength(ifile); - RowIndexEntry.Serializer.skip(ifile, descriptor.version); - keys++; - } - assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0 : "Unexpected empty index file: " + ifile; - long estimatedRows = ifile.length() / (ifile.getFilePointer() / keys); - ifile.seek(0); - return estimatedRows; - } - @Override public String toString() { - return getClass().getSimpleName() + "(" + - "path='" + getFilename() + '\'' + - ')'; - } - - /** - * Reads the list of components from the TOC component. - * @return set of components found in the TOC - */ - protected static Set readTOC(Descriptor descriptor) throws IOException - { - return readTOC(descriptor, true); - } - - /** - * Reads the list of components from the TOC component. - * @param skipMissing, skip adding the component to the returned set if the corresponding file is missing. - * @return set of components found in the TOC - */ - protected static Set readTOC(Descriptor descriptor, boolean skipMissing) throws IOException - { - File tocFile = new File(descriptor.filenameFor(Component.TOC)); - List componentNames = Files.readAllLines(tocFile.toPath()); - Set components = Sets.newHashSetWithExpectedSize(componentNames.size()); - for (String componentName : componentNames) - { - Component component = new Component(Component.Type.fromRepresentation(componentName), componentName); - if (skipMissing && !new File(descriptor.filenameFor(component)).exists()) - logger.error("Missing component: {}", descriptor.filenameFor(component)); - else - components.add(component); - } - return components; - } - - /** - * Appends new component names to the TOC component. - */ - protected static void appendTOC(Descriptor descriptor, Collection components) - { - File tocFile = new File(descriptor.filenameFor(Component.TOC)); - try (FileOutputStreamPlus out = tocFile.newOutputStream(APPEND); - PrintWriter w = new PrintWriter(out)) - { - for (Component component : components) - w.println(component.name); - w.flush(); - out.sync(); - } - catch (IOException e) - { - throw new FSWriteError(e, tocFile); - } - } - - /** - * Registers new custom components. Used by custom compaction strategies. - * Adding a component for the second time is a no-op. - * Don't remove this - this method is a part of the public API, intended for use by custom compaction strategies. - * @param newComponents collection of components to be added - */ - public synchronized void addComponents(Collection newComponents) - { - Collection componentsToAdd = Collections2.filter(newComponents, Predicates.not(Predicates.in(components))); - appendTOC(descriptor, componentsToAdd); - components.addAll(componentsToAdd); - } - - public AbstractBounds getBounds() - { - return AbstractBounds.bounds(first.getToken(), true, last.getToken(), true); + return String.format("%s:%s(path='%s')", getClass().getSimpleName(), descriptor.formatType.name, getFilename()); } public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRepair, boolean isTransient) @@ -402,6 +299,118 @@ public abstract class SSTable "pendingRepair cannot be set on a repaired sstable"); Preconditions.checkArgument(!isTransient || (pendingRepair != NO_PENDING_REPAIR), "isTransient can only be true for sstables pending repair"); + } + /** + * Registers new custom components. Used by custom compaction strategies. + * Adding a component for the second time is a no-op. + * Don't remove this - this method is a part of the public API, intended for use by custom compaction strategies. + * + * @param newComponents collection of components to be added + */ + public synchronized void addComponents(Collection newComponents) + { + Collection componentsToAdd = Collections2.filter(newComponents, Predicates.not(Predicates.in(components))); + TOCComponent.appendTOC(descriptor, componentsToAdd); + components.addAll(componentsToAdd); + } + + public interface Owner + { + Double getCrcCheckChance(); + + OpOrder.Barrier newReadOrderingBarrier(); + + TableMetrics getMetrics(); + } + + /** + * A builder of this sstable representation. It should be extended for each implementation with the specific fields. + * + * @param type of the sstable representation to be build with this builder + * @param type of this builder + */ + public static class Builder> + { + public final Descriptor descriptor; + + private Set components; + private TableMetadataRef tableMetadataRef; + private ChunkCache chunkCache = ChunkCache.instance; + private IOOptions ioOptions = IOOptions.fromDatabaseDescriptor(); + + public Builder(Descriptor descriptor) + { + checkNotNull(descriptor); + this.descriptor = descriptor; + } + + public B setComponents(Collection components) + { + if (components != null) + { + components.forEach(c -> Preconditions.checkState(c.isValidFor(descriptor), "Invalid component type for sstable format " + descriptor.formatType.name)); + this.components = ImmutableSet.copyOf(components); + } + else + { + this.components = null; + } + return (B) this; + } + + public B addComponents(Collection components) + { + if (components == null || components.isEmpty()) + return (B) this; + + if (this.components == null) + return setComponents(components); + + return setComponents(Sets.union(this.components, ImmutableSet.copyOf(components))); + } + + public B setTableMetadataRef(TableMetadataRef ref) + { + this.tableMetadataRef = ref; + return (B) this; + } + + public B setChunkCache(ChunkCache chunkCache) + { + this.chunkCache = chunkCache; + return (B) this; + } + + public B setIOOptions(IOOptions ioOptions) + { + this.ioOptions = ioOptions; + return (B) this; + } + + public Descriptor getDescriptor() + { + return descriptor; + } + + public Set getComponents() + { + return components; + } + + public TableMetadataRef getTableMetadataRef() + { + return tableMetadataRef; + } + + public ChunkCache getChunkCache() + { + return chunkCache; + } + + public IOOptions getIOOptions() + { + return ioOptions; + } } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java similarity index 59% rename from src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java rename to src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java index 569925eb15..025d8cbaf4 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFlushObserver.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java @@ -15,10 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable.format; +package org.apache.cassandra.io.sstable; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.io.sstable.format.SSTableReader; /** * Observer for events in the lifecycle of writing out an sstable. @@ -34,10 +36,22 @@ public interface SSTableFlushObserver * Called when a new partition in being written to the sstable, * but before any cells are processed (see {@link #nextUnfilteredCluster(Unfiltered)}). * - * @param key The key being appended to SSTable. - * @param indexPosition The position of the key in the SSTable PRIMARY_INDEX file. + * @param key the key being appended to SSTable. + * @param keyPosition the position of the key in the SSTable data file + * @param KeyPositionForSASI SSTable format specific key position for storage attached indexes, it can be + * in data file or in some index file. It is the same position as returned by + * {@link KeyReader#keyPositionForSecondaryIndex()} for the same format, and the same + * position as expected by {@link SSTableReader#keyAtPositionFromSecondaryIndex(long)}. */ - void startPartition(DecoratedKey key, long indexPosition); + void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI); + + /** + * Called when a static row is being written to the sstable. If static columns are present in the table, it is called + * after {@link #startPartition(DecoratedKey, long, long)} and before any calls to {@link #nextUnfilteredCluster(Unfiltered)}. + * + * @param staticRow static row appended to the sstable, can be empty, may not be {@code null} + */ + void staticRow(Row staticRow); /** * Called after the unfiltered cluster is written to the sstable. @@ -52,4 +66,4 @@ public interface SSTableFlushObserver * Called when all data is written to the file and it's ready to be finished up. */ void complete(); -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java index f78200a31f..d0a4486b33 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableHeaderFix.java @@ -35,7 +35,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +53,11 @@ import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.UserType; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components.Types; import org.apache.cassandra.io.sstable.metadata.MetadataComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Schema; @@ -301,7 +303,7 @@ public abstract class SSTableHeaderFix .filter(p -> { try { - return Descriptor.fromFilenameWithComponent(new File(p)).right.type == Component.Type.DATA; + return Descriptor.fromFileWithComponent(new File(p)).right.type == Types.DATA; } catch (IllegalArgumentException t) { @@ -309,8 +311,8 @@ public abstract class SSTableHeaderFix return false; } }) - .map(Path::toString) - .map(Descriptor::fromFilename) + .map(File::new) + .map(file -> Descriptor.fromFileWithComponent(file, false).left) .forEach(descriptors::add); } @@ -342,8 +344,8 @@ public abstract class SSTableHeaderFix return; } - Set components = SSTable.discoverComponentsFor(desc); - if (components.stream().noneMatch(c -> c.type == Component.Type.STATS)) + Set components = desc.discoverComponents(); + if (components.stream().noneMatch(c -> c.type == Types.STATS)) { error("sstable %s has no -Statistics.db component.", desc); return; @@ -845,7 +847,7 @@ public abstract class SSTableHeaderFix private void writeNewMetadata(Descriptor desc, Map newMetadata) { - String file = desc.filenameFor(Component.STATS); + File file = desc.fileFor(Components.STATS); info.accept(String.format(" Writing new metadata file %s", file)); try { diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableId.java b/src/java/org/apache/cassandra/io/sstable/SSTableId.java index a3d95dd887..7a2235b8f9 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableId.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableId.java @@ -33,7 +33,7 @@ import org.apache.cassandra.io.util.File; * A new implementation must adhere to the following invariants: * - Must be locally sortable - that is, the comparison must reflect the comparison of generation times of identifiers * generated on the same node - * - String representation must *not* include the {@link Descriptor#FILENAME_SEPARATOR} character, see {@link Descriptor#fromFilenameWithComponent(File)} + * - String representation must *not* include the {@link Descriptor#FILENAME_SEPARATOR} character, see {@link Descriptor#fromFileWithComponent(File)} * - must be case-insensitive because the sstables can be stored on case-insensitive file system *

*/ @@ -50,7 +50,7 @@ public interface SSTableId * {@link Builder#fromString(String)} *

* Must not contain any {@link Descriptor#FILENAME_SEPARATOR} character as it is used in the Descriptor - * see {@link Descriptor#fromFilenameWithComponent(File)} + * see {@link Descriptor#fromFileWithComponent(File)} */ @Override String toString(); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java index 36ea5b13ec..b05b3c3dd5 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java @@ -17,17 +17,24 @@ */ package org.apache.cassandra.io.sstable; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; +import java.io.IOError; +import java.io.IOException; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.UnfilteredValidation; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; -import java.io.IOError; -import java.io.IOException; - import static org.apache.cassandra.utils.vint.VIntCoding.VIntOutOfRangeException; public class SSTableIdentityIterator implements Comparable, UnfilteredRowIterator @@ -72,13 +79,16 @@ public class SSTableIdentityIterator implements Comparable indexEntry, DecoratedKey key, boolean tombstoneOnly) + public static SSTableIdentityIterator create(SSTableReader sstable, FileDataInput dfile, long dataPosition, DecoratedKey key, boolean tombstoneOnly) { try { - dfile.seek(indexEntry.position); + dfile.seek(dataPosition); ByteBufferUtil.skipShortLength(dfile); // Skip partition key DeletionTime partitionLevelDeletion = DeletionTime.serializer.deserialize(dfile); + if (!partitionLevelDeletion.validate()) + UnfilteredValidation.handleInvalid(sstable.metadata(), key, sstable, "partitionLevelDeletion="+partitionLevelDeletion.toString()); + DeserializationHelper helper = new DeserializationHelper(sstable.metadata(), sstable.descriptor.version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL); SSTableSimpleIterator iterator = tombstoneOnly ? SSTableSimpleIterator.createTombstoneOnly(sstable.metadata(), dfile, sstable.header, helper, partitionLevelDeletion) diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java index dc87c99453..187d6ce2de 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java @@ -17,25 +17,43 @@ */ package org.apache.cassandra.io.sstable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; -import org.apache.cassandra.db.streaming.CassandraOutgoingFile; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.io.FSError; -import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.streaming.CassandraOutgoingFile; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.streaming.*; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.streaming.OutgoingStream; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.streaming.StreamEvent; +import org.apache.cassandra.streaming.StreamEventHandler; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; +import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.streaming.StreamingChannel; import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.Pair; - import org.apache.cassandra.utils.concurrent.Ref; import static org.apache.cassandra.streaming.StreamingChannel.Factory.Global.streamingFactory; @@ -55,7 +73,6 @@ public class SSTableLoader implements StreamEventHandler private final Set failedHosts = new HashSet<>(); private final List sstables = new ArrayList<>(); - private final Multimap streamingDetails = HashMultimap.create(); public SSTableLoader(File directory, Client client, OutputHandler outputHandler) { @@ -78,10 +95,11 @@ public class SSTableLoader implements StreamEventHandler } @SuppressWarnings("resource") - protected Collection openSSTables(final Map>> ranges) + private Multimap openSSTables(final Map>> ranges) { outputHandler.output("Opening sstables and calculating sections to stream"); + Multimap streamingDetails = HashMultimap.create(); LifecycleTransaction.getFiles(directory.toPath(), (file, type) -> { @@ -105,13 +123,16 @@ public class SSTableLoader implements StreamEventHandler } Descriptor desc = p == null ? null : p.left; - if (p == null || !p.right.equals(Component.DATA)) + if (p == null || !p.right.equals(Components.DATA)) return false; - if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) + for (Component c : desc.getFormat().primaryComponents()) { - outputHandler.output(String.format("Skipping file %s because index is missing", name)); - return false; + if (!desc.fileFor(c).exists()) + { + outputHandler.output(String.format("Skipping file %s because %s is missing", name, c.name)); + return false; + } } TableMetadataRef metadata = client.getTableMetadata(desc.cfname); @@ -121,22 +142,14 @@ public class SSTableLoader implements StreamEventHandler return false; } - Set components = new HashSet<>(); - components.add(Component.DATA); - components.add(Component.PRIMARY_INDEX); - if (new File(desc.filenameFor(Component.SUMMARY)).exists()) - components.add(Component.SUMMARY); - if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) - components.add(Component.COMPRESSION_INFO); - if (new File(desc.filenameFor(Component.STATS)).exists()) - components.add(Component.STATS); + Set components = desc.getComponents(desc.getFormat().primaryComponents(), desc.getFormat().uploadComponents()); try { // To conserve memory, open SSTableReaders without bloom filters and discard // the index summary after calculating the file sections to stream and the estimated // number of keys for each endpoint. See CASSANDRA-5555 for details. - SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata); + SSTableReader sstable = SSTableReader.openForBatch(null, desc, components, metadata); sstables.add(sstable); // calculate the sstable sections to stream as well as the estimated number of @@ -154,12 +167,12 @@ public class SSTableLoader implements StreamEventHandler long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges); Ref ref = sstable.ref(); - OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys); + CassandraOutgoingFile stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys); streamingDetails.put(endpoint, stream); } // to conserve heap space when bulk loading - sstable.releaseSummary(); + sstable.releaseInMemoryComponents(); } catch (FSError e) { @@ -170,7 +183,7 @@ public class SSTableLoader implements StreamEventHandler }, Directories.OnTxnErr.IGNORE); - return sstables; + return streamingDetails; } public StreamResultFuture stream() @@ -186,14 +199,14 @@ public class SSTableLoader implements StreamEventHandler StreamPlan plan = new StreamPlan(StreamOperation.BULK_LOAD, connectionsPerHost, false, null, PreviewKind.NONE).connectionFactory(client.getConnectionFactory()); Map>> endpointToRanges = client.getEndpointToRangesMap(); - openSSTables(endpointToRanges); - if (sstables.isEmpty()) + Multimap streamingDetails = openSSTables(endpointToRanges); + if (streamingDetails.isEmpty()) { // return empty result return plan.execute(); } - outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet())); + outputHandler.output(String.format("Streaming relevant part of %s to %s", names(streamingDetails.values()), endpointToRanges.keySet())); for (Map.Entry>> entry : endpointToRanges.entrySet()) { @@ -201,13 +214,8 @@ public class SSTableLoader implements StreamEventHandler if (toIgnore.contains(remote)) continue; - List streams = new LinkedList<>(); - // references are acquired when constructing the SSTableStreamingSections above - for (OutgoingStream stream : streamingDetails.get(remote)) - { - streams.add(stream); - } + List streams = new LinkedList<>(streamingDetails.get(remote)); plan.transferStreams(remote, streams); } @@ -229,10 +237,13 @@ public class SSTableLoader implements StreamEventHandler */ private void releaseReferences() { - for (SSTableReader sstable : sstables) + Iterator it = sstables.iterator(); + while (it.hasNext()) { + SSTableReader sstable = it.next(); sstable.selfRef().release(); assert sstable.selfRef().globalCount() == 0 : String.format("for sstable = %s, ref count = %d", sstable, sstable.selfRef().globalCount()); + it.remove(); } } @@ -246,12 +257,9 @@ public class SSTableLoader implements StreamEventHandler } } - private String names(Collection sstables) + private String names(Collection sstables) { - StringBuilder builder = new StringBuilder(); - for (SSTableReader sstable : sstables) - builder.append(sstable.descriptor.filenameFor(Component.DATA)).append(" "); - return builder.toString(); + return sstables.stream().map(CassandraOutgoingFile::getName).distinct().collect(Collectors.joining(" ")); } public Set getFailedHosts() diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java similarity index 62% rename from src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java rename to src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java index 6d384bfb72..e1d4155376 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReadsListener.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReadsListener.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable.format; +package org.apache.cassandra.io.sstable; -import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.tracing.Tracing; /** * Listener for receiving notifications associated with reading SSTables. @@ -29,10 +30,22 @@ public interface SSTableReadsListener */ enum SkippingReason { - BLOOM_FILTER, - MIN_MAX_KEYS, - PARTITION_INDEX_LOOKUP, - INDEX_ENTRY_NOT_FOUND; + BLOOM_FILTER("Bloom filter allows skipping sstable {}"), + MIN_MAX_KEYS("Check against min and max keys allows skipping sstable {}"), + PARTITION_INDEX_LOOKUP("Partition index lookup allows skipping sstable {}"), + INDEX_ENTRY_NOT_FOUND("Partition index lookup complete (bloom filter false positive) for sstable {}"); + + private final String message; + + SkippingReason(String message) + { + this.message = message; + } + + public void trace(Descriptor descriptor) + { + Tracing.trace(message, descriptor.id); + } } /** @@ -40,8 +53,20 @@ public interface SSTableReadsListener */ enum SelectionReason { - KEY_CACHE_HIT, - INDEX_ENTRY_FOUND; + KEY_CACHE_HIT("Key cache hit for sstable {}, size = {}"), + INDEX_ENTRY_FOUND("Partition index found for sstable {}, size = {}"); + + private final String message; + + SelectionReason(String message) + { + this.message = message; + } + + public void trace(Descriptor descriptor, AbstractRowIndexEntry entry) + { + Tracing.trace(message, descriptor.id, entry.columnsIndexCount()); + } } /** @@ -66,7 +91,7 @@ public interface SSTableReadsListener * @param indexEntry the index entry * @param reason the reason for which the SSTable has been selected */ - default void onSSTableSelected(SSTableReader sstable, RowIndexEntry indexEntry, SelectionReason reason) + default void onSSTableSelected(SSTableReader sstable, SelectionReason reason) { } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java index e394bbd24a..82ae4dc73c 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java @@ -17,22 +17,18 @@ */ package org.apache.cassandra.io.sstable; -import java.lang.ref.WeakReference; -import java.util.*; +import java.util.ArrayList; +import java.util.List; import com.google.common.annotations.VisibleForTesting; -import org.apache.cassandra.cache.InstrumentingCache; -import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.lifecycle.ILifecycleTransaction; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.utils.NativeLibrary; import org.apache.cassandra.utils.concurrent.Transactional; /** @@ -43,12 +39,12 @@ import org.apache.cassandra.utils.concurrent.Transactional; * for on-close (i.e. when all references expire) that drops the page cache prior to that key position * * hard-links are created for each partially written sstable so that readers opened against them continue to work past - * the rename of the temporary file, which is deleted once all readers against the hard-link have been closed. + * renaming of the temporary file, which is deleted once all readers against the hard-link have been closed. * If for any reason the writer is rolled over, we immediately rename and fully expose the completed file in the Tracker. * - * On abort we restore the original lower bounds to the existing readers and delete any temporary files we had in progress, - * but leave any hard-links in place for the readers we opened to cleanup when they're finished as we would had we finished - * successfully. + * On abort, we restore the original lower bounds to the existing readers and delete any temporary files we had in progress, + * but leave any hard-links in place for the readers we opened, and clean-up when the readers finish as we would do + * if we had finished successfully. */ public class SSTableRewriter extends Transactional.AbstractTransactional implements Transactional { @@ -69,7 +65,6 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme private final boolean eagerWriterMetaRelease; // true if the writer metadata should be released when switch is called private SSTableWriter writer; - private Map cachedKeys = new HashMap<>(); // for testing (TODO: remove when have byteman setup) private boolean throwEarly, throwLate; @@ -117,31 +112,16 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme return writer; } - public RowIndexEntry append(UnfilteredRowIterator partition) + public AbstractRowIndexEntry append(UnfilteredRowIterator partition) { - // we do this before appending to ensure we can resetAndTruncate() safely if the append fails + // we do this before appending to ensure we can resetAndTruncate() safely if appending fails DecoratedKey key = partition.partitionKey(); maybeReopenEarly(key); - RowIndexEntry index = writer.append(partition); - if (DatabaseDescriptor.shouldMigrateKeycacheOnCompaction()) - { - if (!transaction.isOffline() && index != null) - { - for (SSTableReader reader : transaction.originals()) - { - if (reader.getCachedPosition(key, false) != null) - { - cachedKeys.put(key, index); - break; - } - } - } - } - return index; + return writer.append(partition); } // attempts to append the row, if fails resets the writer position - public RowIndexEntry tryAppend(UnfilteredRowIterator partition) + public AbstractRowIndexEntry tryAppend(UnfilteredRowIterator partition) { writer.mark(); try @@ -163,20 +143,18 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme { for (SSTableReader reader : transaction.originals()) { - RowIndexEntry index = reader.getPosition(key, SSTableReader.Operator.GE); - NativeLibrary.trySkipCache(reader.getFilename(), 0, index == null ? 0 : index.position); + reader.trySkipFileCacheBefore(key); } } else { - SSTableReader reader = writer.setMaxDataAge(maxAge).openEarly(); - if (reader != null) - { + writer.setMaxDataAge(maxAge); + writer.openEarly(reader -> { transaction.update(reader, false); currentlyOpenedEarlyAt = writer.getFilePointer(); - moveStarts(reader, reader.last); + moveStarts(reader.last); transaction.checkpoint(); - } + }); } } } @@ -210,27 +188,13 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme * one, the old *instance* will have reference count == 0 and if we were to start a new compaction with that old * instance, we would get exceptions. * - * @param newReader the rewritten reader that replaces them for this region * @param lowerbound if !reset, must be non-null, and marks the exclusive lowerbound of the start for each sstable */ - private void moveStarts(SSTableReader newReader, DecoratedKey lowerbound) + private void moveStarts(DecoratedKey lowerbound) { if (transaction.isOffline() || preemptiveOpenInterval == Long.MAX_VALUE) return; - newReader.setupOnline(); - List invalidateKeys = null; - if (!cachedKeys.isEmpty()) - { - invalidateKeys = new ArrayList<>(cachedKeys.size()); - for (Map.Entry cacheKey : cachedKeys.entrySet()) - { - invalidateKeys.add(cacheKey.getKey()); - newReader.cacheKey(cacheKey.getKey(), cacheKey.getValue()); - } - } - - cachedKeys.clear(); for (SSTableReader sstable : transaction.originals()) { // we call getCurrentReplacement() to support multiple rewriters operating over the same source readers at once. @@ -241,49 +205,19 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme if (latest.first.compareTo(lowerbound) > 0) continue; - Runnable runOnClose = invalidateKeys != null ? new InvalidateKeys(latest, invalidateKeys) : null; if (lowerbound.compareTo(latest.last) >= 0) { if (!transaction.isObsolete(latest)) - { - if (runOnClose != null) - { - latest.runOnClose(runOnClose); - } transaction.obsolete(latest); - } continue; } - DecoratedKey newStart = latest.firstKeyBeyond(lowerbound); - assert newStart != null; - SSTableReader replacement = latest.cloneWithNewStart(newStart, runOnClose); - transaction.update(replacement, true); - } - } - - private static final class InvalidateKeys implements Runnable - { - final List cacheKeys = new ArrayList<>(); - final WeakReference> cacheRef; - - private InvalidateKeys(SSTableReader reader, Collection invalidate) - { - this.cacheRef = new WeakReference<>(reader.getKeyCache()); - if (cacheRef.get() != null) + if (!transaction.isObsolete(latest)) { - for (DecoratedKey key : invalidate) - cacheKeys.add(reader.getCacheKey(key)); - } - } - - public void run() - { - for (KeyCacheKey key : cacheKeys) - { - InstrumentingCache cache = cacheRef.get(); - if (cache != null) - cache.remove(key); + DecoratedKey newStart = latest.firstKeyBeyond(lowerbound); + assert newStart != null; + SSTableReader replacement = latest.cloneWithNewStart(newStart); + transaction.update(replacement, true); } } } @@ -291,7 +225,10 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme public void switchWriter(SSTableWriter newWriter) { if (newWriter != null) - writers.add(newWriter.setMaxDataAge(maxAge)); + { + newWriter.setMaxDataAge(maxAge); + writers.add(newWriter); + } if (eagerWriterMetaRelease && writer != null) writer.releaseMetadataOverhead(); @@ -310,12 +247,15 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme return; } + // Open fully completed sstables early. This is also required for the final sstable in a set (where newWriter + // is null) to permit the compilation of a canonical set of sstables (see View.select). if (preemptiveOpenInterval != Long.MAX_VALUE) { // we leave it as a tmp file, but we open it and add it to the Tracker - SSTableReader reader = writer.setMaxDataAge(maxAge).openFinalEarly(); + writer.setMaxDataAge(maxAge); + SSTableReader reader = writer.openFinalEarly(); transaction.update(reader, false); - moveStarts(reader, reader.last); + moveStarts(reader.last); transaction.checkpoint(); } @@ -370,7 +310,9 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme for (SSTableWriter writer : writers) { assert writer.getFilePointer() > 0; - writer.setRepairedAt(repairedAt).setOpenResult(true).prepareToCommit(); + writer.setRepairedAt(repairedAt); + writer.setOpenResult(true); + writer.prepareToCommit(); SSTableReader reader = writer.finished(); transaction.update(reader, false); preparedForCommit.add(reader); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java index 9ba2ab13ff..fb1fad7aa8 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java @@ -214,7 +214,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter if (b == SENTINEL) return; - try (SSTableTxnWriter writer = createWriter()) + try (SSTableTxnWriter writer = createWriter(null)) { for (Map.Entry entry : b.entrySet()) writer.append(entry.getValue().build().unfilteredIterator()); diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java index 317ac2a85d..744f35a41e 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java @@ -21,7 +21,8 @@ import java.io.IOException; import com.google.common.base.Throwables; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.TableMetadataRef; @@ -51,7 +52,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter private SSTableTxnWriter getOrCreateWriter() throws IOException { if (writer == null) - writer = createWriter(); + writer = createWriter(null); return writer; } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java index 83c5542a1b..1544157379 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java @@ -28,7 +28,6 @@ import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.index.Index; -import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; @@ -144,12 +143,13 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem boolean isTransient, int sstableLevel, SerializationHeader header, - Collection indexes) + Collection indexes, + SSTable.Owner owner) { // if the column family store does not exist, we create a new default SSTableMultiWriter to use: LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel); - SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn); + SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn, owner); return new SSTableTxnWriter(txn, writer); } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java similarity index 70% rename from src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java rename to src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java index f6403494d5..91e6490971 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java @@ -15,89 +15,81 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable.format.big; +package org.apache.cassandra.io.sstable; import java.io.EOFException; import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.Collection; -import java.util.EnumMap; +import java.util.HashMap; import java.util.Map; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.io.util.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.compress.BufferType; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.SequentialWriter; -import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.net.AsyncStreamingInputPlus; import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadataRef; import static java.lang.String.format; import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory; -public class BigTableZeroCopyWriter extends SSTable implements SSTableMultiWriter +public class SSTableZeroCopyWriter extends SSTable implements SSTableMultiWriter { - private static final Logger logger = LoggerFactory.getLogger(BigTableZeroCopyWriter.class); + private static final Logger logger = LoggerFactory.getLogger(SSTableZeroCopyWriter.class); - private final TableMetadataRef metadata; private volatile SSTableReader finalReader; private final Map componentWriters; - private static final SequentialWriterOption WRITER_OPTION = - SequentialWriterOption.newBuilder() - .trickleFsync(false) - .bufferSize(2 << 20) - .bufferType(BufferType.OFF_HEAP) - .build(); - - private static final ImmutableSet SUPPORTED_COMPONENTS = - ImmutableSet.of(Component.DATA, - Component.PRIMARY_INDEX, - Component.SUMMARY, - Component.STATS, - Component.COMPRESSION_INFO, - Component.FILTER, - Component.DIGEST, - Component.CRC); - - public BigTableZeroCopyWriter(Descriptor descriptor, - TableMetadataRef metadata, - LifecycleNewTracker lifecycleNewTracker, - final Collection components) + public SSTableZeroCopyWriter(Builder builder, + LifecycleNewTracker lifecycleNewTracker, + SSTable.Owner owner) { - super(descriptor, ImmutableSet.copyOf(components), metadata, DatabaseDescriptor.getDiskOptimizationStrategy()); + super(builder, owner); lifecycleNewTracker.trackNew(this); - this.metadata = metadata; - this.componentWriters = new EnumMap<>(Component.Type.class); + this.componentWriters = new HashMap<>(); - if (!SUPPORTED_COMPONENTS.containsAll(components)) + if (!descriptor.getFormat().streamingComponents().containsAll(components)) throw new AssertionError(format("Unsupported streaming component detected %s", - Sets.difference(ImmutableSet.copyOf(components), SUPPORTED_COMPONENTS))); + Sets.difference(ImmutableSet.copyOf(components), descriptor.getFormat().streamingComponents()))); for (Component c : components) componentWriters.put(c.type, makeWriter(descriptor, c)); } - private static SequentialWriter makeWriter(Descriptor descriptor, Component component) + @Override + public DecoratedKey getFirst() { - return new SequentialWriter(new File(descriptor.filenameFor(component)), WRITER_OPTION, false); + throw new UnsupportedOperationException(); + } + + @Override + public DecoratedKey getLast() + { + throw new UnsupportedOperationException(); + } + + @Override + public AbstractBounds getBounds() + { + throw new UnsupportedOperationException(); + } + + private SequentialWriter makeWriter(Descriptor descriptor, Component component) + { + return new SequentialWriter(descriptor.fileFor(component), ioOptions.writerOptions, false); } private void write(DataInputPlus in, long size, SequentialWriter out) throws FSWriteError @@ -119,14 +111,14 @@ public class BigTableZeroCopyWriter extends SSTable implements SSTableMultiWrite } catch (IOException e) { - throw new FSWriteError(e, out.getPath()); + throw new FSWriteError(e, out.getFile()); } } @Override public boolean append(UnfilteredRowIterator partition) { - throw new UnsupportedOperationException("Operation not supported by BigTableBlockWriter"); + throw new UnsupportedOperationException(); } @Override @@ -150,7 +142,7 @@ public class BigTableZeroCopyWriter extends SSTable implements SSTableMultiWrite public Collection finished() { if (finalReader == null) - finalReader = SSTableReader.open(descriptor, components, metadata); + finalReader = SSTableReader.open(owner().orElse(null), descriptor, components, metadata); return ImmutableList.of(finalReader); } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable_API.md b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md new file mode 100644 index 0000000000..aa637da6a1 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md @@ -0,0 +1,267 @@ +# SSTable API + +[CEP-17](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-17%3A+SSTable+format+API) +/ [CASSANDRA-17056](https://issues.apache.org/jira/browse/CASSANDRA-17056) + +## Configuration specification + +SSTable formats and options are specified under the `sstable_formats` key in the _cassandra.yaml_ configuration file. +By default, the first sstable format implementation specified is the default one, which means that newly produced +sstables will use that implementation. Other implementations are used to read the sstables. + +```yaml +sstable_formats: + - class_name: 〈class name of the default SSTableFormat implementation〉 + parameters: + id: 〈unique integer identifier of the implementation〉 + name: 〈unique string identifier of the implementation〉 + param1: 〈format specific parameter 1〉 + param2: 〈format specific parameter 2〉 + # ... + - class_name: 〈class name of additional SSTableFormat implementation〉 + parameters: + id: 〈unique integer identifier of the implementation〉 + name: 〈unique string identifier of the implementation〉 + param1: 〈format specific parameter 1〉 + param2: 〈format specific parameter 2〉 + # ... + # ... +``` + +Each sstable format definition includes the class name of the class implementing [`SSTableFormat`](format/SSTableFormat.java) +interface and a map of parameters. Two parameters are mandatory - _id_ and _name_, and they have to be unique across +all the defined formats. Additionally, they need to obey the following rules: +- _id_ must be an integer between 0 and 127 +- _name_ must contain only lowercase ASCII letters + +The _name_ parameter is the string put in the sstable file name. Therefore, it is the way Cassandra chooses the sstable +format implementation when loading an sstable. The _id_ parameter is used in saved cache files. Both parameters should +remain the same for the same format when upgrading. In particular, previous Cassandra versions have only +[_big_ format](format/big/BigFormat.java) implementation, thus when upgrading from the old version, the configuration +should keep _id_ set to _0_ and _name_ set to _big_ for that implementation (this is the default configuration), and any +new implementations should use different values for those parameters. + +The remaining parameters are optional and passed as a map to the `setup` method of a class implementing `SSTableFormat`. + +Default configuration: +```yaml +sstable_formats: + - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat + parameters: + id: 0 + name: big +``` + +Example configuration which uses `BtiFormat` as the default one keeping the right parameters for the `BigFormat` +implementation. +```yaml +sstable_formats: + - class_name: org.apache.cassandra.io.sstable.format.bti.BtiFormat + parameters: + id: 1 + name: bti + - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat + parameters: + id: 0 + name: big +``` + +## Components + +Each sstable consists of a set of components - required and optional. A component constitutes an identifier required +to obtain the exact file with an sstable descriptor. Components are grouped by type. A type may define either +a singleton component (for example, _stats_ component) or a non-singleton component (for example, _secondary index_ +component). + +A set of generic types of components that are thought of as common to all the sstable implementations is defined in the +[`SSTableFormat.Components`](format/SSTableFormat.java) class. They include singleton types like `DATA`, +`COMPRESSION_INFO`, `STATS`, `FILTER`, `DIGEST`, `CRC`, and `TOC`, which comes with predefined singleton component +instances, as well as non-singleton types like `SECONDARY_INDEX` and `CUSTOM`. + +Apart from the generic components, each sstable format implementation may describe its specific component types. +For example, the _big table_ format describes additionally `PRIMARY_INDEX` and `SUMMARY` singleton types and +the corresponding singleton components (see [`BigFormat.Components`](format/big/BigFormat.java)). + +Custom types can be created with one of the `Component.Type.create(name, repr, formatClass)`, +`Component.Type.createSingleton(name, repr, formatClass)` methods. Each created type is registered in a global types' +registry. Types registry is hierarchical which means that an sstable implementation may use types defined for its +format class and for all parent format classes (for example, the types defined for the `BigFormat` class extend the set +of types defined for the `SSTableFormat` interface). + +For example, types defined for `BigFormat`: + +```java +public static class Types extends SSTableFormat.Components.Types +{ + public static final Component.Type PRIMARY_INDEX = Component.Type.createSingleton("PRIMARY_INDEX", "Index.db", BigFormat.class); + public static final Component.Type SUMMARY = Component.Type.createSingleton("SUMMARY", "Summary.db", BigFormat.class); +} +``` + +Singleton components are immediately associated with the singleton types and retrieved with the `.getSingleton()` +method: + +```java +public static class Components extends AbstractSSTableFormat.Components +{ + public final static Component PRIMARY_INDEX = Types.PRIMARY_INDEX.getSingleton(); + public final static Component SUMMARY = Types.SUMMARY.getSingleton(); +} +``` + +Non-singleton components are created explicitly as follows: + +```java +Component idx1 = Types.SECONDARY_INDEX.createComponent("SI_idx1.db"); +``` + +## Implementation + +We strongly suggest the main format class to extend [`AbstractSSTableFormat`](format/AbstractSSTableFormat.java) because +it includes the expected implementation of a couple of methods that should not be reimplemented differently. + +### Initialization + +Cassandra either initializes the sstable format class as a singleton by calling its constructor or obtains the instance +by accessing a static field called `instance` in the class. As a part of the initialization, Cassandra calls the `setup` +method and provides the configuration parameters. Right after initialization, Cassandra calls the `allComponents` method +to confirm all the components defined for the format are initialized and usable. + +### Predefined sets of components + +SSTable format defines a couple of collections of components. You should declare those collections as constant and +immutable sets. + +### Reader + +#### Construction + +An sstable reader ([`SSTableReader`](format/SSTableReader.java)) is responsible for reading the data from an sstable. +It is created by a _simple builder_ ([`SSTableReader.Builder`](format/SSTableReader.java)) or a _loading +builder_ ([`SSTableReaderLoadingBuilder`](format/SSTableReaderLoadingBuilder.java)). The builders are supplied by +a _reader factory_ ([`SSTableFormat.SSTableReaderFactory`](format/SSTableFormat.java)). + +The constructor of a particular `SSTableReader` implementation should accept two parameters - one is the format-specific +_simple builder_ and the other one is an sstable owner (usually a `ColumnFamilyStore` instance, but it can be null +either). The constructor should be simple and not do anything but assign internal fields with values from the builder. + +A simple builder does not perform any logic except basic validation - it only stores the provided values the reader +constructor can access. A new reader implementation should include a public static simple builder inner class that +extends the `SSTableReader.Builder` generic reader builder (or `SSTableReaderWithFilter.Builder`, see [below](#filter)). + +In contrast to the simple builder, a loading builder can perform additional operations like more complex validation, +opening resources, loading caches, indexes, filters, etc. It internally creates a simple builder and eventually +instantiates a reader. + +#### General notes + +Note that if the builder carries some closeable resources to the reader, they should be returned by the `setupInstance` +method. + +You will find some `cloneXXX` methods to implement - remember to create a reader clone in a lambda passed to +the `runWithLock()` method. + +#### Unbuilding +It is convenient to implement the `unbuildTo` method, which takes a _simple builder_ and initializes it so that +the builder can produce the same reader. The method should also take the `sharedCopy` boolean argument denoting whether +it should copy the fields referencing closeable resources to the builder directly or as (shared) copies. The convention +also requires copying the resources only if they are unset in the builder (the field is null). The method should call +the `super.unbuildTo` method as a first step so that all the fields managed by the parent class are copied and in +the actual implementation only the fields specific to this format have to be assigned. + +For example, the implementation of that method in a reader for the _big table_ format is as follows: + +```java +protected final Builder unbuildTo(Builder builder, boolean sharedCopy) +{ + Builder b = super.unbuildTo(builder, sharedCopy); + if (builder.getIndexFile() == null) + b.setIndexFile(sharedCopy ? sharedCopyOrNull(ifile) : ifile); + if (builder.getIndexSummary() == null) + b.setIndexSummary(sharedCopy ? sharedCopyOrNull(indexSummary) : indexSummary); + + b.setKeyCache(keyCache); + + return b; +} +``` + +#### Filter + +If the sstable includes a _filter_, the reader class should extend +the [`SSTableReaderWithFilter`](format/SSTableReaderWithFilter.java) abstract reader (and its _simple builder_ should +extend the `SSTableReaderWithFilter.SSTableReaderWithFilterBuilder` builder). + +The reader with filter provides the `isPresentInFilter` method for extending implementation. It also implements other +filter-specific methods the system relies on if the implemented reader extends that class. + +Note that if the implemented reader extends the `SSTableReaderWithFilter` class, it should include the `FILTER` +component in the appropriate component sets. + +The reader with filter implementation comes with additional [metrics](filter/BloomFilterMetrics.java) - read more about custom +metrics support [here](#metrics). + +#### Index summary + +Some sstable format implementations, such as _big table_ format, may use _index summaries_. If a reader uses _index +summaries_ it should implement the [`IndexSummarySupport`](indexsummary/IndexSummarySupport.java) interface. + +The support for _index summaries_ comes with additional [metrics](indexsummary/IndexSummaryMetrics.java) - read more +about custom metrics support [here](#metrics). + +#### Key cache + +If an sstable format implementation uses row key cache, it should implement +the[`KeyCacheSupport`](keycache/KeyCacheSupport.java) interface. In particular, it should store a `KeyCache` instance +and return it with the `getKeyCache()` method. The interface has the default implementations of several methods +the system relies on if the reader implements the `KeyCacheSupport` interface. + +The interface comes with additional [metrics](keycache/KeyCacheMetrics.java) - read more about custom metrics support +[here](#metrics). + +#### Metrics + +A custom sstable format implementation may provide additional metrics on a table, keyspace, and global level. Those +metrics are accessible via JMX. The `SSTableFormat` implementation exposes the additional metrics by implementing the +`SSTableFormat.getFormatSpecificMetricsProviders` method. The method should return a singleton object implementing the +[`MetricsProviders`](MetricsProviders.java) interface. Currently, there is only support for custom gauges, but it can be +extended when needed. + +Each custom metric (gauge) is an implementation of the [GaugeProvider](GaugeProvider.java) abstract class. Although the +class expects the implementation to provide a gauge for each level of aggregation, there is a helper class - +[SimpleGaugeProvider](SimpleGaugeProvider.java) - which does that automatically with a supplied reduction lambda. There +is [`AbstractMetricsProviders`](AbstractMetricsProviders.java) class which is a partial implementation of the +`MetricsProviders` interface and leverages `SimpleGaugeProvider` in the offered methods. + +Example - additional metrics for sstables supporting index summaries (see +[`IndexSummaryMetrics`](indexsummary/IndexSummaryMetrics.java) for a full example): +```java +private final GaugeProvider indexSummaryOffHeapMemoryUsed = newGaugeProvider("IndexSummaryOffHeapMemoryUsed", + 0L, + r -> r.getIndexSummary().getOffHeapSize(), + Long::sum); +``` + +### Writer + +#### Construction + +An sstable writer ([`SSTableWriter`](format/SSTableWriter.java)) is responsible for writing the data to sstable files. +It is created by a _builder_ ([`SSTableWriter.Builder`](format/SSTableWriter.java)). The builder is supplied by +a _writer factory_ ([`SSTableFormat.SSTableWriterFactory`](format/SSTableFormat.java)). + +#### SortedTableWriter + +There are not many methods to be implemented in the writer. The most notable one is `append` which should write +the provided partition to disk. However, there is a generic default implementation +[`SortedTableWriter`](format/SortedTableWriter.java) which handles things like writing to the data file using +the default serializers, generic support for a partition index, notifications, metadata collection, and building +a filter. The writer triggers fine-grained events when data are added and those methods can be overridden in +the subclasses to apply specific behaviours (for example, `onPartitionStart`, `onRow`, `onStaticRow`, etc.). +Eventually it calls an abstract `createRowIndexEntry` method which should be implemented in the subclass. + +### Scrubber and verifier + +A custom sstable format should also come with its own verifier and scrubber implementing [`IVerifier`](IVerifier.java) +and [`IScrubber`](IScrubber.java) interfaces correspondingly. A generic partial implementation is provided in +[`SortedTableVerifier`](format/SortedTableVerifier.java) and [`SortedTableScrubber`](format/SortedTableScrubber.java). diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java b/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java new file mode 100644 index 0000000000..192909000c --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/SimpleGaugeProvider.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.Objects; +import java.util.function.Function; + +import com.google.common.collect.Iterables; + +import com.codahale.metrics.Gauge; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +public class SimpleGaugeProvider extends GaugeProvider +{ + private final Function mapper; + private final Function, T> combiner; + + public SimpleGaugeProvider(Function mapper, String name, Function, T> combiner) + { + super(name); + this.mapper = mapper; + this.combiner = combiner; + } + + @Override + public Gauge getTableGauge(ColumnFamilyStore cfs) + { + return () -> combine(cfs.getLiveSSTables()); + } + + @Override + public Gauge getKeyspaceGauge(Keyspace keyspace) + { + return () -> combine(getAllReaders(keyspace)); + } + + @Override + public Gauge getGlobalGauge() + { + return () -> combine(Iterables.concat(Iterables.transform(Keyspace.all(), SimpleGaugeProvider::getAllReaders))); + } + + private T combine(Iterable allReaders) + { + Iterable readers = Iterables.filter(Iterables.transform(allReaders, mapper::apply), Objects::nonNull); + return combiner.apply(readers); + } + + private static Iterable getAllReaders(Keyspace keyspace) + { + return Iterables.concat(Iterables.transform(keyspace.getColumnFamilyStores(), ColumnFamilyStore::getLiveSSTables)); + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java index ec28528a05..381c5eb33f 100644 --- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java @@ -20,7 +20,6 @@ package org.apache.cassandra.io.sstable; import java.util.Collection; import java.util.Collections; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.rows.UnfilteredRowIterator; @@ -45,13 +44,15 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter public boolean append(UnfilteredRowIterator partition) { - RowIndexEntry indexEntry = writer.append(partition); + AbstractRowIndexEntry indexEntry = writer.append(partition); return indexEntry != null; } public Collection finish(long repairedAt, long maxDataAge, boolean openResult) { - return Collections.singleton(writer.finish(repairedAt, maxDataAge, openResult)); + writer.setRepairedAt(repairedAt); + writer.setMaxDataAge(maxDataAge); + return Collections.singleton(writer.finish(openResult)); } public Collection finish(boolean openResult) @@ -116,9 +117,20 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter MetadataCollector metadataCollector, SerializationHeader header, Collection indexes, - LifecycleNewTracker lifecycleNewTracker) + LifecycleNewTracker lifecycleNewTracker, + SSTable.Owner owner) { - SSTableWriter writer = SSTableWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, indexes, lifecycleNewTracker); + SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor) + .setKeyCount(keyCount) + .setRepairedAt(repairedAt) + .setPendingRepair(pendingRepair) + .setTransientSSTable(isTransient) + .setTableMetadataRef(metadata) + .setMetadataCollector(metadataCollector) + .setSerializationHeader(header) + .addDefaultComponents() + .addFlushObserversForSecondaryIndexes(indexes, lifecycleNewTracker.opType()) + .build(lifecycleNewTracker, owner); return new SimpleSSTableMultiWriter(writer, lifecycleNewTracker); } } diff --git a/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java new file mode 100644 index 0000000000..ee5fc3c111 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterMetrics.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.filter; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cassandra.io.sstable.AbstractMetricsProviders; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; + +public class BloomFilterMetrics extends AbstractMetricsProviders +{ + public final static BloomFilterMetrics instance = new BloomFilterMetrics<>(); + + @Override + protected R map(SSTableReader r) + { + if (r instanceof SSTableReaderWithFilter) + return (R) r; + return null; + } + + /** + * Number of false positives in bloom filter + */ + public final GaugeProvider bloomFilterFalsePositives = newGaugeProvider("BloomFilterFalsePositives", + 0L, + r -> r.getFilterTracker().getFalsePositiveCount(), + Long::sum); + + /** + * Number of false positives in bloom filter from last read + */ + public final GaugeProvider recentBloomFilterFalsePositives = newGaugeProvider("RecentBloomFilterFalsePositives", + 0L, + r -> r.getFilterTracker().getRecentFalsePositiveCount(), + Long::sum); + + /** + * Disk space used by bloom filter + */ + public final GaugeProvider bloomFilterDiskSpaceUsed = newGaugeProvider("BloomFilterDiskSpaceUsed", + 0L, + SSTableReaderWithFilter::getFilterSerializedSize, + Long::sum); + + /** + * Off heap memory used by bloom filter + */ + public final GaugeProvider bloomFilterOffHeapMemoryUsed = newGaugeProvider("BloomFilterOffHeapMemoryUsed", + 0L, + SSTableReaderWithFilter::getFilterOffHeapSize, + Long::sum); + + /** + * False positive ratio of bloom filter + */ + public final GaugeProvider bloomFilterFalseRatio = newGaugeProvider("BloomFilterFalseRatio", readers -> { + long falsePositiveCount = 0L; + long truePositiveCount = 0L; + long trueNegativeCount = 0L; + for (SSTableReaderWithFilter sstable : readers) + { + falsePositiveCount += sstable.getFilterTracker().getFalsePositiveCount(); + truePositiveCount += sstable.getFilterTracker().getTruePositiveCount(); + trueNegativeCount += sstable.getFilterTracker().getTrueNegativeCount(); + } + if (falsePositiveCount == 0L && truePositiveCount == 0L) + return 0d; + return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); + }); + + /** + * False positive ratio of bloom filter from last read + */ + public final GaugeProvider recentBloomFilterFalseRatio = newGaugeProvider("RecentBloomFilterFalseRatio", readers -> { + long falsePositiveCount = 0L; + long truePositiveCount = 0L; + long trueNegativeCount = 0L; + for (SSTableReaderWithFilter sstable : readers) + { + falsePositiveCount += sstable.getFilterTracker().getRecentFalsePositiveCount(); + truePositiveCount += sstable.getFilterTracker().getRecentTruePositiveCount(); + trueNegativeCount += sstable.getFilterTracker().getRecentTrueNegativeCount(); + } + if (falsePositiveCount == 0L && truePositiveCount == 0L) + return 0d; + return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); + }); + + private final List> gaugeProviders = Arrays.asList(bloomFilterFalsePositives, + recentBloomFilterFalsePositives, + bloomFilterDiskSpaceUsed, + bloomFilterOffHeapMemoryUsed, + bloomFilterFalseRatio, + recentBloomFilterFalseRatio); + + public List> getGaugeProviders() + { + return gaugeProviders; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java similarity index 57% rename from src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java rename to src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java index 07523a001a..362926338f 100644 --- a/src/java/org/apache/cassandra/io/sstable/BloomFilterTracker.java +++ b/src/java/org/apache/cassandra/io/sstable/filter/BloomFilterTracker.java @@ -15,85 +15,68 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.filter; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; public class BloomFilterTracker { - private final AtomicLong falsePositiveCount = new AtomicLong(0); - private final AtomicLong truePositiveCount = new AtomicLong(0); - private final AtomicLong trueNegativeCount = new AtomicLong(0); - private long lastFalsePositiveCount = 0L; - private long lastTruePositiveCount = 0L; - private long lastTrueNegativeCount = 0L; + private final LongAdder falsePositiveCount = new LongAdder(); + private final LongAdder truePositiveCount = new LongAdder(); + private final LongAdder trueNegativeCount = new LongAdder(); + private final AtomicLong lastFalsePositiveCount = new AtomicLong(); + private final AtomicLong lastTruePositiveCount = new AtomicLong(); + private final AtomicLong lastTrueNegativeCount = new AtomicLong(); public void addFalsePositive() { - falsePositiveCount.incrementAndGet(); + falsePositiveCount.increment(); } public void addTruePositive() { - truePositiveCount.incrementAndGet(); + truePositiveCount.increment(); } public void addTrueNegative() { - trueNegativeCount.incrementAndGet(); + trueNegativeCount.increment(); } public long getFalsePositiveCount() { - return falsePositiveCount.get(); + return falsePositiveCount.sum(); } public long getRecentFalsePositiveCount() { long fpc = getFalsePositiveCount(); - try - { - return (fpc - lastFalsePositiveCount); - } - finally - { - lastFalsePositiveCount = fpc; - } + long last = lastFalsePositiveCount.getAndSet(fpc); + return fpc - last; } public long getTruePositiveCount() { - return truePositiveCount.get(); + return truePositiveCount.sum(); } public long getRecentTruePositiveCount() { long tpc = getTruePositiveCount(); - try - { - return (tpc - lastTruePositiveCount); - } - finally - { - lastTruePositiveCount = tpc; - } + long last = lastTruePositiveCount.getAndSet(tpc); + return tpc - last; } public long getTrueNegativeCount() { - return trueNegativeCount.get(); + return trueNegativeCount.sum(); } public long getRecentTrueNegativeCount() { long tnc = getTrueNegativeCount(); - try - { - return (tnc - lastTrueNegativeCount); - } - finally - { - lastTrueNegativeCount = tnc; - } + long last = lastTrueNegativeCount.getAndSet(tnc); + return tnc - last; } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java new file mode 100644 index 0000000000..5085cbc28b --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.util.Map; + +import com.google.common.base.Preconditions; + +public abstract class AbstractSSTableFormat implements SSTableFormat +{ + private String name; + private int ordinal; + + @Override + public final synchronized void setup(int ordinal, String name, Map options) + { + if (this.name != null) + throw new IllegalStateException("Attempted to set up already configured sstable format"); + this.name = name; + this.ordinal = ordinal; + setup(options); + } + + protected void setup(Map options) + { + } + + @Override + public final Type getType() + { + Preconditions.checkState(name != null, "Not configured"); + Type type = Type.getByOrdinal(ordinal); + assert type != null; + return type; + } + + @Override + public int ordinal() + { + return ordinal; + } + + @Override + public String name() + { + return name; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java new file mode 100644 index 0000000000..82c10bfb31 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/CompressionInfoComponent.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Set; + +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.util.File; + +public class CompressionInfoComponent +{ + public static CompressionMetadata maybeLoad(Descriptor descriptor, Set components) + { + if (components.contains(Components.COMPRESSION_INFO)) + return load(descriptor); + + return null; + } + + public static CompressionMetadata loadIfExists(Descriptor descriptor) + { + if (descriptor.fileFor(Components.COMPRESSION_INFO).exists()) + return load(descriptor); + + return null; + } + + public static CompressionMetadata load(Descriptor descriptor) + { + return CompressionMetadata.open(descriptor.fileFor(Components.COMPRESSION_INFO), + descriptor.fileFor(Components.DATA).length(), + descriptor.version.hasMaxCompressedLength()); + } + + /** + * Best-effort checking to verify the expected compression info component exists, according to the TOC file. + * The verification depends on the existence of TOC file. If absent, the verification is skipped. + * + * @param descriptor + * @param actualComponents, actual components listed from the file system. + * @throws CorruptSSTableException, if TOC expects compression info but not found from disk. + * @throws FSReadError, if unable to read from TOC file. + */ + public static void verifyCompressionInfoExistenceIfApplicable(Descriptor descriptor, Set actualComponents) throws CorruptSSTableException, FSReadError + { + File tocFile = descriptor.fileFor(Components.TOC); + if (tocFile.exists()) + { + try + { + Set expectedComponents = TOCComponent.loadTOC(descriptor, false); + if (expectedComponents.contains(Components.COMPRESSION_INFO) && !actualComponents.contains(Components.COMPRESSION_INFO)) + { + File compressionInfoFile = descriptor.fileFor(Components.COMPRESSION_INFO); + throw new CorruptSSTableException(new NoSuchFileException(compressionInfoFile.absolutePath()), compressionInfoFile); + } + } + catch (IOException e) + { + throw new FSReadError(e, tocFile); + } + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java new file mode 100644 index 0000000000..9367cb444d --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/DataComponent.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import org.apache.cassandra.config.Config.FlushCompression; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.io.compress.CompressedSequentialWriter; +import org.apache.cassandra.io.compress.ICompressor; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.ChecksummedSequentialWriter; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.schema.CompressionParams; +import org.apache.cassandra.schema.TableMetadata; + +public class DataComponent +{ + public static SequentialWriter buildWriter(Descriptor descriptor, + TableMetadata metadata, + SequentialWriterOption options, + MetadataCollector metadataCollector, + OperationType operationType, + FlushCompression flushCompression) + { + if (metadata.params.compression.isEnabled()) + { + final CompressionParams compressionParams = buildCompressionParams(metadata, operationType, flushCompression); + + return new CompressedSequentialWriter(descriptor.fileFor(Components.DATA), + descriptor.fileFor(Components.COMPRESSION_INFO), + descriptor.fileFor(Components.DIGEST), + options, + compressionParams, + metadataCollector); + } + else + { + return new ChecksummedSequentialWriter(descriptor.fileFor(Components.DATA), + descriptor.fileFor(Components.CRC), + descriptor.fileFor(Components.DIGEST), + options); + } + } + + /** + * Given an OpType, determine the correct Compression Parameters + * + * @return {@link CompressionParams} + */ + private static CompressionParams buildCompressionParams(TableMetadata metadata, OperationType operationType, FlushCompression flushCompression) + { + CompressionParams compressionParams = metadata.params.compression; + final ICompressor compressor = compressionParams.getSstableCompressor(); + + if (null != compressor && operationType == OperationType.FLUSH) + { + // When we are flushing out of the memtable throughput of the compressor is critical as flushes, + // especially of large tables, can queue up and potentially block writes. + // This optimization allows us to fall back to a faster compressor if a particular + // compression algorithm indicates we should. See CASSANDRA-15379 for more details. + switch (flushCompression) + { + // It is relatively easier to insert a Noop compressor than to disable compressed writing + // entirely as the "compression" member field is provided outside the scope of this class. + // It may make sense in the future to refactor the ownership of the compression flag so that + // We can bypass the CompressedSequentialWriter in this case entirely. + case none: + compressionParams = CompressionParams.NOOP; + break; + case fast: + if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION)) + { + // The default compressor is generally fast (LZ4 with 16KiB block size) + compressionParams = CompressionParams.DEFAULT; + break; + } + // else fall through + case table: + default: + break; + } + } + return compressionParams; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java new file mode 100644 index 0000000000..9f99d7dac0 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/FilterComponent.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.BloomFilterSerializer; +import org.apache.cassandra.utils.FilterFactory; +import org.apache.cassandra.utils.IFilter; + +public class FilterComponent +{ + private static final Logger logger = LoggerFactory.getLogger(FilterComponent.class); + + final static boolean rebuildFilterOnFPChanceChange = false; + final static double filterFPChanceTolerance = 0d; + + private FilterComponent() + { + } + + /** + * Load bloom filter from Filter.db file. + */ + public static IFilter load(Descriptor descriptor) throws IOException + { + File filterFile = descriptor.fileFor(Components.FILTER); + + if (!filterFile.exists()) + return null; + + if (filterFile.length() == 0) + return FilterFactory.AlwaysPresent; + + try (FileInputStreamPlus stream = descriptor.fileFor(Components.FILTER).newInputStream()) + { + return BloomFilterSerializer.forVersion(descriptor.version.hasOldBfFormat()).deserialize(stream); + } + catch (IOException ex) + { + throw new IOException("Failed to load Bloom filter for SSTable: " + descriptor.baseFile(), ex); + } + } + + public static void save(IFilter filter, Descriptor descriptor, boolean deleteOnFailure) throws IOException + { + File filterFile = descriptor.fileFor(Components.FILTER); + try (FileOutputStreamPlus stream = filterFile.newOutputStream(File.WriteMode.OVERWRITE)) + { + filter.serialize(stream, descriptor.version.hasOldBfFormat()); + stream.flush(); + stream.sync(); + } + catch (IOException ex) + { + if (deleteOnFailure) + descriptor.fileFor(Components.FILTER).deleteIfExists(); + throw new IOException("Failed to save Bloom filter for SSTable: " + descriptor.baseFile(), ex); + } + } + + /** + * Optionally loads a Bloom filter. + * If the filter is not needed (FP chance is neglectable), it returns {@link FilterFactory#AlwaysPresent}. + * If the filter is expected to be recreated for various reasons the method returns {@code null}. + * Otherwise, an attempt to load the filter is made and if it succeeds, the loaded filter is returned. + * If loading fails, the method returns {@code null}. + * + * @return {@link FilterFactory#AlwaysPresent}, loaded filter or {@code null} (which means that the filter should be rebuilt) + */ + public static IFilter maybeLoadBloomFilter(Descriptor descriptor, Set components, TableMetadata metadata, ValidationMetadata validationMetadata) + { + double currentFPChance = validationMetadata != null ? validationMetadata.bloomFilterFPChance : Double.NaN; + double desiredFPChance = metadata.params.bloomFilterFpChance; + + IFilter filter = null; + if (!shouldUseBloomFilter(desiredFPChance)) + { + if (logger.isTraceEnabled()) + logger.trace("Bloom filter for {} will not be loaded because fpChance={} is negligible", descriptor, desiredFPChance); + + return FilterFactory.AlwaysPresent; + } + else if (!components.contains(Components.FILTER) || Double.isNaN(currentFPChance)) + { + if (logger.isTraceEnabled()) + logger.trace("Bloom filter for {} will not be loaded because the filter component is missing or sstable lacks validation metadata", descriptor); + + return null; + } + else if (!isFPChanceDiffNegligible(desiredFPChance, currentFPChance) && rebuildFilterOnFPChanceChange) + { + if (logger.isTraceEnabled()) + logger.trace("Bloom filter for {} will not be loaded because fpChance has changed from {} to {} and the filter should be recreated", descriptor, currentFPChance, desiredFPChance); + + return null; + } + + try + { + filter = load(descriptor); + if (filter == null || !filter.isInformative()) + logger.info("Bloom filter for {} is missing or invalid", descriptor); + } + catch (IOException ex) + { + logger.info("Bloom filter for " + descriptor + " could not be deserialized", ex); + } + + return filter; + } + + static boolean shouldUseBloomFilter(double fpChance) + { + return !(Math.abs(1 - fpChance) <= filterFPChanceTolerance); + } + + static boolean isFPChanceDiffNegligible(double fpChance1, double fpChance2) + { + return Math.abs(fpChance1 - fpChance2) <= filterFPChanceTolerance; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java new file mode 100644 index 0000000000..45dfc62b2c --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/IndexComponent.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.IOOptions; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileHandle; + +public class IndexComponent +{ + public static FileHandle.Builder fileBuilder(File file, IOOptions ioOptions, ChunkCache chunkCache) + { + return new FileHandle.Builder(file).withChunkCache(chunkCache) + .mmapped(ioOptions.indexDiskAccessMode); + } + + public static FileHandle.Builder fileBuilder(Component component, SSTable ssTable) + { + return fileBuilder(ssTable.descriptor.fileFor(component), ssTable.ioOptions, ssTable.chunkCache); + } + + public static FileHandle.Builder fileBuilder(Component component, SSTable.Builder builder) + { + return fileBuilder(builder.descriptor.fileFor(component), builder.getIOOptions(), builder.getChunkCache()); + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java index 14f660258f..0f0e55efe0 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java @@ -17,61 +17,285 @@ */ package org.apache.cassandra.io.sstable.format; -import com.google.common.base.CharMatcher; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.function.Supplier; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.io.sstable.format.big.BigFormat; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.MetricsProviders; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Pair; /** * Provides the accessors to data on disk. */ -public interface SSTableFormat +public interface SSTableFormat { - static boolean enableSSTableDevelopmentTestMode = Boolean.getBoolean("cassandra.test.sstableformatdevelopment"); - + int ordinal(); + String name(); Version getLatestVersion(); Version getVersion(String version); - SSTableWriter.Factory getWriterFactory(); - SSTableReader.Factory getReaderFactory(); + SSTableWriterFactory getWriterFactory(); - RowIndexEntry.IndexSerializer getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header); + SSTableReaderFactory getReaderFactory(); - public static enum Type + /** + * All the components that the writter can produce when saving an sstable, as well as all the components + * that the reader can read. + */ + Set allComponents(); + + Set streamingComponents(); + + Set primaryComponents(); + + /** + * Returns components required by offline compaction tasks - like splitting sstables. + */ + Set batchComponents(); + + /** + * Returns the components which should be selected for upload by the sstable loader. + */ + Set uploadComponents(); + + /** + * Returns a set of the components that can be changed after an sstable was written. + */ + Set mutableComponents(); + + /** + * Returns a set of components that can be automatically generated when loading sstable and thus are not mandatory. + */ + Set generatedOnLoadComponents(); + + KeyCacheValueSerializer getKeyCacheValueSerializer(); + + /** + * Returns a new scrubber for an sstable. Note that the transaction must contain only one reader + * and the reader must match the provided cfs. + */ + IScrubber getScrubber(ColumnFamilyStore cfs, + LifecycleTransaction transaction, + OutputHandler outputHandler, + IScrubber.Options options); + + R cast(SSTableReader sstr); + + W cast(SSTableWriter sstw); + + MetricsProviders getFormatSpecificMetricsProviders(); + + void deleteOrphanedComponents(Descriptor descriptor, Set components); + + void setup(int id, String name, Map options); + + Type getType(); + + /** + * Deletes the existing components of the sstables represented by the provided descriptor. + * The method is also responsible for cleaning up the in-memory resources occupied by the stuff related to that + * sstables, such as row key cache entries. + */ + void delete(Descriptor descriptor); + + class Type { - //The original sstable format - BIG("big", BigFormat.instance); + private final static ImmutableList types; + private final static Type[] typesById; - public final SSTableFormat info; + static + { + Pair, Type[]> factories = readFactories(DatabaseDescriptor.getSSTableFormatFactories()); + types = ImmutableList.copyOf(factories.left); + typesById = factories.right; + } + + @VisibleForTesting + public static Pair, Type[]> readFactories(Map>> factories) + { + List typesList = new ArrayList<>(factories.size()); + factories.forEach((key, factory) -> { + SSTableFormat format = factory.get(); + typesList.add(new Type(format.ordinal(), format.name(), format)); + }); + List types = ImmutableList.copyOf(typesList); + int maxId = typesList.stream().mapToInt(t -> t.ordinal).max().getAsInt(); + Type[] typesById = new Type[maxId + 1]; + typesList.forEach(t -> typesById[t.ordinal] = t); + return Pair.create(types, typesById); + } + + public final int ordinal; + public final SSTableFormat info; public final String name; + private static Type currentType; + public static Type current() { - return BIG; + if (currentType != null) + return currentType; + + String name = CassandraRelevantProperties.SSTABLE_FORMAT_DEFAULT.getString(); + if (name == null) + return types.get(0); + + try + { + Type type = getByName(name); + currentType = type; + return type; + } + catch (RuntimeException ex) + { + throw new ConfigurationException("SSTable format " + name + " is not registered. Registered formats are: " + types); + } } - private Type(String name, SSTableFormat info) + private Type(int ordinal, String name, SSTableFormat info) { //Since format comes right after generation //we disallow formats with numeric names assert !CharMatcher.digit().matchesAllOf(name); - + this.ordinal = ordinal; this.name = name; this.info = info; } - public static Type validate(String name) + public static Type getByName(String name) { - for (Type valid : Type.values()) - { - if (valid.name.equalsIgnoreCase(name)) - return valid; - } + for (int i = 0; i < types.size(); i++) + if (types.get(i).name.equals(name)) + return types.get(i); + throw new NoSuchElementException(name); + } - throw new IllegalArgumentException("No Type constant " + name); + public static Type getByOrdinal(int ordinal) + { + if (ordinal < 0 || ordinal >= typesById.length || typesById[ordinal] == null) + throw new NoSuchElementException(String.valueOf(ordinal)); + return typesById[ordinal]; + } + + public static Iterable values() + { + return types; } } + + interface SSTableReaderFactory> + { + /** + * A simple builder which creates an instnace of {@link SSTableReader} with the provided parameters. + * It expects that all the required resources to be opened/loaded externally by the caller. + *

+ * The builder is expected to perform basic validation of the provided parameters. + */ + SSTableReader.Builder builder(Descriptor descriptor); + + /** + * A builder which opens/loads all the required resources upon execution of + * {@link SSTableReaderLoadingBuilder#build(SSTable.Owner, boolean, boolean)} and passed them to the created + * reader instance. If the creation of {@link SSTableReader} fails, no resources should be left opened. + *

+ * The builder is expected to perform basic validation of the provided parameters. + */ + SSTableReaderLoadingBuilder loadingBuilder(Descriptor descriptor, TableMetadataRef tableMetadataRef, Set components); + + /** + * Retrieves a key range for the given sstable at the lowest cost - that is, without opening all sstables files + * if possible. + */ + Pair readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException; + + Class getReaderClass(); + } + + interface SSTableWriterFactory> + { + /** + * Returns a new builder which can create instance of {@link SSTableWriter} with the provided parameters. + * Similarly to the loading builder, it should open the required resources when + * the {@link SSTableWriter.Builder#build(LifecycleNewTracker, SSTable.Owner)} method is called. + * It should not let the caller passing any closeable resources directly, that is, via setters. + * If building fails, all the opened resources should be released. + */ + B builder(Descriptor descriptor); + + /** + * Tries to estimate the size of all the sstable files from the provided parameters. + */ + long estimateSize(SSTableWriter.SSTableSizeParameters parameters); + } + + class Components + { + public static class Types + { + // the base data for an sstable: the remaining components can be regenerated + // based on the data component + public static final Component.Type DATA = Component.Type.createSingleton("DATA", "Data.db", null); + // file to hold information about uncompressed data length, chunk offsets etc. + public static final Component.Type COMPRESSION_INFO = Component.Type.createSingleton("COMPRESSION_INFO", "CompressionInfo.db", null); + // statistical metadata about the content of the sstable + public static final Component.Type STATS = Component.Type.createSingleton("STATS", "Statistics.db", null); + // serialized bloom filter for the row keys in the sstable + public static final Component.Type FILTER = Component.Type.createSingleton("FILTER", "Filter.db", null); + // holds CRC32 checksum of the data file + public static final Component.Type DIGEST = Component.Type.createSingleton("DIGEST", "Digest.crc32", null); + // holds the CRC32 for chunks in an uncompressed file. + public static final Component.Type CRC = Component.Type.createSingleton("CRC", "CRC.db", null); + // table of contents, stores the list of all components for the sstable + public static final Component.Type TOC = Component.Type.createSingleton("TOC", "TOC.txt", null); + // built-in secondary index (may exist multiple per sstable) + public static final Component.Type SECONDARY_INDEX = Component.Type.create("SECONDARY_INDEX", "SI_.*.db", null); + // custom component, used by e.g. custom compaction strategy + public static final Component.Type CUSTOM = Component.Type.create("CUSTOM", null, null); + } + + // singleton components for types that don't need ids + public final static Component DATA = Types.DATA.getSingleton(); + public final static Component COMPRESSION_INFO = Types.COMPRESSION_INFO.getSingleton(); + public final static Component STATS = Types.STATS.getSingleton(); + public final static Component FILTER = Types.FILTER.getSingleton(); + public final static Component DIGEST = Types.DIGEST.getSingleton(); + public final static Component CRC = Types.CRC.getSingleton(); + public final static Component TOC = Types.TOC.getSingleton(); + } + + interface KeyCacheValueSerializer + { + void skip(DataInputPlus input) throws IOException; + + T deserialize(R reader, DataInputPlus input) throws IOException; + + void serialize(T entry, DataOutputPlus output) throws IOException; + } + } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index fe8c537b61..609309c608 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -20,123 +20,133 @@ package org.apache.cassandra.io.sstable.format; import java.io.IOException; import java.lang.ref.WeakReference; -import java.nio.ByteBuffer; -import java.nio.file.NoSuchFileException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; - -import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.db.rows.UnfilteredSource; -import org.apache.cassandra.concurrent.ExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.clearspring.analytics.stream.cardinality.CardinalityMergeException; -import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; import com.clearspring.analytics.stream.cardinality.ICardinality; - -import org.apache.cassandra.cache.InstrumentingCache; -import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredSource; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.UnknownColumnException; import org.apache.cassandra.io.FSError; -import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.compress.CompressionMetadata; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.metadata.*; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.IVerifier; +import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableIdFactory; +import org.apache.cassandra.io.sstable.SSTableIdentityIterator; +import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.ChannelProxy; +import org.apache.cassandra.io.util.CheckedFunction; +import org.apache.cassandra.io.util.DataIntegrityMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.metrics.RestorableMeter; -import org.apache.cassandra.schema.CachingParams; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.service.CacheService; -import org.apache.cassandra.utils.*; -import org.apache.cassandra.utils.concurrent.*; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.NativeLibrary; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.OpOrder; +import org.apache.cassandra.utils.concurrent.Ref; +import org.apache.cassandra.utils.concurrent.SelfRefCounted; +import org.apache.cassandra.utils.concurrent.SharedCloseable; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR; import static org.apache.cassandra.utils.concurrent.BlockingQueues.newBlockingQueue; +import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull; /** - * An SSTableReader can be constructed in a number of places, but typically is either - * read from disk at startup, or constructed from a flushed memtable, or after compaction - * to replace some existing sstables. However once created, an sstablereader may also be modified. - * - * A reader's OpenReason describes its current stage in its lifecycle, as follows: - * - * - * 

 {@code
- * NORMAL
- * From:       None        => Reader has been read from disk, either at startup or from a flushed memtable
- *             EARLY       => Reader is the final result of a compaction
- *             MOVED_START => Reader WAS being compacted, but this failed and it has been restored to NORMAL status
- *
- * EARLY
- * From:       None        => Reader is a compaction replacement that is either incomplete and has been opened
- *                            to represent its partial result status, or has been finished but the compaction
- *                            it is a part of has not yet completed fully
- *             EARLY       => Same as from None, only it is not the first time it has been
- *
- * MOVED_START
- * From:       NORMAL      => Reader is being compacted. This compaction has not finished, but the compaction result
- *                            is either partially or fully opened, to either partially or fully replace this reader.
- *                            This reader's start key has been updated to represent this, so that reads only hit
- *                            one or the other reader.
- *
- * METADATA_CHANGE
- * From:       NORMAL      => Reader has seen low traffic and the amount of memory available for index summaries is
- *                            constrained, so its index summary has been downsampled.
- *         METADATA_CHANGE => Same
- * }
- * - * Note that in parallel to this, there are two different Descriptor types; TMPLINK and FINAL; the latter corresponds - * to NORMAL state readers and all readers that replace a NORMAL one. TMPLINK is used for EARLY state readers and - * no others. - * - * When a reader is being compacted, if the result is large its replacement may be opened as EARLY before compaction - * completes in order to present the result to consumers earlier. In this case the reader will itself be changed to - * a MOVED_START state, where its start no longer represents its on-disk minimum key. This is to permit reads to be - * directed to only one reader when the two represent the same data. The EARLY file can represent a compaction result - * that is either partially complete and still in-progress, or a complete and immutable sstable that is part of a larger - * macro compaction action that has not yet fully completed. - * - * Currently ALL compaction results at least briefly go through an EARLY open state prior to completion, regardless - * of if early opening is enabled. - * - * Since a reader can be created multiple times over the same shared underlying resources, and the exact resources - * it shares between each instance differ subtly, we track the lifetime of any underlying resource with its own - * reference count, which each instance takes a Ref to. Each instance then tracks references to itself, and once these - * all expire it releases its Refs to these underlying resources. - * - * There is some shared cleanup behaviour needed only once all sstablereaders in a certain stage of their lifecycle - * (i.e. EARLY or NORMAL opening), and some that must only occur once all readers of any kind over a single logical - * sstable have expired. These are managed by the TypeTidy and GlobalTidy classes at the bottom, and are effectively - * managed as another resource each instance tracks its own Ref instance to, to ensure all of these resources are - * cleaned up safely and can be debugged otherwise. - * + * An SSTableReader can be constructed in a number of places, but typically is either read from disk at startup, or + * constructed from a flushed memtable, or after compaction to replace some existing sstables. However once created, + * an sstablereader may also be modified. + *

+ * A reader's {@link OpenReason} describes its current stage in its lifecycle. Note that in parallel to this, there are + * two different Descriptor types; TMPLINK and FINAL; the latter corresponds to {@link OpenReason#NORMAL} state readers + * and all readers that replace a {@link OpenReason#NORMAL} one. TMPLINK is used for {@link OpenReason#EARLY} state + * readers and no others. + *

+ * When a reader is being compacted, if the result is large its replacement may be opened as {@link OpenReason#EARLY} + * before compaction completes in order to present the result to consumers earlier. In this case the reader will itself + * be changed to a {@link OpenReason#MOVED_START} state, where its start no longer represents its on-disk minimum key. + * This is to permit reads to be directed to only one reader when the two represent the same data. + * The {@link OpenReason#EARLY} file can represent a compaction result that is either partially complete and still + * in-progress, or a complete and immutable sstable that is part of a larger macro compaction action that has not yet + * fully completed. + *

+ * Currently ALL compaction results at least briefly go through an {@link OpenReason#EARLY} open state prior to completion, + * regardless of if early opening is enabled. + *

+ * Since a reader can be created multiple times over the same shared underlying resources, and the exact resources it + * shares between each instance differ subtly, we track the lifetime of any underlying resource with its own reference + * count, which each instance takes a {@link Ref} to. Each instance then tracks references to itself, and once these + * all expire it releases all its {@link Ref} to these underlying resources. + *

+ * There is some shared cleanup behaviour needed only once all readers in a certain stage of their lifecycle + * (i.e. {@link OpenReason#EARLY} or {@link OpenReason#NORMAL} opening), and some that must only occur once all readers + * of any kind over a single logical sstable have expired. These are managed by the {@link InstanceTidier} and + * {@link GlobalTidy} classes at the bottom, and are effectively managed as another resource each instance tracks its + * own {@link Ref} instance to, to ensure all of these resources are cleaned up safely and can be debugged otherwise. + *

* TODO: fill in details about Tracker and lifecycle interactions for tools, and for compaction strategies */ public abstract class SSTableReader extends SSTable implements UnfilteredSource, SelfRefCounted @@ -146,6 +156,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, private static final boolean TRACK_ACTIVITY = CassandraRelevantProperties.DISABLE_SSTABLE_ACTIVITY_TRACKING.getBoolean(); private static final ScheduledExecutorPlus syncExecutor = initSyncExecutor(); + private static ScheduledExecutorPlus initSyncExecutor() { if (DatabaseDescriptor.isClientOrToolInitialized()) @@ -157,81 +168,98 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, // Immediately remove readMeter sync task when cancelled. // TODO: should we set this by default on all scheduled executors? if (syncExecutor instanceof ScheduledThreadPoolExecutor) - ((ScheduledThreadPoolExecutor)syncExecutor).setRemoveOnCancelPolicy(true); + ((ScheduledThreadPoolExecutor) syncExecutor).setRemoveOnCancelPolicy(true); return syncExecutor; } + private static final RateLimiter meterSyncThrottle = RateLimiter.create(100.0); - public static final Comparator maxTimestampDescending = (o1, o2) -> Long.compare(o2.getMaxTimestamp(), o1.getMaxTimestamp()); - public static final Comparator maxTimestampAscending = (o1, o2) -> Long.compare(o1.getMaxTimestamp(), o2.getMaxTimestamp()); + public static final Comparator maxTimestampAscending = Comparator.comparingLong(SSTableReader::getMaxTimestamp); + public static final Comparator maxTimestampDescending = maxTimestampAscending.reversed(); // it's just an object, which we use regular Object equality on; we introduce a special class just for easy recognition - public static final class UniqueIdentifier {} + public static final class UniqueIdentifier + { + } + public final UniqueIdentifier instanceId = new UniqueIdentifier(); - public static final Comparator sstableComparator = (o1, o2) -> o1.first.compareTo(o2.first); + public static final Comparator sstableComparator = Comparator.comparing(o -> o.first); + public static final Ordering sstableOrdering = Ordering.from(sstableComparator); public static final Comparator idComparator = Comparator.comparing(t -> t.descriptor.id, SSTableIdFactory.COMPARATOR); public static final Comparator idReverseComparator = idComparator.reversed(); - public static final Ordering sstableOrdering = Ordering.from(sstableComparator); - - public static final Comparator sizeComparator = new Comparator() - { - public int compare(SSTableReader o1, SSTableReader o2) - { - return Longs.compare(o1.onDiskLength(), o2.onDiskLength()); - } - }; + public static final Comparator sizeComparator = (o1, o2) -> Longs.compare(o1.onDiskLength(), o2.onDiskLength()); /** * maxDataAge is a timestamp in local server time (e.g. Global.currentTimeMilli) which represents an upper bound * to the newest piece of data stored in the sstable. In other words, this sstable does not contain items created * later than maxDataAge. - * + *

* The field is not serialized to disk, so relying on it for more than what truncate does is not advised. - * + *

* When a new sstable is flushed, maxDataAge is set to the time of creation. * When a sstable is created from compaction, maxDataAge is set to max of all merged sstables. - * + *

* The age is in milliseconds since epoc and is local to this host. */ public final long maxDataAge; public enum OpenReason { + /** + *

    + *
  • From {@code None} - Reader has been read from disk, either at startup or from a flushed memtable
    • + *
  • From {@link #EARLY} - Reader is the final result of a compaction
    • + *
  • From {@link #MOVED_START} - Reader WAS being compacted, but this failed and it has been restored + * to {code NORMAL}status
    • + *
+ */ NORMAL, + + /** + *
    + *
  • From {@code None} - Reader is a compaction replacement that is either incomplete and has been opened + * to represent its partial result status, or has been finished but the compaction it is a part of has not + * yet completed fully
    • + *
  • From {@link #EARLY} - Same as from {@code None}, only it is not the first time it has been + *
+ */ EARLY, + + /** + * From: + *
    + *
  • From {@link #NORMAL} - Reader has seen low traffic and the amount of memory available for index summaries + * is constrained, so its index summary has been downsampled
    • + *
  • From {@link #METADATA_CHANGE} - Same + *
+ */ METADATA_CHANGE, + + /** + *
    + *
  • From {@link #NORMAL} - Reader is being compacted. This compaction has not finished, but the compaction + * result is either partially or fully opened, to either partially or fully replace this reader. This reader's + * start key has been updated to represent this, so that reads only hit one or the other reader.
    • + *
+ */ MOVED_START } public final OpenReason openReason; - public final UniqueIdentifier instanceId = new UniqueIdentifier(); - // indexfile and datafile: might be null before a call to load() - protected final FileHandle ifile; protected final FileHandle dfile; - protected final IFilter bf; - public final IndexSummary indexSummary; - - protected final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; - - protected InstrumentingCache keyCache; - - protected final BloomFilterTracker bloomFilterTracker = new BloomFilterTracker(); // technically isCompacted is not necessary since it should never be unreferenced unless it is also compacted, // but it seems like a good extra layer of protection against reference counting bugs to not delete data based on that alone - protected final AtomicBoolean isSuspect = new AtomicBoolean(false); + public final AtomicBoolean isSuspect = new AtomicBoolean(false); // not final since we need to be able to change level on a file. protected volatile StatsMetadata sstableMetadata; public final SerializationHeader header; - protected final AtomicLong keyCacheHit = new AtomicLong(0); - protected final AtomicLong keyCacheRequest = new AtomicLong(0); - private final InstanceTidier tidy; private final Ref selfRef; @@ -239,6 +267,10 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, private volatile double crcCheckChance; + public final DecoratedKey first; + public final DecoratedKey last; + public final AbstractBounds bounds; + /** * Calculate approximate key count. * If cardinality estimator is available on all given sstables, then this method use them to estimate @@ -264,7 +296,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, try { - CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION); + CompactionMetadata metadata = StatsComponent.load(sstable.descriptor).compactionMetadata(); // If we can't load the CompactionMetadata, we are forced to estimate the keys using the index // summary. (CASSANDRA-10676) if (metadata == null) @@ -305,233 +337,66 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return count; } - /** - * Estimates how much of the keys we would keep if the sstables were compacted together - */ - public static double estimateCompactionGain(Set overlapping) + public static SSTableReader open(SSTable.Owner owner, Descriptor descriptor) { - Set cardinalities = new HashSet<>(overlapping.size()); - for (SSTableReader sstable : overlapping) - { - try - { - ICardinality cardinality = ((CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION)).cardinalityEstimator; - if (cardinality != null) - cardinalities.add(cardinality); - else - logger.trace("Got a null cardinality estimator in: {}", sstable.getFilename()); - } - catch (IOException e) - { - logger.warn("Could not read up compaction metadata for {}", sstable, e); - } - } - long totalKeyCountBefore = 0; - for (ICardinality cardinality : cardinalities) - { - totalKeyCountBefore += cardinality.cardinality(); - } - if (totalKeyCountBefore == 0) - return 1; - - long totalKeyCountAfter = mergeCardinalities(cardinalities).cardinality(); - logger.trace("Estimated compaction gain: {}/{}={}", totalKeyCountAfter, totalKeyCountBefore, ((double)totalKeyCountAfter)/totalKeyCountBefore); - return ((double)totalKeyCountAfter)/totalKeyCountBefore; + return open(owner, descriptor, null); } - private static ICardinality mergeCardinalities(Collection cardinalities) + public static SSTableReader open(SSTable.Owner owner, Descriptor desc, TableMetadataRef metadata) { - ICardinality base = new HyperLogLogPlus(13, 25); // see MetadataCollector.cardinality - try - { - base = base.merge(cardinalities.toArray(new ICardinality[cardinalities.size()])); - } - catch (CardinalityMergeException e) - { - logger.warn("Could not merge cardinalities", e); - } - return base; + return open(owner, desc, null, metadata); } - public static SSTableReader open(Descriptor descriptor) + public static SSTableReader open(SSTable.Owner owner, Descriptor descriptor, Set components, TableMetadataRef metadata) { - TableMetadataRef metadata; - if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR)) - { - int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); - String indexName = descriptor.cfname.substring(i + 1); - metadata = Schema.instance.getIndexTableMetadataRef(descriptor.ksname, indexName); - if (metadata == null) - throw new AssertionError("Could not find index metadata for index cf " + i); - } - else - { - metadata = Schema.instance.getTableMetadataRef(descriptor.ksname, descriptor.cfname); - } - return open(descriptor, metadata); - } - - public static SSTableReader open(Descriptor desc, TableMetadataRef metadata) - { - return open(desc, componentsFor(desc), metadata); - } - - public static SSTableReader open(Descriptor descriptor, Set components, TableMetadataRef metadata) - { - return open(descriptor, components, metadata, true, false); + return open(owner, descriptor, components, metadata, true, false); } // use only for offline or "Standalone" operations public static SSTableReader openNoValidation(Descriptor descriptor, Set components, ColumnFamilyStore cfs) { - return open(descriptor, components, cfs.metadata, false, true); + return open(cfs, descriptor, components, cfs.metadata, false, true); } // use only for offline or "Standalone" operations - public static SSTableReader openNoValidation(Descriptor descriptor, TableMetadataRef metadata) + public static SSTableReader openNoValidation(SSTable.Owner owner, Descriptor descriptor, TableMetadataRef metadata) { - return open(descriptor, componentsFor(descriptor), metadata, false, true); + return open(owner, descriptor, null, metadata, false, true); } /** * Open SSTable reader to be used in batch mode(such as sstableloader). - * - * @param descriptor - * @param components - * @param metadata - * @return opened SSTableReader - * @throws IOException */ - public static SSTableReader openForBatch(Descriptor descriptor, Set components, TableMetadataRef metadata) + public static SSTableReader openForBatch(SSTable.Owner owner, Descriptor descriptor, Set components, TableMetadataRef metadata) { - // Minimum components without which we can't do anything - assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor; - assert components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor; - verifyCompressionInfoExistenceIfApplicable(descriptor, components); - - EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); - Map sstableMetadata; - try - { - sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); - } - catch (IOException e) - { - throw new CorruptSSTableException(e, descriptor.filenameFor(Component.STATS)); - } - - ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); - StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); - SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); - - // Check if sstable is created using same partitioner. - // Partitioner can be null, which indicates older version of sstable or no stats available. - // In that case, we skip the check. - String partitionerName = metadata.get().partitioner.getClass().getCanonicalName(); - if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner)) - { - logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.", - descriptor, validationMetadata.partitioner, partitionerName); - System.exit(1); - } - - try - { - return new SSTableReaderBuilder.ForBatch(descriptor, metadata, components, statsMetadata, header.toHeader(metadata.get())).build(); - } - catch (UnknownColumnException e) - { - throw new IllegalStateException(e); - } + return open(owner, descriptor, components, metadata, true, true); } /** * Open an SSTable for reading + * + * @param owner owning entity * @param descriptor SSTable to open * @param components Components included with this SSTable - * @param metadata for this SSTables CF - * @param validate Check SSTable for corruption (limited) - * @param isOffline Whether we are opening this SSTable "offline", for example from an external tool or not for inclusion in queries (validations) - * This stops regenerating BF + Summaries and also disables tracking of hotness for the SSTable. + * @param metadata for this SSTables CF + * @param validate Check SSTable for corruption (limited) + * @param isOffline Whether we are opening this SSTable "offline", for example from an external tool or not for inclusion in queries (validations) + * This stops regenerating BF + Summaries and also disables tracking of hotness for the SSTable. * @return {@link SSTableReader} - * @throws IOException */ - public static SSTableReader open(Descriptor descriptor, + public static SSTableReader open(Owner owner, + Descriptor descriptor, Set components, TableMetadataRef metadata, boolean validate, boolean isOffline) { - // Minimum components without which we can't do anything - assert components.contains(Component.DATA) : "Data component is missing for sstable " + descriptor; - assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor; + SSTableReaderLoadingBuilder builder = descriptor.getFormat().getReaderFactory().loadingBuilder(descriptor, metadata, components); - // For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content - assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor; - - verifyCompressionInfoExistenceIfApplicable(descriptor, components); - - EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); - - Map sstableMetadata; - try - { - sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types); - } - catch (Throwable t) - { - throw new CorruptSSTableException(t, descriptor.filenameFor(Component.STATS)); - } - ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); - StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS); - SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); - assert header != null; - - // Check if sstable is created using same partitioner. - // Partitioner can be null, which indicates older version of sstable or no stats available. - // In that case, we skip the check. - String partitionerName = metadata.get().partitioner.getClass().getCanonicalName(); - if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner)) - { - logger.error("Cannot open {}; partitioner {} does not match system partitioner {}. Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, so you will need to edit that to match your old partitioner if upgrading.", - descriptor, validationMetadata.partitioner, partitionerName); - System.exit(1); - } - - SSTableReader sstable; - try - { - sstable = new SSTableReaderBuilder.ForRead(descriptor, - metadata, - validationMetadata, - isOffline, - components, - statsMetadata, - header.toHeader(metadata.get())).build(); - } - catch (UnknownColumnException e) - { - throw new IllegalStateException(e); - } - - try - { - if (validate) - sstable.validate(); - - if (sstable.getKeyCache() != null) - logger.trace("key cache contains {}/{} keys", sstable.getKeyCache().size(), sstable.getKeyCache().getCapacity()); - - return sstable; - } - catch (Throwable t) - { - sstable.selfRef().release(); - throw new CorruptSSTableException(t, sstable.getFilename()); - } + return builder.build(owner, validate, !isOffline); } - public static Collection openAll(Set>> entries, + public static Collection openAll(SSTable.Owner owner, Set>> entries, final TableMetadataRef metadata) { final Collection sstables = newBlockingQueue(); @@ -541,29 +406,25 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, { for (final Map.Entry> entry : entries) { - Runnable runnable = new Runnable() - { - public void run() + Runnable runnable = () -> { + SSTableReader sstable; + try { - SSTableReader sstable; - try - { - sstable = open(entry.getKey(), entry.getValue(), metadata); - } - catch (CorruptSSTableException ex) - { - JVMStabilityInspector.inspectThrowable(ex); - logger.error("Corrupt sstable {}; skipping table", entry, ex); - return; - } - catch (FSError ex) - { - JVMStabilityInspector.inspectThrowable(ex); - logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex); - return; - } - sstables.add(sstable); + sstable = open(owner, entry.getKey(), entry.getValue(), metadata); } + catch (CorruptSSTableException ex) + { + JVMStabilityInspector.inspectThrowable(ex); + logger.error("Corrupt sstable {}; skipping table", entry, ex); + return; + } + catch (FSError ex) + { + JVMStabilityInspector.inspectThrowable(ex); + logger.error("Cannot read sstable {}; file system error, skipping table", entry, ex); + return; + } + sstables.add(sstable); }; executor.submit(runnable); } @@ -583,102 +444,62 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, } return sstables; - } - /** - * Open a RowIndexedReader which already has its state initialized (by SSTableWriter). - */ - public static SSTableReader internalOpen(Descriptor desc, - Set components, - TableMetadataRef metadata, - FileHandle ifile, - FileHandle dfile, - IndexSummary summary, - IFilter bf, - long maxDataAge, - StatsMetadata sstableMetadata, - OpenReason openReason, - SerializationHeader header) + protected SSTableReader(Builder builder, Owner owner) { - assert desc != null && ifile != null && dfile != null && summary != null && bf != null && sstableMetadata != null; + super(builder, owner); - return new SSTableReaderBuilder.ForWriter(desc, metadata, maxDataAge, components, sstableMetadata, openReason, header) - .bf(bf).ifile(ifile).dfile(dfile).summary(summary).build(); - } + this.sstableMetadata = builder.getStatsMetadata(); + this.header = builder.getSerializationHeader(); + this.dfile = builder.getDataFile(); + this.maxDataAge = builder.getMaxDataAge(); + this.openReason = builder.getOpenReason(); + this.first = builder.getFirst(); + this.last = builder.getLast(); + this.bounds = AbstractBounds.strictlyWrapsAround(first.getToken(), last.getToken()) + ? null // this will cause the validation to fail, but the reader is opened with no validation, + // e.g. for scrubbing, we should accept screwed bounds + : AbstractBounds.bounds(first.getToken(), true, last.getToken(), true); - /** - * Best-effort checking to verify the expected compression info component exists, according to the TOC file. - * The verification depends on the existence of TOC file. If absent, the verification is skipped. - * @param descriptor - * @param actualComponents, actual components listed from the file system. - * @throws CorruptSSTableException, if TOC expects compression info but not found from disk. - * @throws FSReadError, if unable to read from TOC file. - */ - public static void verifyCompressionInfoExistenceIfApplicable(Descriptor descriptor, - Set actualComponents) - throws CorruptSSTableException, FSReadError - { - File tocFile = new File(descriptor.filenameFor(Component.TOC)); - if (tocFile.exists()) - { - try - { - Set expectedComponents = readTOC(descriptor, false); - if (expectedComponents.contains(Component.COMPRESSION_INFO) && !actualComponents.contains(Component.COMPRESSION_INFO)) - { - String compressionInfoFileName = descriptor.filenameFor(Component.COMPRESSION_INFO); - throw new CorruptSSTableException(new NoSuchFileException(compressionInfoFileName), compressionInfoFileName); - } - } - catch (IOException e) - { - throw new FSReadError(e, tocFile); - } - } - } - - protected SSTableReader(SSTableReaderBuilder builder) - { - this(builder.descriptor, - builder.components, - builder.metadataRef, - builder.maxDataAge, - builder.statsMetadata, - builder.openReason, - builder.header, - builder.summary, - builder.dfile, - builder.ifile, - builder.bf); - } - - protected SSTableReader(final Descriptor desc, - Set components, - TableMetadataRef metadata, - long maxDataAge, - StatsMetadata sstableMetadata, - OpenReason openReason, - SerializationHeader header, - IndexSummary summary, - FileHandle dfile, - FileHandle ifile, - IFilter bf) - { - super(desc, components, metadata, DatabaseDescriptor.getDiskOptimizationStrategy()); - this.sstableMetadata = sstableMetadata; - this.header = header; - this.indexSummary = summary; - this.dfile = dfile; - this.ifile = ifile; - this.bf = bf; - this.maxDataAge = maxDataAge; - this.openReason = openReason; - this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), desc.version, header); - tidy = new InstanceTidier(descriptor, metadata.id); + tidy = new InstanceTidier(descriptor, owner); selfRef = new Ref<>(this, tidy); } + @Override + public DecoratedKey getFirst() + { + return first; + } + + @Override + public DecoratedKey getLast() + { + return last; + } + + @Override + public AbstractBounds getBounds() + { + return Objects.requireNonNull(bounds, "Bounds were not created because the sstable is out of order"); + } + + public DataIntegrityMetadata.ChecksumValidator maybeGetChecksumValidator() throws IOException + { + if (descriptor.fileFor(Components.CRC).exists()) + return new DataIntegrityMetadata.ChecksumValidator(descriptor.fileFor(Components.DATA), descriptor.fileFor(Components.CRC)); + else + return null; + } + + public DataIntegrityMetadata.FileDigestValidator maybeGetDigestValidator() throws IOException + { + if (descriptor.fileFor(Components.DIGEST).exists()) + return new DataIntegrityMetadata.FileDigestValidator(descriptor.fileFor(Components.DATA), descriptor.fileFor(Components.DIGEST)); + else + return null; + } + public static long getTotalBytes(Iterable sstables) { long sum = 0; @@ -713,60 +534,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, public void setupOnline() { - // under normal operation we can do this at any time, but SSTR is also used outside C* proper, - // e.g. by BulkLoader, which does not initialize the cache. As a kludge, we set up the cache - // here when we know we're being wired into the rest of the server infrastructure. - InstrumentingCache maybeKeyCache = CacheService.instance.keyCache; - if (maybeKeyCache.getCapacity() > 0) - keyCache = maybeKeyCache; - - final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata().id); - if (cfs != null) - setCrcCheckChance(cfs.getCrcCheckChance()); - } - - /** - * Save index summary to Summary.db file. - */ - public static void saveSummary(Descriptor descriptor, DecoratedKey first, DecoratedKey last, IndexSummary summary) - { - File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY)); - if (summariesFile.exists()) - FileUtils.deleteWithConfirm(summariesFile); - - try (DataOutputStreamPlus oStream = new FileOutputStreamPlus(summariesFile)) - { - IndexSummary.serializer.serialize(summary, oStream); - ByteBufferUtil.writeWithLength(first.getKey(), oStream); - ByteBufferUtil.writeWithLength(last.getKey(), oStream); - } - catch (IOException e) - { - logger.error("Cannot save SSTable Summary: ", e); - - // corrupted hence delete it and let it load it now. - if (summariesFile.exists()) - FileUtils.deleteWithConfirm(summariesFile); - } - } - - public static void saveBloomFilter(Descriptor descriptor, IFilter filter) - { - File filterFile = new File(descriptor.filenameFor(Component.FILTER)); - try (DataOutputStreamPlus stream = new FileOutputStreamPlus(filterFile)) - { - BloomFilterSerializer.serialize((BloomFilter) filter, stream); - stream.flush(); - } - catch (IOException e) - { - logger.trace("Cannot save SSTable bloomfilter: ", e); - - // corrupted hence delete it and let it load it now. - if (filterFile.exists()) - FileUtils.deleteWithConfirm(filterFile); - } - + owner().ifPresent(o -> setCrcCheckChance(o.getCrcCheckChance())); } /** @@ -774,7 +542,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, * * @param task to be guarded by sstable lock */ - public R runWithLock(CheckedFunction task) throws IOException + public R runWithLock(CheckedFunction task) throws E { synchronized (tidy.global) { @@ -799,317 +567,111 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, } } - // These runnables must NOT be an anonymous or non-static inner class, nor must it retain a reference chain to this reader + /** + * The runnable passed to this method must not be an anonymous or non-static inner class. It can be a lambda or a + * method reference provided that it does not retain a reference chain to this reader. + */ public void runOnClose(final Runnable runOnClose) { + if (runOnClose == null) + return; + synchronized (tidy.global) { final Runnable existing = tidy.runOnClose; - tidy.runOnClose = AndThen.get(existing, runOnClose); - } - } - - private static class AndThen implements Runnable - { - final Runnable runFirst; - final Runnable runSecond; - - private AndThen(Runnable runFirst, Runnable runSecond) - { - this.runFirst = runFirst; - this.runSecond = runSecond; - } - - public void run() - { - runFirst.run(); - runSecond.run(); - } - - static Runnable get(Runnable runFirst, Runnable runSecond) - { - if (runFirst == null) - return runSecond; - return new AndThen(runFirst, runSecond); + if (existing == null) + tidy.runOnClose = runOnClose; + else + tidy.runOnClose = () -> { + existing.run(); + runOnClose.run(); + }; } } /** - * Clone this reader with the provided start and open reason, and set the clone as replacement. + * The method sets fields specific to this {@link SSTableReader} and the parent {@link SSTable} on the provided + * {@link Builder}. The method is intended to be called from the overloaded {@code unbuildTo} method in subclasses. * - * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive - * opening of compaction results). - * @param reason the {@code OpenReason} for the replacement. - * - * @return the cloned reader. That reader is set as a replacement by the method. + * @param builder the builder on which the fields should be set + * @param sharedCopy whether the {@link SharedCloseable} resources should be passed as shared copies or directly; + * note that the method will overwrite the fields representing {@link SharedCloseable} only if + * they are not set in the builder yet (the relevant fields in the builder are {@code null}). + * @return the same instance of builder as provided */ - private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason) + protected final > B unbuildTo(B builder, boolean sharedCopy) { - return cloneAndReplace(newFirst, reason, indexSummary.sharedCopy()); + B b = super.unbuildTo(builder, sharedCopy); + if (builder.getDataFile() == null) + b.setDataFile(sharedCopy ? sharedCopyOrNull(dfile) : dfile); + + b.setStatsMetadata(sstableMetadata); + b.setSerializationHeader(header); + b.setMaxDataAge(maxDataAge); + b.setOpenReason(openReason); + b.setFirst(first); + b.setLast(last); + b.setSuspected(isSuspect.get()); + return b; } - /** - * Clone this reader with the new values and set the clone as replacement. - * - * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive - * opening of compaction results). - * @param reason the {@code OpenReason} for the replacement. - * @param newSummary the index summary for the replacement. - * - * @return the cloned reader. That reader is set as a replacement by the method. - */ - private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason, IndexSummary newSummary) - { - SSTableReader replacement = internalOpen(descriptor, - components, - metadata, - ifile != null ? ifile.sharedCopy() : null, - dfile.sharedCopy(), - newSummary, - bf.sharedCopy(), - maxDataAge, - sstableMetadata, - reason, - header); + public abstract SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart); - replacement.first = newFirst; - replacement.last = last; - replacement.isSuspect.set(isSuspect.get()); - return replacement; - } - - /** - * Clone this reader with the new values and set the clone as replacement. - * - * @param newBloomFilter for the replacement - * - * @return the cloned reader. That reader is set as a replacement by the method. - */ - @VisibleForTesting - public SSTableReader cloneAndReplace(IFilter newBloomFilter) - { - SSTableReader replacement = internalOpen(descriptor, - components, - metadata, - ifile.sharedCopy(), - dfile.sharedCopy(), - indexSummary, - newBloomFilter, - maxDataAge, - sstableMetadata, - openReason, - header); - - replacement.first = first; - replacement.last = last; - replacement.isSuspect.set(isSuspect.get()); - return replacement; - } - - public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart) - { - synchronized (tidy.global) - { - return cloneAndReplace(restoredStart, OpenReason.NORMAL); - } - } - - // runOnClose must NOT be an anonymous or non-static inner class, nor must it retain a reference chain to this reader - public SSTableReader cloneWithNewStart(DecoratedKey newStart, final Runnable runOnClose) - { - synchronized (tidy.global) - { - assert openReason != OpenReason.EARLY; - // TODO: merge with caller's firstKeyBeyond() work,to save time - if (newStart.compareTo(first) > 0) - { - final long dataStart = getPosition(newStart, Operator.EQ).position; - final long indexStart = getIndexScanPosition(newStart); - this.tidy.runOnClose = new DropPageCache(dfile, dataStart, ifile, indexStart, runOnClose); - } - - return cloneAndReplace(newStart, OpenReason.MOVED_START); - } - } - - private static class DropPageCache implements Runnable - { - final FileHandle dfile; - final long dfilePosition; - final FileHandle ifile; - final long ifilePosition; - final Runnable andThen; - - private DropPageCache(FileHandle dfile, long dfilePosition, FileHandle ifile, long ifilePosition, Runnable andThen) - { - this.dfile = dfile; - this.dfilePosition = dfilePosition; - this.ifile = ifile; - this.ifilePosition = ifilePosition; - this.andThen = andThen; - } - - public void run() - { - dfile.dropPageCache(dfilePosition); - - if (ifile != null) - ifile.dropPageCache(ifilePosition); - if (andThen != null) - andThen.run(); - } - } - - /** - * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will - * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have - * its DeletingTask removed, and have its periodic read-meter sync task cancelled. - * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader - * @return a new SSTableReader - * @throws IOException - */ - @SuppressWarnings("resource") - public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException - { - assert openReason != OpenReason.EARLY; - - int minIndexInterval = metadata().params.minIndexInterval; - int maxIndexInterval = metadata().params.maxIndexInterval; - double effectiveInterval = indexSummary.getEffectiveIndexInterval(); - - IndexSummary newSummary; - - // We have to rebuild the summary from the on-disk primary index in three cases: - // 1. The sampling level went up, so we need to read more entries off disk - // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary - // at full sampling (and consequently at any other sampling level) - // 3. The max_index_interval was lowered, forcing us to raise the sampling level - if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval) - { - newSummary = buildSummaryAtLevel(samplingLevel); - } - else if (samplingLevel < indexSummary.getSamplingLevel()) - { - // we can use the existing index summary to make a smaller one - newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, getPartitioner()); - } - else - { - throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " + - "no adjustments to min/max_index_interval"); - } - - // Always save the resampled index with lock to avoid racing with entire-sstable streaming - synchronized (tidy.global) - { - saveSummary(descriptor, first, last, newSummary); - return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary); - } - } - - private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException - { - // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. - RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))); - try - { - long indexSize = primaryIndex.length(); - try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata().params.minIndexInterval, newSamplingLevel)) - { - long indexPosition; - while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) - { - summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); - RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); - } - - return summaryBuilder.build(getPartitioner()); - } - } - finally - { - FileUtils.closeQuietly(primaryIndex); - } - } + public abstract SSTableReader cloneWithNewStart(DecoratedKey newStart); public RestorableMeter getReadMeter() { return readMeter; } - public int getIndexSummarySamplingLevel() + /** + * All the resources which should be released upon closing this sstable reader are registered with in + * {@link GlobalTidy}. This method lets close a provided resource explicitly any time and unregister it from + * {@link GlobalTidy} so that it is not tried to be released twice. + * + * @param closeable a resource to be closed + */ + protected void closeInternalComponent(AutoCloseable closeable) { - return indexSummary.getSamplingLevel(); + synchronized (tidy.global) + { + boolean removed = tidy.closeables.remove(closeable); + Preconditions.checkState(removed); + try + { + closeable.close(); + } + catch (Exception ex) + { + throw new RuntimeException("Failed to close " + closeable, ex); + } + } } - public long getIndexSummaryOffHeapSize() - { - return indexSummary.getOffHeapSize(); - } + /** + * This method is expected to close the components which occupy memory but are not needed when we just want to + * stream the components (for example, when SSTable is opened with SSTableLoader). The method should call + * {@link #closeInternalComponent(AutoCloseable)} for each such component. Leaving the implementation empty is + * valid given there are not such resources to release. + */ + public abstract void releaseInMemoryComponents(); - public int getMinIndexInterval() + /** + * Perform any validation needed for the reader upon creation before returning it from the {@link Builder}. + */ + public void validate() { - return indexSummary.getMinIndexInterval(); - } - - public double getEffectiveIndexInterval() - { - return indexSummary.getEffectiveIndexInterval(); - } - - public void releaseSummary() - { - tidy.releaseSummary(); - } - - private void validate() - { - if (this.first.compareTo(this.last) > 0) + if (this.first.compareTo(this.last) > 0 || bounds == null) { throw new CorruptSSTableException(new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)), getFilename()); } } /** - * Gets the position in the index file to start scanning to find the given key (at most indexInterval keys away, - * modulo downsampling of the index summary). Always returns a {@code value >= 0} - */ - public long getIndexScanPosition(PartitionPosition key) - { - if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0) - key = first; - - return getIndexScanPositionFromBinarySearchResult(indexSummary.binarySearch(key), indexSummary); - } - - @VisibleForTesting - public static long getIndexScanPositionFromBinarySearchResult(int binarySearchResult, IndexSummary referencedIndexSummary) - { - if (binarySearchResult == -1) - return 0; - else - return referencedIndexSummary.getPosition(getIndexSummaryIndexFromBinarySearchResult(binarySearchResult)); - } - - public static int getIndexSummaryIndexFromBinarySearchResult(int binarySearchResult) - { - if (binarySearchResult < 0) - { - // binary search gives us the first index _greater_ than the key searched for, - // i.e., its insertion position - int greaterThan = (binarySearchResult + 1) * -1; - if (greaterThan == 0) - return -1; - return greaterThan - 1; - } - else - { - return binarySearchResult; - } - } - - /** - * Returns the compression metadata for this sstable. + * Returns the compression metadata for this sstable. Note that the compression metdata is a resource and should not + * be closed by the caller. + * TODO do not return a closeable resource or return a shared copy + * * @throws IllegalStateException if the sstable is not compressed */ public CompressionMetadata getCompressionMetadata() @@ -1122,6 +684,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, /** * Returns the amount of memory in bytes used off heap by the compression meta-data. + * * @return the amount of memory in bytes used off heap by the compression meta-data */ public long getCompressionMetadataOffHeapSize() @@ -1132,166 +695,30 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return getCompressionMetadata().offHeapSize(); } - public IFilter getBloomFilter() - { - return bf; - } - - public long getBloomFilterSerializedSize() - { - return bf.serializedSize(); - } + /** + * Calculates an estimate of the number of keys in the sstable represented by this reader. + */ + public abstract long estimatedKeys(); /** - * Returns the amount of memory in bytes used off heap by the bloom filter. - * @return the amount of memory in bytes used off heap by the bloom filter + * Calculates an estimate of the number of keys for the given ranges in the sstable represented by this reader. */ - public long getBloomFilterOffHeapSize() - { - return bf.offHeapSize(); - } + public abstract long estimatedKeysForRanges(Collection> ranges); /** - * @return An estimate of the number of keys in this SSTable based on the index summary. + * Returns whether methods like {@link #estimatedKeys()} or {@link #estimatedKeysForRanges(Collection)} can return + * sensible estimations. */ - public long estimatedKeys() - { - return indexSummary.getEstimatedKeyCount(); - } + public abstract boolean isEstimationInformative(); /** - * @param ranges - * @return An estimate of the number of keys for given ranges in this SSTable. + * Returns sample keys for the provided token range. */ - public long estimatedKeysForRanges(Collection> ranges) - { - long sampleKeyCount = 0; - List sampleIndexes = getSampleIndexesForRanges(indexSummary, ranges); - for (IndexesBounds sampleIndexRange : sampleIndexes) - sampleKeyCount += (sampleIndexRange.upperPosition - sampleIndexRange.lowerPosition + 1); - - // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling - long estimatedKeys = sampleKeyCount * ((long) Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel(); - return Math.max(1, estimatedKeys); - } - - /** - * Returns the number of entries in the IndexSummary. At full sampling, this is approximately 1/INDEX_INTERVALth of - * the keys in this SSTable. - */ - public int getIndexSummarySize() - { - return indexSummary.size(); - } - - /** - * Returns the approximate number of entries the IndexSummary would contain if it were at full sampling. - */ - public int getMaxIndexSummarySize() - { - return indexSummary.getMaxNumberOfEntries(); - } - - /** - * Returns the key for the index summary entry at `index`. - */ - public byte[] getIndexSummaryKey(int index) - { - return indexSummary.getKey(index); - } - - private static List getSampleIndexesForRanges(IndexSummary summary, Collection> ranges) - { - // use the index to determine a minimal section for each range - List positions = new ArrayList<>(); - - for (Range range : Range.normalize(ranges)) - { - PartitionPosition leftPosition = range.left.maxKeyBound(); - PartitionPosition rightPosition = range.right.maxKeyBound(); - - int left = summary.binarySearch(leftPosition); - if (left < 0) - left = (left + 1) * -1; - else - // left range are start exclusive - left = left + 1; - if (left == summary.size()) - // left is past the end of the sampling - continue; - - int right = Range.isWrapAround(range.left, range.right) - ? summary.size() - 1 - : summary.binarySearch(rightPosition); - if (right < 0) - { - // range are end inclusive so we use the previous index from what binarySearch give us - // since that will be the last index we will return - right = (right + 1) * -1; - if (right == 0) - // Means the first key is already stricly greater that the right bound - continue; - right--; - } - - if (left > right) - // empty range - continue; - positions.add(new IndexesBounds(left, right)); - } - return positions; - } - - public Iterable getKeySamples(final Range range) - { - final List indexRanges = getSampleIndexesForRanges(indexSummary, Collections.singletonList(range)); - - if (indexRanges.isEmpty()) - return Collections.emptyList(); - - return new Iterable() - { - public Iterator iterator() - { - return new Iterator() - { - private Iterator rangeIter = indexRanges.iterator(); - private IndexesBounds current; - private int idx; - - public boolean hasNext() - { - if (current == null || idx > current.upperPosition) - { - if (rangeIter.hasNext()) - { - current = rangeIter.next(); - idx = current.lowerPosition; - return true; - } - return false; - } - - return true; - } - - public DecoratedKey next() - { - byte[] bytes = indexSummary.getKey(idx++); - return decorateKey(ByteBuffer.wrap(bytes)); - } - - public void remove() - { - throw new UnsupportedOperationException(); - } - }; - } - }; - } + public abstract Iterable getKeySamples(final Range range); /** * Determine the minimal set of sections that can be extracted from this SSTable to cover the given ranges. + * * @return A sorted list of (offset,end) pairs that cover the given ranges in the datafile for this SSTable. */ public List getPositionsForRanges(Collection> ranges) @@ -1309,10 +736,10 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, if (leftBound.compareTo(last) > 0 || rightBound.compareTo(first) < 0) continue; - long left = getPosition(leftBound, Operator.GT).position; + long left = getPosition(leftBound, Operator.GT); long right = (rightBound.compareTo(last) > 0) ? uncompressedLength() - : getPosition(rightBound, Operator.GT).position; + : getPosition(rightBound, Operator.GT); if (left == right) // empty range @@ -1324,146 +751,95 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return positions; } - public KeyCacheKey getCacheKey(DecoratedKey key) - { - return new KeyCacheKey(metadata(), descriptor, key.getKey()); - } - - public void cacheKey(DecoratedKey key, RowIndexEntry info) - { - CachingParams caching = metadata().params.caching; - - if (!caching.cacheKeys() || keyCache == null || keyCache.getCapacity() == 0) - return; - - KeyCacheKey cacheKey = new KeyCacheKey(metadata(), descriptor, key.getKey()); - logger.trace("Adding cache entry for {} -> {}", cacheKey, info); - keyCache.put(cacheKey, info); - } - - public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats) - { - if (isKeyCacheEnabled()) - return getCachedPosition(new KeyCacheKey(metadata(), descriptor, key.getKey()), updateStats); - return null; - } - - protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats) - { - if (isKeyCacheEnabled()) - { - if (updateStats) - { - RowIndexEntry cachedEntry = keyCache.get(unifiedKey); - keyCacheRequest.incrementAndGet(); - if (cachedEntry != null) - { - keyCacheHit.incrementAndGet(); - bloomFilterTracker.addTruePositive(); - } - return cachedEntry; - } - else - { - return keyCache.getInternal(unifiedKey); - } - } - return null; - } - - public boolean isKeyCacheEnabled() - { - return keyCache != null && metadata().params.caching.cacheKeys(); - } - /** * Retrieves the position while updating the key cache and the stats. + * * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to - * allow key selection by token bounds but only if op != * EQ - * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. + * allow key selection by token bounds but only if op != * EQ + * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. */ - public final RowIndexEntry getPosition(PartitionPosition key, Operator op) + public final long getPosition(PartitionPosition key, Operator op) { return getPosition(key, op, SSTableReadsListener.NOOP_LISTENER); } - /** - * Retrieves the position while updating the key cache and the stats. - * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to - * allow key selection by token bounds but only if op != * EQ - * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. - * @param listener the {@code SSTableReaderListener} that must handle the notifications. - */ - public final RowIndexEntry getPosition(PartitionPosition key, Operator op, SSTableReadsListener listener) + public final long getPosition(PartitionPosition key, Operator op, SSTableReadsListener listener) { return getPosition(key, op, true, false, listener); } - public final RowIndexEntry getPosition(PartitionPosition key, - Operator op, - boolean updateCacheAndStats) + public final long getPosition(PartitionPosition key, + Operator op, + boolean updateStats) { - return getPosition(key, op, updateCacheAndStats, false, SSTableReadsListener.NOOP_LISTENER); + return getPosition(key, op, updateStats, false, SSTableReadsListener.NOOP_LISTENER); } /** - * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to - * allow key selection by token bounds but only if op != * EQ - * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. - * @param updateCacheAndStats true if updating stats and cache - * @param listener a listener used to handle internal events + * Retrieve a position in data file according to the provided key and operator. + * + * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to + * allow key selection by token bounds but only if op != * EQ + * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. + * @param updateStats true if updating stats and cache + * @param listener a listener used to handle internal events + * * @return The index entry corresponding to the key, or null if the key is not present */ - protected abstract RowIndexEntry getPosition(PartitionPosition key, - Operator op, - boolean updateCacheAndStats, - boolean permitMatchPastLast, - SSTableReadsListener listener); + protected long getPosition(PartitionPosition key, + Operator op, + boolean updateStats, + boolean permitMatchPastLast, + SSTableReadsListener listener) + { + AbstractRowIndexEntry rie = getRowIndexEntry(key, op, updateStats, permitMatchPastLast, listener); + return rie != null ? rie.position : -1; + } - public abstract UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed); + /** + * Retrieve an index entry for the partition found according to the provided key and operator. + * + * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to + * allow key selection by token bounds but only if op != * EQ + * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. + * @param updateStats true if updating stats and cache + * @param listener a listener used to handle internal events + * + * @return The index entry corresponding to the key, or null if the key is not present + */ + @VisibleForTesting + protected abstract AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key, + Operator op, + boolean updateStats, + boolean permitMatchPastLast, + SSTableReadsListener listener); - public abstract UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly); + public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, long dataPosition, boolean tombstoneOnly) + { + return SSTableIdentityIterator.create(this, file, dataPosition, key, tombstoneOnly); + } + + /** + * Returns a {@link KeyReader} over all keys in the sstable. + */ + public abstract KeyReader keyReader() throws IOException; + + /** + * Returns a {@link KeyIterator} over all keys in the sstable. + */ + public KeyIterator keyIterator() throws IOException + { + return new KeyIterator(keyReader(), getPartitioner(), uncompressedLength(), new ReentrantReadWriteLock()); + } /** * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. */ - public DecoratedKey firstKeyBeyond(PartitionPosition token) - { - if (token.compareTo(first) < 0) - return first; - - long sampledPosition = getIndexScanPosition(token); - - if (ifile == null) - return null; - - String path = null; - try (FileDataInput in = ifile.createReader(sampledPosition)) - { - path = in.getPath(); - while (!in.isEOF()) - { - ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); - DecoratedKey indexDecoratedKey = decorateKey(indexKey); - if (indexDecoratedKey.compareTo(token) > 0) - return indexDecoratedKey; - - RowIndexEntry.Serializer.skip(in, descriptor.version); - } - } - catch (IOException e) - { - markSuspect(); - throw new CorruptSSTableException(e, path); - } - - return null; - } + public abstract DecoratedKey firstKeyBeyond(PartitionPosition token); /** - * @return The length in bytes of the data for this SSTable. For - * compressed files, this is not the same thing as the on disk size (see - * onDiskLength()) + * Returns the length in bytes of the (uncompressed) data for this SSTable. For compressed files, this is not + * the same thing as the on disk size (see {@link #onDiskLength()}). */ public long uncompressedLength() { @@ -1471,9 +847,8 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, } /** - * @return The length in bytes of the on disk size for this SSTable. For - * compressed files, this is not the same thing as the data length (see - * length()) + * The length in bytes of the on disk size for this SSTable. For compressed files, this is not the same thing + * as the data length (see {@link #uncompressedLength()}). */ public long onDiskLength() { @@ -1487,10 +862,8 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, } /** - * Set the value of CRC check chance. The argument supplied is obtained - * from the the property of the owning CFS. Called when either the SSTR - * is initialized, or the CFS's property is updated via JMX - * @param crcCheckChance + * Set the value of CRC check chance. The argument supplied is obtained from the property of the owning CFS. + * Called when either the SSTR is initialized, or the CFS's property is updated via JMX */ public void setCrcCheckChance(double crcCheckChance) { @@ -1500,11 +873,11 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, /** * Mark the sstable as obsolete, i.e., compacted into newer sstables. - * - * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere - * except for threads holding a reference. - * - * multiple times is usually buggy (see exceptions in Tracker.unmarkCompacting and removeOldSSTablesSize). + *

+ * When calling this function, the caller must ensure that the SSTableReader is not referenced anywhere except for + * threads holding a reference. + *

+ * Calling it multiple times is usually buggy. */ public void markObsolete(Runnable tidier) { @@ -1514,7 +887,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, synchronized (tidy.global) { assert !tidy.isReplaced; - assert tidy.global.obsoletion == null: this + " was already marked compacted"; + assert tidy.global.obsoletion == null : this + " was already marked compacted"; tidy.global.obsoletion = tidier; tidy.global.stopReadMeterPersistence(); @@ -1581,20 +954,27 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, */ public abstract ISSTableScanner getScanner(Iterator> rangeIterator); + /** + * Create a {@link FileDataInput} for the data file of the sstable represented by this reader. This method returns + * a newly opened resource which must be closed by the caller. + * + * @param position the data input will be opened and seek to this position + */ public FileDataInput getFileDataInput(long position) { return dfile.createReader(position); } /** - * Tests if the sstable contains data newer than the given age param (in localhost currentMilli time). - * This works in conjunction with maxDataAge which is an upper bound on the create of data in this sstable. - * @param age The age to compare the maxDataAre of this sstable. Measured in millisec since epoc on this host - * @return True iff this sstable contains data that's newer than the given age parameter. + * Tests if the sstable contains data newer than the given age param (in localhost currentMillis time). + * This works in conjunction with maxDataAge which is an upper bound on the data in the sstable represented + * by this reader. + * + * @return {@code true} iff this sstable contains data that's newer than the given timestamp */ - public boolean newSince(long age) + public boolean newSince(long timestampMillis) { - return maxDataAge > age; + return maxDataAge > timestampMillis; } public void createLinks(String snapshotDirectoryPath) @@ -1616,7 +996,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, { for (Component component : components) { - File sourceFile = new File(descriptor.filenameFor(component)); + File sourceFile = descriptor.fileFor(component); if (!sourceFile.exists()) continue; if (null != limiter) @@ -1631,25 +1011,17 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE; } - public DecoratedKey keyAt(long indexPosition) throws IOException - { - DecoratedKey key; - try (FileDataInput in = ifile.createReader(indexPosition)) - { - if (in.isEOF()) - return null; - - key = decorateKey(ByteBufferUtil.readWithShortLength(in)); - - // hint read path about key location if caching is enabled - // this saves index summary lookup and index file iteration which whould be pretty costly - // especially in presence of promoted column indexes - if (isKeyCacheEnabled()) - cacheKey(key, rowIndexEntrySerializer.deserialize(in)); - } - - return key; - } + /** + * Reads the key stored at the position saved in SASI. + *

+ * When SASI is created, it uses key locations retrieved from {@link KeyReader#keyPositionForSecondaryIndex()}. + * This method is to read the key stored at such position. It is up to the concrete SSTable format implementation + * what that position means and which file it refers. The only requirement is that it is consistent with what + * {@link KeyReader#keyPositionForSecondaryIndex()} returns. + * + * @return key if found, {@code null} otherwise + */ + public abstract DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException; public boolean isPendingRepair() { @@ -1694,55 +1066,29 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, final static class Equals extends Operator { - public int apply(int comparison) { return -comparison; } + public int apply(int comparison) + { + return -comparison; + } } final static class GreaterThanOrEqualTo extends Operator { - public int apply(int comparison) { return comparison >= 0 ? 0 : 1; } + public int apply(int comparison) + { + return comparison >= 0 ? 0 : 1; + } } final static class GreaterThan extends Operator { - public int apply(int comparison) { return comparison > 0 ? 0 : 1; } + public int apply(int comparison) + { + return comparison > 0 ? 0 : 1; + } } } - public long getBloomFilterFalsePositiveCount() - { - return bloomFilterTracker.getFalsePositiveCount(); - } - - public long getRecentBloomFilterFalsePositiveCount() - { - return bloomFilterTracker.getRecentFalsePositiveCount(); - } - - public long getBloomFilterTruePositiveCount() - { - return bloomFilterTracker.getTruePositiveCount(); - } - - public long getRecentBloomFilterTruePositiveCount() - { - return bloomFilterTracker.getRecentTruePositiveCount(); - } - - public long getBloomFilterTrueNegativeCount() - { - return bloomFilterTracker.getTrueNegativeCount(); - } - - public long getRecentBloomFilterTrueNegativeCount() - { - return bloomFilterTracker.getRecentTrueNegativeCount(); - } - - public InstrumentingCache getKeyCache() - { - return keyCache; - } - public EstimatedHistogram getEstimatedPartitionSize() { return sstableMetadata.estimatedPartitionSize; @@ -1825,8 +1171,8 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, public int getAvgColumnSetPerRow() { return sstableMetadata.totalRows < 0 - ? -1 - : (sstableMetadata.totalRows == 0 ? 0 : (int)(sstableMetadata.totalColumnsSet / sstableMetadata.totalRows)); + ? -1 + : (sstableMetadata.totalRows == 0 ? 0 : (int) (sstableMetadata.totalColumnsSet / sstableMetadata.totalRows)); } public int getSSTableLevel() @@ -1860,16 +1206,16 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, /** * Reloads the sstable metadata from disk. - * + *

* Called after level is changed on sstable, for example if the sstable is dropped to L0 - * + *

* Might be possible to remove in future versions * * @throws IOException */ public void reloadSSTableMetadata() throws IOException { - this.sstableMetadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS); + this.sstableMetadata = StatsComponent.load(descriptor).statsMetadata(); } public StatsMetadata getSSTableMetadata() @@ -1888,11 +1234,10 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return dfile.createReader(); } - public RandomAccessReader openIndexReader() + public void trySkipFileCacheBefore(DecoratedKey key) { - if (ifile != null) - return ifile.createReader(); - return null; + long position = getPosition(key, SSTableReader.Operator.GE); + NativeLibrary.trySkipCache(descriptor.fileFor(Components.DATA).absolutePath(), 0, position < 0 ? 0 : position); } public ChannelProxy getDataChannel() @@ -1900,39 +1245,12 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return dfile.channel; } - public ChannelProxy getIndexChannel() - { - return ifile.channel; - } - - public FileHandle getIndexFile() - { - return ifile; - } - /** - * @param component component to get timestamp. - * @return last modified time for given component. 0 if given component does not exist or IO error occurs. + * @return last modified time for data component. 0 if given component does not exist or IO error occurs. */ - public long getCreationTimeFor(Component component) + public long getDataCreationTime() { - return new File(descriptor.filenameFor(component)).lastModified(); - } - - /** - * @return Number of key cache hit - */ - public long getKeyCacheHit() - { - return keyCacheHit.get(); - } - - /** - * @return Number of key cache request - */ - public long getKeyCacheRequest() - { - return keyCacheRequest.get(); + return descriptor.fileFor(Components.DATA).lastModified(); } /** @@ -1967,9 +1285,16 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return selfRef.ref(); } - void setup(boolean trackHotness) + protected List setupInstance(boolean trackHotness) { - tidy.setup(this, TRACK_ACTIVITY && trackHotness); + return Collections.singletonList(dfile); + } + + public void setup(boolean trackHotness) + { + assert tidy.closeables == null; + trackHotness &= TRACK_ACTIVITY; + tidy.setup(this, trackHotness, setupInstance(trackHotness)); this.readMeter = tidy.global.readMeter; } @@ -1983,39 +1308,35 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, { identities.add(this); identities.add(tidy.globalRef); - dfile.addTo(identities); - ifile.addTo(identities); - bf.addTo(identities); - indexSummary.addTo(identities); - - } - - public boolean maybePresent(DecoratedKey key) - { - // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing), - // we check index file instead. - return bf instanceof AlwaysPresentFilter && getPosition(key, Operator.EQ, false) != null || bf.isPresent(key); + tidy.closeables.forEach(c -> { + if (c instanceof SharedCloseable) + ((SharedCloseable) c).addTo(identities); + }); } + /** + * The method verifies whether the sstable may contain the provided key. The method does approximation using + * Bloom filter if it is present and if it is not, performs accurate check in the index. + */ + public abstract boolean mayContainAssumingKeyIsInRange(DecoratedKey key); + /** * One instance per SSTableReader we create. - * + *

* We can create many InstanceTidiers (one for every time we reopen an sstable with MOVED_START for example), * but there can only be one GlobalTidy for one single logical sstable. - * + *

* When the InstanceTidier cleansup, it releases its reference to its GlobalTidy; when all InstanceTidiers * for that type have run, the GlobalTidy cleans up. */ - private static final class InstanceTidier implements Tidy + protected static final class InstanceTidier implements Tidy { private final Descriptor descriptor; - private final TableId tableId; - private IFilter bf; - private IndexSummary summary; + private final WeakReference owner; - private FileHandle dfile; - private FileHandle ifile; + private List closeables; private Runnable runOnClose; + private boolean isReplaced = false; // a reference to our shared tidy instance, that @@ -2025,26 +1346,24 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, private volatile boolean setup; - void setup(SSTableReader reader, boolean trackHotness) + public void setup(SSTableReader reader, boolean trackHotness, Collection closeables) { this.setup = true; - this.bf = reader.bf; - this.summary = reader.indexSummary; - this.dfile = reader.dfile; - this.ifile = reader.ifile; // get a new reference to the shared descriptor-type tidy this.globalRef = GlobalTidy.get(reader); this.global = globalRef.get(); if (trackHotness) global.ensureReadMeter(); + this.closeables = new ArrayList<>(closeables); } - InstanceTidier(Descriptor descriptor, TableId tableId) + private InstanceTidier(Descriptor descriptor, Owner owner) { this.descriptor = descriptor; - this.tableId = tableId; + this.owner = new WeakReference<>(owner); } + @Override public void tidy() { if (logger.isTraceEnabled()) @@ -2054,15 +1373,17 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, if (!setup) return; - final ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId); final OpOrder.Barrier barrier; - if (cfs != null) + Owner owner = this.owner.get(); + if (owner != null) { - barrier = cfs.readOrdering.newBarrier(); + barrier = owner.newReadOrderingBarrier(); barrier.issue(); } else + { barrier = null; + } ScheduledExecutors.nonPeriodicTasks.execute(new Runnable() { @@ -2077,17 +1398,36 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, if (logger.isTraceEnabled()) logger.trace("Async instance tidier for {}, after barrier", descriptor); - if (bf != null) - bf.close(); - if (summary != null) - summary.close(); - if (runOnClose != null) + Throwable exceptions = null; + if (runOnClose != null) try + { runOnClose.run(); - if (dfile != null) - dfile.close(); - if (ifile != null) - ifile.close(); - globalRef.release(); + } + catch (RuntimeException | Error ex) + { + logger.error("Failed to run on-close listeners for sstable " + descriptor.baseFile(), ex); + exceptions = ex; + } + + Throwable closeExceptions = Throwables.close(null, Iterables.filter(closeables, Objects::nonNull)); + if (closeExceptions != null) + { + logger.error("Failed to close some sstable components of " + descriptor.baseFile(), closeExceptions); + exceptions = Throwables.merge(exceptions, closeExceptions); + } + + try + { + globalRef.release(); + } + catch (RuntimeException | Error ex) + { + logger.error("Failed to release the global ref of " + descriptor.baseFile(), ex); + exceptions = Throwables.merge(exceptions, ex); + } + + if (exceptions != null) + JVMStabilityInspector.inspectThrowable(exceptions); if (logger.isTraceEnabled()) logger.trace("Async instance tidier for {}, completed", descriptor); @@ -2101,23 +1441,17 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, }); } + @Override public String name() { return descriptor.toString(); } - - void releaseSummary() - { - summary.close(); - assert summary.isCleanedUp(); - summary = null; - } } /** * One instance per logical sstable. This both tracks shared cleanup and some shared state related * to the sstable's lifecycle. - * + *

* All InstanceTidiers, on setup(), ask the static get() method for their shared state, * and stash a reference to it to be released when they are. Once all such references are * released, this shared tidy will be performed. @@ -2190,8 +1524,8 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, obsoletion.run(); // don't ideally want to dropPageCache for the file until all instances have been released - NativeLibrary.trySkipCache(desc.filenameFor(Component.DATA), 0, 0); - NativeLibrary.trySkipCache(desc.filenameFor(Component.PRIMARY_INDEX), 0, 0); + for (Component c : desc.discoverComponents()) + NativeLibrary.trySkipCache(desc.fileFor(c).absolutePath(), 0, 0); } public String name() @@ -2234,11 +1568,6 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, GlobalTidy.lookup.clear(); } - public static abstract class Factory - { - public abstract SSTableReader open(SSTableReaderBuilder builder); - } - public static class PartitionPositionBounds { public final long lowerPosition; @@ -2254,15 +1583,15 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, public final int hashCode() { int hashCode = (int) lowerPosition ^ (int) (lowerPosition >>> 32); - return 31 * (hashCode ^ (int) ((int) upperPosition ^ (upperPosition >>> 32))); + return 31 * (hashCode ^ (int) ((int) upperPosition ^ (upperPosition >>> 32))); } @Override public final boolean equals(Object o) { - if(!(o instanceof PartitionPositionBounds)) + if (!(o instanceof PartitionPositionBounds)) return false; - PartitionPositionBounds that = (PartitionPositionBounds)o; + PartitionPositionBounds that = (PartitionPositionBounds) o; return lowerPosition == that.lowerPosition && upperPosition == that.upperPosition; } } @@ -2296,7 +1625,7 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, /** * Moves the sstable in oldDescriptor to a new place (with generation etc) in newDescriptor. - * + *

* All components given will be moved/renamed */ public static SSTableReader moveAndOpenSSTable(ColumnFamilyStore cfs, Descriptor oldDescriptor, Descriptor newDescriptor, Set components, boolean copyData) @@ -2314,9 +1643,9 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, logger.error(message); throw new RuntimeException(message); } - if (new File(newDescriptor.filenameFor(Component.DATA)).exists()) + if (newDescriptor.fileFor(Components.DATA).exists()) { - String msg = String.format("File %s already exists, can't move the file there", newDescriptor.filenameFor(Component.DATA)); + String msg = String.format("File %s already exists, can't move the file there", newDescriptor.fileFor(Components.DATA)); logger.error(msg); throw new RuntimeException(msg); } @@ -2326,24 +1655,24 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, try { logger.info("Hardlinking new SSTable {} to {}", oldDescriptor, newDescriptor); - SSTableWriter.hardlink(oldDescriptor, newDescriptor, components); + hardlink(oldDescriptor, newDescriptor, components); } catch (FSWriteError ex) { logger.warn("Unable to hardlink new SSTable {} to {}, falling back to copying", oldDescriptor, newDescriptor, ex); - SSTableWriter.copy(oldDescriptor, newDescriptor, components); + copy(oldDescriptor, newDescriptor, components); } } else { logger.info("Moving new SSTable {} to {}", oldDescriptor, newDescriptor); - SSTableWriter.rename(oldDescriptor, newDescriptor, components); + rename(oldDescriptor, newDescriptor, components); } SSTableReader reader; try { - reader = SSTableReader.open(newDescriptor, components, cfs.metadata); + reader = open(cfs, newDescriptor, components, cfs.metadata); } catch (Throwable t) { @@ -2382,9 +1711,9 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, for (Component component : components) { // Only the data file is compressable. - bytes += logical && component == Component.DATA && compression + bytes += logical && component == Components.DATA && compression ? getCompressionMetadata().dataLength - : new File(descriptor.filenameFor(component)).length(); + : descriptor.fileFor(component).length(); } return bytes; } @@ -2394,4 +1723,179 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, { tidy.global.maybePersistSSTableReadMeter(); } + + /** + * Returns a new verifier for this sstable. Note that the reader must match the provided cfs. + */ + public abstract IVerifier getVerifier(ColumnFamilyStore cfs, + OutputHandler outputHandler, + boolean isOffline, + IVerifier.Options options); + + /** + * A method to be called by {@link #getPosition(PartitionPosition, Operator, boolean, boolean, SSTableReadsListener)} + * and {@link #getRowIndexEntry(PartitionPosition, Operator, boolean, boolean, SSTableReadsListener)} methods when + * a searched key is found. It adds a trace message and notify the provided listener. + */ + protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry) + { + reason.trace(descriptor, entry); + + if (localListener != null) + localListener.onSSTableSelected(this, reason); + } + + /** + * A method to be called by {@link #getPosition(PartitionPosition, Operator, boolean, boolean, SSTableReadsListener)} + * and {@link #getRowIndexEntry(PartitionPosition, Operator, boolean, boolean, SSTableReadsListener)} methods when + * a searched key is not found. It adds a trace message and notify the provided listener. + */ + protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats) + { + reason.trace(descriptor); + + if (localListener != null) + localListener.onSSTableSkipped(this, reason); + } + + /** + * A builder of this sstable reader. It should be extended for each implementation of {@link SSTableReader} with + * the implementation specific fields. + * + * @param type of the reader the builder creates + * @param type of this builder + */ + public abstract static class Builder> extends SSTable.Builder + { + private long maxDataAge; + private StatsMetadata statsMetadata; + private OpenReason openReason; + private SerializationHeader serializationHeader; + private FileHandle dataFile; + private DecoratedKey first; + private DecoratedKey last; + private boolean suspected; + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + public B setMaxDataAge(long maxDataAge) + { + Preconditions.checkArgument(maxDataAge >= 0); + this.maxDataAge = maxDataAge; + return (B) this; + } + + public B setStatsMetadata(StatsMetadata statsMetadata) + { + Preconditions.checkNotNull(statsMetadata); + this.statsMetadata = statsMetadata; + return (B) this; + } + + public B setOpenReason(OpenReason openReason) + { + Preconditions.checkNotNull(openReason); + this.openReason = openReason; + return (B) this; + } + + public B setSerializationHeader(SerializationHeader serializationHeader) + { + this.serializationHeader = serializationHeader; + return (B) this; + } + + public B setDataFile(FileHandle dataFile) + { + this.dataFile = dataFile; + return (B) this; + } + + public B setFirst(DecoratedKey first) + { + this.first = first != null ? first.retainable() : null; + return (B) this; + } + + public B setLast(DecoratedKey last) + { + this.last = last != null ? last.retainable() : null; + return (B) this; + } + + public B setSuspected(boolean suspected) + { + this.suspected = suspected; + return (B) this; + } + + public long getMaxDataAge() + { + return maxDataAge; + } + + public StatsMetadata getStatsMetadata() + { + return statsMetadata; + } + + public OpenReason getOpenReason() + { + return openReason; + } + + public SerializationHeader getSerializationHeader() + { + return serializationHeader; + } + + public FileHandle getDataFile() + { + return dataFile; + } + + public DecoratedKey getFirst() + { + return first; + } + + public DecoratedKey getLast() + { + return last; + } + + public boolean isSuspected() + { + return suspected; + } + + protected abstract R buildInternal(Owner owner); + + public R build(Owner owner, boolean validate, boolean online) + { + R reader = buildInternal(owner); + + try + { + if (isSuspected()) + reader.markSuspect(); + + reader.setup(online); + + if (validate) + reader.validate(); + } + catch (RuntimeException | Error ex) + { + JVMStabilityInspector.inspectThrowable(ex); + reader.selfRef().release(); + throw ex; + } + + return reader; + } + } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java deleted file mode 100644 index 1539023163..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.io.sstable.format; - -import org.apache.cassandra.cache.ChunkCache; -import org.apache.cassandra.config.Config; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; -import org.apache.cassandra.io.util.DiskOptimizationStrategy; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileHandle; -import org.apache.cassandra.io.util.FileInputStreamPlus; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.io.util.RandomAccessReader; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.utils.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.apache.cassandra.io.sstable.format.SSTableReader.OpenReason.NORMAL; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - -public abstract class SSTableReaderBuilder -{ - private static final Logger logger = LoggerFactory.getLogger(SSTableReaderBuilder.class); - - protected final SSTableReader.Factory readerFactory; - protected final Descriptor descriptor; - protected final TableMetadataRef metadataRef; - protected final TableMetadata metadata; - protected final long maxDataAge; - protected final Set components; - protected final StatsMetadata statsMetadata; - protected final SSTableReader.OpenReason openReason; - protected final SerializationHeader header; - - protected IndexSummary summary; - protected DecoratedKey first; - protected DecoratedKey last; - protected IFilter bf; - protected FileHandle ifile; - protected FileHandle dfile; - - public SSTableReaderBuilder(Descriptor descriptor, - TableMetadataRef metadataRef, - long maxDataAge, - Set components, - StatsMetadata statsMetadata, - SSTableReader.OpenReason openReason, - SerializationHeader header) - { - this.descriptor = descriptor; - this.metadataRef = metadataRef; - this.metadata = metadataRef.get(); - this.maxDataAge = maxDataAge; - this.components = components; - this.statsMetadata = statsMetadata; - this.openReason = openReason; - this.header = header; - this.readerFactory = descriptor.getFormat().getReaderFactory(); - - if (statsMetadata.firstKey != null && statsMetadata.lastKey != null) - { - this.first = metadata.partitioner.decorateKey(statsMetadata.firstKey); - this.last = metadata.partitioner.decorateKey(statsMetadata.lastKey); - } - } - - public abstract SSTableReader build(); - - public SSTableReaderBuilder dfile(FileHandle dfile) - { - this.dfile = dfile; - return this; - } - - public SSTableReaderBuilder ifile(FileHandle ifile) - { - this.ifile = ifile; - return this; - } - - public SSTableReaderBuilder bf(IFilter bf) - { - this.bf = bf; - return this; - } - - public SSTableReaderBuilder summary(IndexSummary summary) - { - this.summary = summary; - return this; - } - - /** - * Load index summary, first key and last key from Summary.db file if it exists. - * - * if loaded index summary has different index interval from current value stored in schema, - * then Summary.db file will be deleted and need to be rebuilt. - */ - void loadSummary() - { - File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY)); - if (!summariesFile.exists()) - { - if (logger.isDebugEnabled()) - logger.debug("SSTable Summary File {} does not exist", summariesFile.absolutePath()); - return; - } - - DataInputStream iStream = null; - try - { - iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath())); - summary = IndexSummary.serializer.deserialize(iStream, - metadata.partitioner, - metadata.params.minIndexInterval, - metadata.params.maxIndexInterval); - if (first == null || last == null) - { - // this is a legacy code in order to be able to load older sstables - // since version 'nc', first and last keys are stored in stats file - first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); - last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); - } - } - catch (IOException e) - { - if (summary != null) - summary.close(); - logger.trace("Cannot deserialize SSTable Summary File {}: {}", summariesFile.path(), e.getMessage()); - // corrupted; delete it and fall back to creating a new summary - FileUtils.closeQuietly(iStream); - // delete it and fall back to creating a new summary - FileUtils.deleteWithConfirm(summariesFile); - } - finally - { - FileUtils.closeQuietly(iStream); - } - } - - /** - * Build index summary, first key, last key if {@code summaryLoaded} is false and recreate bloom filter if - * {@code recreteBloomFilter} is true by reading through Index.db file. - * - * @param recreateBloomFilter true if recreate bloom filter - * @param summaryLoaded true if index summary, first key and last key are already loaded and not need to build again - */ - void buildSummaryAndBloomFilter(boolean recreateBloomFilter, - boolean summaryLoaded, - Set components, - StatsMetadata statsMetadata) throws IOException - { - if (!components.contains(Component.PRIMARY_INDEX)) - return; - - if (logger.isDebugEnabled()) - logger.debug("Attempting to build summary for {}", descriptor); - - - // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary. - try (RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)))) - { - long indexSize = primaryIndex.length(); - long histogramCount = statsMetadata.estimatedPartitionSize.count(); - long estimatedKeys = histogramCount > 0 && !statsMetadata.estimatedPartitionSize.isOverflowed() - ? histogramCount - : SSTable.estimateRowsFromIndex(primaryIndex, descriptor); // statistics is supposed to be optional - - if (recreateBloomFilter) - bf = FilterFactory.getFilter(estimatedKeys, metadata.params.bloomFilterFpChance); - - try (IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata.params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL)) - { - long indexPosition; - - while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) - { - ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex); - RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); - DecoratedKey decoratedKey = metadata.partitioner.decorateKey(key); - - if (!summaryLoaded) - { - if (first == null) - first = decoratedKey; - last = decoratedKey; - } - - if (recreateBloomFilter) - bf.add(decoratedKey); - - // if summary was already read from disk we don't want to re-populate it using primary index - if (!summaryLoaded) - { - summaryBuilder.maybeAddEntry(decoratedKey, indexPosition); - } - } - - if (!summaryLoaded) - summary = summaryBuilder.build(metadata.partitioner); - } - } - - if (!summaryLoaded) - { - first = SSTable.getMinimalKey(first); - last = SSTable.getMinimalKey(last); - } - } - - /** - * Load bloom filter from Filter.db file. - * - * @throws IOException - */ - IFilter loadBloomFilter() throws IOException - { - try (FileInputStreamPlus stream = new File(descriptor.filenameFor(Component.FILTER)).newInputStream()) - { - return BloomFilterSerializer.deserialize(stream, descriptor.version.hasOldBfFormat()); - } - } - - public static class ForWriter extends SSTableReaderBuilder - { - public ForWriter(Descriptor descriptor, - TableMetadataRef metadataRef, - long maxDataAge, - Set components, - StatsMetadata statsMetadata, - SSTableReader.OpenReason openReason, - SerializationHeader header) - { - super(descriptor, metadataRef, maxDataAge, components, statsMetadata, openReason, header); - } - - @Override - public SSTableReader build() - { - SSTableReader reader = readerFactory.open(this); - - reader.setup(true); - return reader; - } - } - - public static class ForBatch extends SSTableReaderBuilder - { - public ForBatch(Descriptor descriptor, - TableMetadataRef metadataRef, - Set components, - StatsMetadata statsMetadata, - SerializationHeader header) - { - super(descriptor, metadataRef, currentTimeMillis(), components, statsMetadata, NORMAL, header); - } - - @Override - public SSTableReader build() - { - String dataFilePath = descriptor.filenameFor(Component.DATA); - long fileLength = new File(dataFilePath).length(); - logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength)); - - initSummary(dataFilePath, components, statsMetadata); - - boolean compression = components.contains(Component.COMPRESSION_INFO); - try (FileHandle.Builder ibuilder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)) - .mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap) - .withChunkCache(ChunkCache.instance); - FileHandle.Builder dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression) - .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap) - .withChunkCache(ChunkCache.instance)) - { - long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); - DiskOptimizationStrategy optimizationStrategy = DatabaseDescriptor.getDiskOptimizationStrategy(); - int dataBufferSize = optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); - int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / summary.size()); - ifile = ibuilder.bufferSize(indexBufferSize).complete(); - dfile = dbuilder.bufferSize(dataBufferSize).complete(); - bf = FilterFactory.AlwaysPresent; - - SSTableReader sstable = readerFactory.open(this); - - sstable.first = first; - sstable.last = last; - - sstable.setup(false); - return sstable; - } - } - - void initSummary(String dataFilePath, Set components, StatsMetadata statsMetadata) - { - loadSummary(); - if (summary == null) - { - try - { - buildSummaryAndBloomFilter(false, false, components, statsMetadata); - } - catch (IOException e) - { - throw new CorruptSSTableException(e, dataFilePath); - } - } - } - } - - public static class ForRead extends SSTableReaderBuilder - { - private final ValidationMetadata validationMetadata; - private final boolean isOffline; - - public ForRead(Descriptor descriptor, - TableMetadataRef metadataRef, - ValidationMetadata validationMetadata, - boolean isOffline, - Set components, - StatsMetadata statsMetadata, - SerializationHeader header) - { - super(descriptor, metadataRef, currentTimeMillis(), components, statsMetadata, NORMAL, header); - this.validationMetadata = validationMetadata; - this.isOffline = isOffline; - } - - @Override - public SSTableReader build() - { - String dataFilePath = descriptor.filenameFor(Component.DATA); - long fileLength = new File(dataFilePath).length(); - logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(fileLength)); - - try - { - // load index and filter - long start = nanoTime(); - load(validationMetadata, isOffline, components, DatabaseDescriptor.getDiskOptimizationStrategy(), statsMetadata); - logger.trace("INDEX LOAD TIME for {}: {} ms.", descriptor, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start)); - } - catch (IOException t) - { - throw new CorruptSSTableException(t, dataFilePath); - } - - SSTableReader sstable = readerFactory.open(this); - - sstable.first = first; - sstable.last = last; - - sstable.setup(!isOffline); // Don't track hotness if we're offline. - return sstable; - } - - /** - * @param validation Metadata for SSTable being loaded - * @param isOffline Whether the SSTable is being loaded by an offline tool (sstabledump, scrub, etc) - */ - private void load(ValidationMetadata validation, - boolean isOffline, - Set components, - DiskOptimizationStrategy optimizationStrategy, - StatsMetadata statsMetadata) throws IOException - { - if (metadata.params.bloomFilterFpChance == 1.0) - { - // bf is disabled. - load(false, !isOffline, optimizationStrategy, statsMetadata, components); - bf = FilterFactory.AlwaysPresent; - } - else if (!components.contains(Component.PRIMARY_INDEX)) // What happens if filter component and primary index is missing? - { - // avoid any reading of the missing primary index component. - // this should only happen during StandaloneScrubber - load(false, !isOffline, optimizationStrategy, statsMetadata, components); - } - else if (!components.contains(Component.FILTER) || validation == null) - { - // bf is enabled, but filter component is missing. - load(!isOffline, !isOffline, optimizationStrategy, statsMetadata, components); - if (isOffline) - bf = FilterFactory.AlwaysPresent; - } - else - { - // bf is enabled and fp chance matches the currently configured value. - load(false, !isOffline, optimizationStrategy, statsMetadata, components); - bf = loadBloomFilter(); - } - } - - /** - * Loads ifile, dfile and indexSummary, and optionally recreates and persists the bloom filter. - * @param recreateBloomFilter Recreate the bloomfilter. - * @param saveSummaryIfCreated for bulk loading purposes, if the summary was absent and needed to be built, you can - * avoid persisting it to disk by setting this to false - */ - void load(boolean recreateBloomFilter, - boolean saveSummaryIfCreated, - DiskOptimizationStrategy optimizationStrategy, - StatsMetadata statsMetadata, - Set components) throws IOException - { - try(FileHandle.Builder ibuilder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)) - .mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap) - .withChunkCache(ChunkCache.instance); - FileHandle.Builder dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(components.contains(Component.COMPRESSION_INFO)) - .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap) - .withChunkCache(ChunkCache.instance)) - { - loadSummary(); - boolean buildSummary = summary == null || recreateBloomFilter; - if (buildSummary) - buildSummaryAndBloomFilter(recreateBloomFilter, summary != null, components, statsMetadata); - - int dataBufferSize = optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); - - if (components.contains(Component.PRIMARY_INDEX)) - { - long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); - int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / summary.size()); - ifile = ibuilder.bufferSize(indexBufferSize).complete(); - } - - dfile = dbuilder.bufferSize(dataBufferSize).complete(); - - if (buildSummary) - { - if (saveSummaryIfCreated) - SSTableReader.saveSummary(descriptor, first, last, summary); - if (recreateBloomFilter) - SSTableReader.saveBloomFilter(descriptor, bf); - } - } - catch (Throwable t) - { // Because the tidier has not been set-up yet in SSTableReader.open(), we must release the files in case of error - if (ifile != null) - { - ifile.close(); - } - - if (dfile != null) - { - dfile.close(); - } - - if (summary != null) - { - summary.close(); - } - - throw t; - } - } - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java new file mode 100644 index 0000000000..aedd860922 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderLoadingBuilder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IOOptions; +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; +import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.JVMStabilityInspector; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.cassandra.db.Directories.SECONDARY_INDEX_NAME_SEPARATOR; +import static org.apache.cassandra.io.sstable.format.SSTableReader.OpenReason.NORMAL; + +public abstract class SSTableReaderLoadingBuilder> +{ + private final static Logger logger = LoggerFactory.getLogger(SSTableReaderLoadingBuilder.class); + + protected final Descriptor descriptor; + protected final Set components; + protected final TableMetadataRef tableMetadataRef; + protected final IOOptions ioOptions; + protected final ChunkCache chunkCache; + + public SSTableReaderLoadingBuilder(SSTable.Builder builder) + { + this.descriptor = builder.descriptor; + this.components = builder.getComponents() != null ? ImmutableSet.copyOf(builder.getComponents()) : TOCComponent.loadOrCreate(this.descriptor); + this.tableMetadataRef = builder.getTableMetadataRef() != null ? builder.getTableMetadataRef() : resolveTableMetadataRef(); + this.ioOptions = builder.getIOOptions() != null ? builder.getIOOptions() : IOOptions.fromDatabaseDescriptor(); + this.chunkCache = builder.getChunkCache() != null ? builder.getChunkCache() : ChunkCache.instance; + + checkNotNull(this.components); + checkNotNull(this.tableMetadataRef); + } + + public R build(SSTable.Owner owner, boolean validate, boolean online) + { + checkArgument(components.contains(Components.DATA), "Data component is missing for sstable %s", descriptor); + if (validate) + checkArgument(this.components.containsAll(descriptor.getFormat().primaryComponents()), "Some required components (%s) are missing for sstable %s", Sets.difference(descriptor.getFormat().primaryComponents(), this.components), descriptor); + + B builder = (B) descriptor.getFormat().getReaderFactory().builder(descriptor); + builder.setOpenReason(NORMAL); + builder.setMaxDataAge(Clock.Global.currentTimeMillis()); + builder.setTableMetadataRef(tableMetadataRef); + builder.setComponents(components); + + R reader = null; + + try + { + CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(descriptor, builder.getComponents()); + + long t0 = Clock.Global.currentTimeMillis(); + + openComponents(builder, owner, validate, online); + + if (logger.isTraceEnabled()) + logger.trace("SSTable {} loaded in {}ms", descriptor, Clock.Global.currentTimeMillis() - t0); + + reader = builder.build(owner, validate, online); + + return reader; + } + catch (RuntimeException | IOException | Error ex) + { + if (reader != null) + reader.selfRef().release(); + + JVMStabilityInspector.inspectThrowable(ex); + + if (ex instanceof CorruptSSTableException) + throw (CorruptSSTableException) ex; + + throw new CorruptSSTableException(ex, descriptor.baseFile()); + } + } + + public abstract KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException; + + protected abstract void openComponents(B builder, SSTable.Owner owner, boolean validate, boolean online) throws IOException; + + /** + * Check if sstable is created using same partitioner. + * Partitioner can be null, which indicates older version of sstable or no stats available. + * In that case, we skip the check. + */ + protected void validatePartitioner(TableMetadata metadata, ValidationMetadata validationMetadata) + { + String partitionerName = metadata.partitioner.getClass().getCanonicalName(); + if (validationMetadata != null && !partitionerName.equals(validationMetadata.partitioner)) + { + throw new CorruptSSTableException(new IOException(String.format("Cannot open %s; partitioner %s does not match system partitioner %s. " + + "Note that the default partitioner starting with Cassandra 1.2 is Murmur3Partitioner, " + + "so you will need to edit that to match your old partitioner if upgrading.", + descriptor, validationMetadata.partitioner, partitionerName)), + descriptor.fileFor(Components.STATS)); + } + } + + private TableMetadataRef resolveTableMetadataRef() + { + TableMetadataRef metadata; + if (descriptor.cfname.contains(SECONDARY_INDEX_NAME_SEPARATOR)) + { + int i = descriptor.cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); + String indexName = descriptor.cfname.substring(i + 1); + metadata = Schema.instance.getIndexTableMetadataRef(descriptor.ksname, indexName); + if (metadata == null) + throw new AssertionError("Could not find index metadata for index cf " + i); + } + else + { + metadata = Schema.instance.getTableMetadataRef(descriptor.ksname, descriptor.cfname); + } + + return metadata; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java new file mode 100644 index 0000000000..c31f9c8021 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderWithFilter.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.common.collect.Lists; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.io.sstable.filter.BloomFilterTracker; +import org.apache.cassandra.utils.IFilter; + +import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull; + +public abstract class SSTableReaderWithFilter extends SSTableReader +{ + private final IFilter filter; + private final BloomFilterTracker filterTracker; + + protected SSTableReaderWithFilter(Builder builder, Owner owner) + { + super(builder, owner); + this.filter = Objects.requireNonNull(builder.getFilter()); + this.filterTracker = new BloomFilterTracker(); + } + + @Override + protected List setupInstance(boolean trackHotness) + { + ArrayList closeables = Lists.newArrayList(filter); + closeables.addAll(super.setupInstance(trackHotness)); + return closeables; + } + + protected final > B unbuildTo(B builder, boolean sharedCopy) + { + B b = super.unbuildTo(builder, sharedCopy); + if (builder.getFilter() == null) + b.setFilter(sharedCopy ? sharedCopyOrNull(filter) : filter); + return b; + } + + protected boolean isPresentInFilter(IFilter.FilterKey key) + { + return filter.isPresent(key); + } + + @Override + public boolean mayContainAssumingKeyIsInRange(DecoratedKey key) + { + // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing), + // we check index file instead. + return !filter.isInformative() && getPosition(key, Operator.EQ, false) >= 0 || filter.isPresent(key); + } + + @Override + protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry) + { + super.notifySelected(reason, localListener, op, updateStats, entry); + + if (!updateStats && op == SSTableReader.Operator.EQ) + return; + + filterTracker.addTruePositive(); + } + + @Override + protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats) + { + super.notifySkipped(reason, localListener, op, updateStats); + + if (!updateStats) + return; + + switch (reason) + { + case BLOOM_FILTER: + filterTracker.addTrueNegative(); + break; + case MIN_MAX_KEYS: + // checking bloom filter against keys outside the sstable range make no sense so collecting + // statistics on that makes no sense either + break; + default: + if (op == SSTableReader.Operator.EQ) + filterTracker.addFalsePositive(); + } + } + + public BloomFilterTracker getFilterTracker() + { + return filterTracker; + } + + public long getFilterSerializedSize() + { + return filter.serializedSize(descriptor.version.hasOldBfFormat()); + } + + public long getFilterOffHeapSize() + { + return filter.offHeapSize(); + } + + public abstract SSTableReaderWithFilter cloneAndReplace(IFilter filter); + + public abstract static class Builder> extends SSTableReader.Builder + { + private IFilter filter; + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + public B setFilter(IFilter filter) + { + this.filter = filter; + return (B) this; + } + + public IFilter getFilter() + { + return this.filter; + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 7270fd2e32..255e1cd04a 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -18,48 +18,47 @@ package org.apache.cassandra.io.sstable.format; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; +import com.google.common.collect.ImmutableSet; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.DeletionPurger; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.db.rows.ComplexColumnData; -import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.MetadataComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.io.util.MmappedRegionsCache; +import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Transactional; +import static com.google.common.base.Preconditions.checkNotNull; + /** - * This is the API all table writers must implement. - * - * TableWriter.create() is the primary way to create a writer for a particular format. - * The format information is part of the Descriptor. + * A root class for a writer implementation. A writer must be created by passing an implementation-specific + * {@link Builder}, a {@link LifecycleNewTracker} and {@link SSTable.Owner} instances. Implementing classes should + * not extend that list and all the additional properties should be included in the builder. */ public abstract class SSTableWriter extends SSTable implements Transactional { @@ -69,143 +68,57 @@ public abstract class SSTableWriter extends SSTable implements Transactional protected long maxDataAge = -1; protected final long keyCount; protected final MetadataCollector metadataCollector; - protected final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; protected final SerializationHeader header; - protected final TransactionalProxy txnProxy = txnProxy(); protected final Collection observers; + protected final MmappedRegionsCache mmappedRegionsCache; + protected final TransactionalProxy txnProxy = txnProxy(); + protected final LifecycleNewTracker lifecycleNewTracker; + protected DecoratedKey first; + protected DecoratedKey last; + /** + * The implementing method should return an instance of {@link TransactionalProxy} initialized with a list of all + * transactional resources included in this writer. + */ protected abstract TransactionalProxy txnProxy(); - // due to lack of multiple inheritance, we use an inner class to proxy our Transactional implementation details - protected abstract class TransactionalProxy extends AbstractTransactional + protected SSTableWriter(Builder builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner) { - // should be set during doPrepare() - protected SSTableReader finalReader; - protected boolean openResult; + super(builder, owner); + checkNotNull(builder.getFlushObservers()); + checkNotNull(builder.getMetadataCollector()); + checkNotNull(builder.getSerializationHeader()); + + this.keyCount = builder.getKeyCount(); + this.repairedAt = builder.getRepairedAt(); + this.pendingRepair = builder.getPendingRepair(); + this.isTransient = builder.isTransientSSTable(); + this.metadataCollector = builder.getMetadataCollector(); + this.header = builder.getSerializationHeader(); + this.observers = builder.getFlushObservers(); + this.mmappedRegionsCache = builder.getMmappedRegionsCache(); + this.lifecycleNewTracker = lifecycleNewTracker; + + lifecycleNewTracker.trackNew(this); } - protected SSTableWriter(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - TableMetadataRef metadata, - MetadataCollector metadataCollector, - SerializationHeader header, - Collection observers) + @Override + public DecoratedKey getFirst() { - super(descriptor, components(metadata.getLocal()), metadata, DatabaseDescriptor.getDiskOptimizationStrategy()); - this.keyCount = keyCount; - this.repairedAt = repairedAt; - this.pendingRepair = pendingRepair; - this.isTransient = isTransient; - this.metadataCollector = metadataCollector; - this.header = header; - this.rowIndexEntrySerializer = descriptor.version.getSSTableFormat().getIndexSerializer(metadata.get(), descriptor.version, header); - this.observers = observers == null ? Collections.emptySet() : observers; + return first; } - public static SSTableWriter create(Descriptor descriptor, - Long keyCount, - Long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - TableMetadataRef metadata, - MetadataCollector metadataCollector, - SerializationHeader header, - Collection indexes, - LifecycleNewTracker lifecycleNewTracker) + @Override + public DecoratedKey getLast() { - Factory writerFactory = descriptor.getFormat().getWriterFactory(); - return writerFactory.open(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers(descriptor, indexes, lifecycleNewTracker.opType()), lifecycleNewTracker); + return last; } - public static SSTableWriter create(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - int sstableLevel, - SerializationHeader header, - Collection indexes, - LifecycleNewTracker lifecycleNewTracker) + @Override + public AbstractBounds getBounds() { - TableMetadataRef metadata = Schema.instance.getTableMetadataRef(descriptor); - return create(metadata, descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, indexes, lifecycleNewTracker); - } - - public static SSTableWriter create(TableMetadataRef metadata, - Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - int sstableLevel, - SerializationHeader header, - Collection indexes, - LifecycleNewTracker lifecycleNewTracker) - { - MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel); - return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, lifecycleNewTracker); - } - - @VisibleForTesting - public static SSTableWriter create(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - SerializationHeader header, - Collection indexes, - LifecycleNewTracker lifecycleNewTracker) - { - return create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, 0, header, indexes, lifecycleNewTracker); - } - - private static Set components(TableMetadata metadata) - { - Set components = new HashSet(Arrays.asList(Component.DATA, - Component.PRIMARY_INDEX, - Component.STATS, - Component.SUMMARY, - Component.TOC, - Component.DIGEST)); - - if (metadata.params.bloomFilterFpChance < 1.0) - components.add(Component.FILTER); - - if (metadata.params.compression.isEnabled()) - { - components.add(Component.COMPRESSION_INFO); - } - else - { - // it would feel safer to actually add this component later in maybeWriteDigest(), - // but the components are unmodifiable after construction - components.add(Component.CRC); - } - return components; - } - - private static Collection observers(Descriptor descriptor, - Collection indexes, - OperationType operationType) - { - if (indexes == null) - return Collections.emptyList(); - - List observers = new ArrayList<>(indexes.size()); - for (Index index : indexes) - { - SSTableFlushObserver observer = index.getFlushObserver(descriptor, operationType); - if (observer != null) - { - observer.begin(); - observers.add(observer); - } - } - - return ImmutableList.copyOf(observers); + return (first != null && last != null) ? AbstractBounds.bounds(first.getToken(), true, last.getToken(), true) + : null; } public abstract void mark(); @@ -217,44 +130,62 @@ public abstract class SSTableWriter extends SSTable implements Transactional * @return the created index entry if something was written, that is if {@code iterator} * wasn't empty, {@code null} otherwise. * - * @throws FSWriteError if a write to the dataFile fails + * @throws FSWriteError if writing to the dataFile fails */ - public abstract RowIndexEntry append(UnfilteredRowIterator iterator); + public abstract AbstractRowIndexEntry append(UnfilteredRowIterator iterator); + /** + * Returns a position in the uncompressed data - for uncompressed files it is the same as {@link #getOnDiskFilePointer()} + * but for compressed files it returns a position in the data rather than a position in the file on disk. + */ public abstract long getFilePointer(); + /** + * Returns a position in the (compressed) data file on disk. See {@link #getFilePointer()} + */ public abstract long getOnDiskFilePointer(); + /** + * Returns the amount of data already written to disk that may not be accurate (for example, the position after + * the recently flushed chunk). + */ public long getEstimatedOnDiskBytesWritten() { return getOnDiskFilePointer(); } + /** + * Reset the data file to the marked position (see {@link #mark()}) and truncate the rest of the file. + */ public abstract void resetAndTruncate(); - public SSTableWriter setRepairedAt(long repairedAt) + public void setRepairedAt(long repairedAt) { if (repairedAt > 0) this.repairedAt = repairedAt; - return this; } - public SSTableWriter setMaxDataAge(long maxDataAge) + public void setMaxDataAge(long maxDataAge) { this.maxDataAge = maxDataAge; - return this; } - public SSTableWriter setOpenResult(boolean openResult) + public void setOpenResult(boolean openResult) { txnProxy.openResult = openResult; - return this; } /** - * Open the resultant SSTableReader before it has been fully written + * Open the resultant SSTableReader before it has been fully written. + *

+ * The passed consumer will be called when the necessary data has been flushed to disk/cache. This may never happen + * (e.g. if the table was finished before the flushes materialized, or if this call returns false e.g. if a table + * was already prepared but hasn't reached readiness yet). + *

+ * Uses callback instead of future because preparation and callback happen on the same thread. */ - public abstract SSTableReader openEarly(); + + public abstract void openEarly(Consumer doWhenReady); /** * Open the resultant SSTableReader once it has been fully written, but before the @@ -262,17 +193,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional */ public abstract SSTableReader openFinalEarly(); - public SSTableReader finish(long repairedAt, long maxDataAge, boolean openResult) - { - if (repairedAt > 0) - this.repairedAt = repairedAt; - this.maxDataAge = maxDataAge; - return finish(openResult); - } + protected abstract SSTableReader openFinal(SSTableReader.OpenReason openReason); public SSTableReader finish(boolean openResult) { - setOpenResult(openResult); + this.setOpenResult(openResult); txnProxy.finish(); observers.forEach(SSTableFlushObserver::complete); return finished(); @@ -284,6 +209,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional */ public SSTableReader finished() { + txnProxy.finalReaderAccessed = true; return txnProxy.finalReader; } @@ -328,8 +254,8 @@ public abstract class SSTableWriter extends SSTable implements Transactional pendingRepair, isTransient, header, - SSTable.getMinimalKey(first).getKey(), - SSTable.getMinimalKey(last).getKey()); + first.retainable().getKey(), + last.retainable().getKey()); } protected StatsMetadata statsMetadata() @@ -342,50 +268,8 @@ public abstract class SSTableWriter extends SSTable implements Transactional metadataCollector.release(); } - public static void rename(Descriptor tmpdesc, Descriptor newdesc, Set components) - { - for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY))) - { - FileUtils.renameWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component)); - } - - // do -Data last because -Data present should mean the sstable was completely renamed before crash - FileUtils.renameWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA)); - - // rename it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader - FileUtils.renameWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY)); - } - - public static void copy(Descriptor tmpdesc, Descriptor newdesc, Set components) - { - for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY))) - { - FileUtils.copyWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component)); - } - - // do -Data last because -Data present should mean the sstable was completely copied before crash - FileUtils.copyWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA)); - - // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader - FileUtils.copyWithOutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY)); - } - - public static void hardlink(Descriptor tmpdesc, Descriptor newdesc, Set components) - { - for (Component component : Sets.difference(components, Sets.newHashSet(Component.DATA, Component.SUMMARY))) - { - FileUtils.createHardLinkWithConfirm(tmpdesc.filenameFor(component), newdesc.filenameFor(component)); - } - - // do -Data last because -Data present should mean the sstable was completely copied before crash - FileUtils.createHardLinkWithConfirm(tmpdesc.filenameFor(Component.DATA), newdesc.filenameFor(Component.DATA)); - - // copy it without confirmation because summary can be available for loadNewSSTables but not for closeAndOpenReader - FileUtils.createHardLinkWithoutConfirm(tmpdesc.filenameFor(Component.SUMMARY), newdesc.filenameFor(Component.SUMMARY)); - } - /** - * Parameters for calculating the expected size of an sstable. Exposed on memtable flush sets (i.e. collected + * Parameters for calculating the expected size of an SSTable. Exposed on memtable flush sets (i.e. collected * subsets of a memtable that will be written to sstables). */ public interface SSTableSizeParameters @@ -395,58 +279,225 @@ public abstract class SSTableWriter extends SSTable implements Transactional long dataSize(); } - public static abstract class Factory + // due to lack of multiple inheritance, we use an inner class to proxy our Transactional implementation details + protected class TransactionalProxy extends AbstractTransactional { - public abstract long estimateSize(SSTableSizeParameters parameters); + // should be set during doPrepare() + private final Supplier> transactionals; - public abstract SSTableWriter open(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - TableMetadataRef metadata, - MetadataCollector metadataCollector, - SerializationHeader header, - Collection observers, - LifecycleNewTracker lifecycleNewTracker); + private SSTableReader finalReader; + private boolean openResult; + private boolean finalReaderAccessed; + + public TransactionalProxy(Supplier> transactionals) + { + this.transactionals = transactionals; + } + + // finalise our state on disk, including renaming + protected void doPrepare() + { + transactionals.get().forEach(Transactional::prepareToCommit); + new StatsComponent(finalizeMetadata()).save(descriptor); + + // save the table of components + TOCComponent.appendTOC(descriptor, components); + + if (openResult) + finalReader = openFinal(SSTableReader.OpenReason.NORMAL); + } + + protected Throwable doCommit(Throwable accumulate) + { + for (Transactional t : transactionals.get().reverse()) + accumulate = t.commit(accumulate); + + return accumulate; + } + + protected Throwable doAbort(Throwable accumulate) + { + for (Transactional t : transactionals.get()) + accumulate = t.abort(accumulate); + + if (!finalReaderAccessed && finalReader != null) + { + accumulate = Throwables.perform(accumulate, () -> finalReader.selfRef().release()); + finalReader = null; + finalReaderAccessed = false; + } + + return accumulate; + } + + @Override + protected Throwable doPostCleanup(Throwable accumulate) + { + accumulate = super.doPostCleanup(accumulate); + accumulate = Throwables.close(accumulate, mmappedRegionsCache); + return accumulate; + } } - public static void guardCollectionSize(TableMetadata metadata, DecoratedKey partitionKey, Unfiltered unfiltered) + /** + * A builder of this sstable writer. It should be extended for each implementation with the specific fields. + * + * An implementation should open all the resources when {@link #build(LifecycleNewTracker, Owner)} and pass them + * in builder fields to the writer, so that the writer can access them via getters. + * + * @param type of the sstable writer to be build with this builder + * @param type of this builder + */ + public abstract static class Builder> extends SSTable.Builder { - if (!Guardrails.collectionSize.enabled() && !Guardrails.itemsPerCollection.enabled()) - return; + private MetadataCollector metadataCollector; + private long keyCount; + private long repairedAt; + private TimeUUID pendingRepair; + private boolean transientSSTable; + private SerializationHeader serializationHeader; + private Collection flushObservers; - if (!unfiltered.isRow() || SchemaConstants.isSystemKeyspace(metadata.keyspace)) - return; - - Row row = (Row) unfiltered; - for (ColumnMetadata column : row.columns()) + public B setMetadataCollector(MetadataCollector metadataCollector) { - if (!column.type.isCollection() || !column.type.isMultiCell()) - continue; + this.metadataCollector = metadataCollector; + return (B) this; + } - ComplexColumnData cells = row.getComplexColumnData(column); - if (cells == null) - continue; + public B setKeyCount(long keyCount) + { + this.keyCount = keyCount; + return (B) this; + } - ComplexColumnData liveCells = cells.purge(DeletionPurger.PURGE_ALL, FBUtilities.nowInSeconds()); - if (liveCells == null) - continue; + public B setRepairedAt(long repairedAt) + { + this.repairedAt = repairedAt; + return (B) this; + } - int cellsSize = liveCells.dataSize(); - int cellsCount = liveCells.cellsCount(); + public B setPendingRepair(TimeUUID pendingRepair) + { + this.pendingRepair = pendingRepair; + return (B) this; + } - if (!Guardrails.collectionSize.triggersOn(cellsSize, null) && - !Guardrails.itemsPerCollection.triggersOn(cellsCount, null)) - continue; + public B setTransientSSTable(boolean transientSSTable) + { + this.transientSSTable = transientSSTable; + return (B) this; + } - String keyString = metadata.primaryKeyAsCQLLiteral(partitionKey.getKey(), row.clustering()); - String msg = String.format("%s in row %s in table %s", - column.name.toString(), - keyString, - metadata); - Guardrails.collectionSize.guard(cellsSize, msg, true, null); - Guardrails.itemsPerCollection.guard(cellsCount, msg, true, null); + public B setSerializationHeader(SerializationHeader serializationHeader) + { + this.serializationHeader = serializationHeader; + return (B) this; + } + + public B setFlushObservers(Collection flushObservers) + { + this.flushObservers = ImmutableList.copyOf(flushObservers); + return (B) this; + } + + public B addDefaultComponents() + { + checkNotNull(getTableMetadataRef()); + + addComponents(ImmutableSet.of(Components.DATA, Components.STATS, Components.DIGEST, Components.TOC)); + + if (getTableMetadataRef().getLocal().params.compression.isEnabled()) + { + addComponents(ImmutableSet.of(Components.COMPRESSION_INFO)); + } + else + { + // it would feel safer to actually add this component later in maybeWriteDigest(), + // but the components are unmodifiable after construction + addComponents(ImmutableSet.of(Components.CRC)); + } + + return (B) this; + } + + public B addFlushObserversForSecondaryIndexes(Collection indexes, OperationType operationType) + { + if (indexes == null) + return (B) this; + + Collection current = this.flushObservers != null ? this.flushObservers : Collections.emptyList(); + List observers = new ArrayList<>(indexes.size() + current.size()); + observers.addAll(current); + + for (Index index : indexes) + { + SSTableFlushObserver observer = index.getFlushObserver(descriptor, operationType); + if (observer != null) + { + observer.begin(); + observers.add(observer); + } + } + + return setFlushObservers(observers); + } + + public MetadataCollector getMetadataCollector() + { + return metadataCollector; + } + + public long getKeyCount() + { + return keyCount; + } + + public long getRepairedAt() + { + return repairedAt; + } + + public TimeUUID getPendingRepair() + { + return pendingRepair; + } + + public boolean isTransientSSTable() + { + return transientSSTable; + } + + public SerializationHeader getSerializationHeader() + { + return serializationHeader; + } + + public Collection getFlushObservers() + { + return flushObservers; + } + + public abstract MmappedRegionsCache getMmappedRegionsCache(); + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + public W build(LifecycleNewTracker lifecycleNewTracker, Owner owner) + { + checkNotNull(getComponents()); + + validateRepairedMetadata(getRepairedAt(), getPendingRepair(), isTransientSSTable()); + + return buildInternal(lifecycleNewTracker, owner); + } + + protected abstract W buildInternal(LifecycleNewTracker lifecycleNewTracker, Owner owner); + + public SSTableZeroCopyWriter createZeroCopyWriter(LifecycleNewTracker lifecycleNewTracker, Owner owner) + { + return new SSTableZeroCopyWriter(this, lifecycleNewTracker, owner); } } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java new file mode 100644 index 0000000000..6634af3910 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; + +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredSerializer; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static com.google.common.base.Preconditions.checkState; + +public abstract class SortedTablePartitionWriter implements AutoCloseable +{ + protected final UnfilteredSerializer unfilteredSerializer; + + private final SerializationHeader header; + private final SequentialWriter writer; + private final SerializationHelper helper; + private final int version; + + private long previousRowStart; + private long initialPosition; + private long headerLength; + + protected long startPosition; + protected int written; + + protected ClusteringPrefix firstClustering; + protected ClusteringPrefix lastClustering; + + protected DeletionTime openMarker = DeletionTime.LIVE; + protected DeletionTime startOpenMarker = DeletionTime.LIVE; + + // Sequence control, also used to add empty static row if `addStaticRow` is not called. + private enum State + { + AWAITING_PARTITION_HEADER, + AWAITING_STATIC_ROW, + AWAITING_ROWS, + COMPLETED + } + + State state = State.AWAITING_PARTITION_HEADER; + + protected SortedTablePartitionWriter(SerializationHeader header, SequentialWriter writer, Version version) + { + this.header = header; + this.writer = writer; + this.unfilteredSerializer = UnfilteredSerializer.serializer; + this.helper = new SerializationHelper(header); + this.version = version.correspondingMessagingVersion(); + } + + protected void reset() + { + this.initialPosition = writer.position(); + this.startPosition = -1; + this.previousRowStart = 0; + this.written = 0; + this.firstClustering = null; + this.lastClustering = null; + this.openMarker = DeletionTime.LIVE; + this.headerLength = -1; + this.state = State.AWAITING_PARTITION_HEADER; + } + + public long getHeaderLength() + { + return headerLength; + } + + public void start(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException + { + if (state == State.COMPLETED) + reset(); + + checkState(state == State.AWAITING_PARTITION_HEADER); + + ByteBufferUtil.writeWithShortLength(key.getKey(), writer); + DeletionTime.serializer.serialize(partitionLevelDeletion, writer); + + if (!header.hasStatic()) + { + this.headerLength = writer.position() - initialPosition; + state = State.AWAITING_ROWS; + return; + } + + state = State.AWAITING_STATIC_ROW; + } + + public void addStaticRow(Row staticRow) throws IOException + { + checkState(state == State.AWAITING_STATIC_ROW); + checkState(staticRow.isStatic()); + + UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version); + + this.headerLength = writer.position() - initialPosition; + state = State.AWAITING_ROWS; + } + + public void addUnfiltered(Unfiltered unfiltered) throws IOException + { + checkState(state == State.AWAITING_ROWS); + + long pos = currentPosition(); + + if (firstClustering == null) + { + // Beginning of an index block. Remember the start and position + firstClustering = unfiltered.clustering(); + startOpenMarker = openMarker; + startPosition = pos; + } + + long unfilteredPosition = writer.position(); + unfilteredSerializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version); + + lastClustering = unfiltered.clustering(); + previousRowStart = pos; + ++written; + + if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) + { + RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; + openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : DeletionTime.LIVE; + } + } + + protected long finish() throws IOException + { + checkState(state == State.AWAITING_ROWS); + + state = State.COMPLETED; + + long endPosition = currentPosition(); + unfilteredSerializer.writeEndOfPartition(writer); + + return endPosition; + } + + protected long currentPosition() + { + return writer.position() - initialPosition; + } + + public long getInitialPosition() + { + return initialPosition; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java new file mode 100644 index 0000000000..07255873e6 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java @@ -0,0 +1,631 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOError; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.concurrent.NotThreadSafe; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.partitions.ImmutableBTreePartition; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.rows.AbstractCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.ColumnData; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.SSTableIdentityIterator; +import org.apache.cassandra.io.sstable.SSTableRewriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.AbstractIterator; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.Refs; +import org.apache.cassandra.utils.memory.HeapCloner; + +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; + +@NotThreadSafe +public abstract class SortedTableScrubber implements IScrubber +{ + private final static Logger logger = LoggerFactory.getLogger(SortedTableScrubber.class); + + protected final ColumnFamilyStore cfs; + protected final LifecycleTransaction transaction; + protected final File destination; + protected final IScrubber.Options options; + protected final R sstable; + protected final OutputHandler outputHandler; + protected final boolean isCommutative; + protected final long expectedBloomFilterSize; + protected final ReadWriteLock fileAccessLock = new ReentrantReadWriteLock(); + protected final RandomAccessReader dataFile; + protected final ScrubInfo scrubInfo; + + protected final NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics = new NegativeLocalDeletionInfoMetrics(); + + private static final Comparator partitionComparator = Comparator.comparing(Partition::partitionKey); + protected final SortedSet outOfOrder = new TreeSet<>(partitionComparator); + + + protected int goodPartitions; + protected int badPartitions; + protected int emptyPartitions; + + + protected SortedTableScrubber(ColumnFamilyStore cfs, + LifecycleTransaction transaction, + OutputHandler outputHandler, + Options options) + { + this.sstable = (R) transaction.onlyOne(); + Preconditions.checkNotNull(sstable.metadata()); + assert sstable.metadata().keyspace.equals(cfs.keyspace.getName()); + if (!sstable.descriptor.cfname.equals(cfs.metadata().name)) + { + logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name); + } + try + { + sstable.metadata().validateCompatibility(cfs.metadata()); + } + catch (ConfigurationException ex) + { + logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name); + } + + this.cfs = cfs; + this.transaction = transaction; + this.outputHandler = outputHandler; + this.options = options; + this.destination = cfs.getDirectories().getLocationForDisk(cfs.getDiskBoundaries().getCorrectDiskForSSTable(sstable)); + this.isCommutative = cfs.metadata().isCounter(); + + List toScrub = Collections.singletonList(sstable); + + long approximateKeyCount; + try + { + approximateKeyCount = SSTableReader.getApproximateKeyCount(toScrub); + } + catch (RuntimeException ex) + { + approximateKeyCount = 0; + } + this.expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, approximateKeyCount); + + // loop through each partition, deserializing to check for damage. + // We'll also loop through the index at the same time, using the position from the index to recover if the + // partition header (key or data size) is corrupt. (This means our position in the index file will be one + // partition "ahead" of the data file.) + this.dataFile = transaction.isOffline() + ? sstable.openDataReader() + : sstable.openDataReader(CompactionManager.instance.getRateLimiter()); + + this.scrubInfo = new ScrubInfo(dataFile, sstable, fileAccessLock.readLock()); + + if (options.reinsertOverflowedTTLRows) + outputHandler.output("Starting scrub with reinsert overflowed TTL option"); + } + + public static void deleteOrphanedComponents(Descriptor descriptor, Set components) + { + File dataFile = descriptor.fileFor(Components.DATA); + if (components.contains(Components.DATA) && dataFile.length() > 0) + // everything appears to be in order... moving on. + return; + + // missing the DATA file! all components are orphaned + logger.warn("Removing orphans for {}: {}", descriptor, components); + for (Component component : components) + { + File file = descriptor.fileFor(component); + if (file.exists()) + descriptor.fileFor(component).delete(); + } + } + + @Override + public void scrub() + { + List finished = new ArrayList<>(); + outputHandler.output("Scrubbing %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())); + try (SSTableRewriter writer = SSTableRewriter.construct(cfs, transaction, false, sstable.maxDataAge); + Refs refs = Refs.ref(Collections.singleton(sstable))) + { + StatsMetadata metadata = sstable.getSSTableMetadata(); + writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)); + + scrubInternal(writer); + + if (!outOfOrder.isEmpty()) + finished.add(writeOutOfOrderPartitions(metadata)); + + // finish obsoletes the old sstable + transaction.obsoleteOriginals(); + finished.addAll(writer.setRepairedAt(badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt).finish()); + } + catch (IOException ex) + { + throw new RuntimeException(ex); + } + finally + { + if (transaction.isOffline()) + finished.forEach(sstable -> sstable.selfRef().release()); + } + + outputSummary(finished); + } + + protected abstract void scrubInternal(SSTableRewriter writer) throws IOException; + + private void outputSummary(List finished) + { + if (!finished.isEmpty()) + { + outputHandler.output("Scrub of %s complete: %d partitions in new sstable and %d empty (tombstoned) partitions dropped", sstable, goodPartitions, emptyPartitions); + if (negativeLocalDeletionInfoMetrics.fixedRows > 0) + outputHandler.output("Fixed %d rows with overflowed local deletion time.", negativeLocalDeletionInfoMetrics.fixedRows); + if (badPartitions > 0) + outputHandler.warn("Unable to recover %d partitions that were skipped. You can attempt manual recovery from the pre-scrub snapshot. You can also run nodetool repair to transfer the data from a healthy replica, if any", badPartitions); + } + else + { + if (badPartitions > 0) + outputHandler.warn("No valid partitions found while scrubbing %s; it is marked for deletion now. If you want to attempt manual recovery, you can find a copy in the pre-scrub snapshot", sstable); + else + outputHandler.output("Scrub of %s complete; looks like all %d partitions were tombstoned", sstable, emptyPartitions); + } + } + + private SSTableReader writeOutOfOrderPartitions(StatsMetadata metadata) + { + // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time + long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt; + SSTableReader newInOrderSstable; + try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)) + { + for (Partition partition : outOfOrder) + inOrderWriter.append(partition.unfilteredIterator()); + inOrderWriter.setRepairedAt(-1); + inOrderWriter.setMaxDataAge(sstable.maxDataAge); + newInOrderSstable = inOrderWriter.finish(true); + } + transaction.update(newInOrderSstable, false); + outputHandler.warn("%d out of order partition (or partitions without of order rows) found while scrubbing %s; " + + "Those have been written (in order) to a new sstable (%s)", outOfOrder.size(), sstable, newInOrderSstable); + return newInOrderSstable; + } + + protected abstract UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename); + + @Override + @VisibleForTesting + public ScrubResult scrubWithResult() + { + scrub(); + return new ScrubResult(goodPartitions, badPartitions, emptyPartitions); + } + + @Override + public CompactionInfo.Holder getScrubInfo() + { + return scrubInfo; + } + + protected String keyString(DecoratedKey key) + { + if (key == null) + return "(unknown)"; + + try + { + return cfs.metadata().partitionKeyType.getString(key.getKey()); + } + catch (Exception e) + { + return String.format("(corrupted; hex value: %s)", ByteBufferUtil.bytesToHex(key.getKey())); + } + } + + protected boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer) + { + // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects + // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added + // to the outOfOrder set that will be later written to a new SSTable. + try (OrderCheckerIterator sstableIterator = new OrderCheckerIterator(getIterator(key), cfs.metadata().comparator); + UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath())) + { + if (prevKey != null && prevKey.compareTo(key) > 0) + { + saveOutOfOrderPartition(prevKey, key, iterator); + return false; + } + + if (writer.tryAppend(iterator) == null) + emptyPartitions++; + else + goodPartitions++; + + if (sstableIterator.hasRowsOutOfOrder()) + { + outputHandler.warn("Out of order rows found in partition: %s", keyString(key)); + outOfOrder.add(sstableIterator.getRowsOutOfOrder()); + } + } + + return true; + } + + /** + * Only wrap with {@link FixNegativeLocalDeletionTimeIterator} if {@link IScrubber.Options#reinsertOverflowedTTLRows} option + * is specified + */ + private UnfilteredRowIterator getIterator(DecoratedKey key) + { + RowMergingSSTableIterator rowMergingIterator = new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key), outputHandler); + if (options.reinsertOverflowedTTLRows) + return new FixNegativeLocalDeletionTimeIterator(rowMergingIterator, outputHandler, negativeLocalDeletionInfoMetrics); + else + return rowMergingIterator; + } + + private void saveOutOfOrderPartition(DecoratedKey prevKey, DecoratedKey key, UnfilteredRowIterator iterator) + { + // TODO bitch if the row is too large? if it is there's not much we can do ... + outputHandler.warn("Out of order partition detected (%s found after %s)", keyString(key), keyString(prevKey)); + outOfOrder.add(ImmutableBTreePartition.create(iterator)); + } + + protected static void throwIfFatal(Throwable th) + { + if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError)) + throw (Error) th; + } + + protected void throwIfCannotContinue(DecoratedKey key, Throwable th) + { + if (isCommutative && !options.skipCorrupted) + { + outputHandler.warn("An error occurred while scrubbing the partition with key '%s'. Skipping corrupt " + + "data in counter tables will result in undercounts for the affected " + + "counters (see CASSANDRA-2759 for more details), so by default the scrub will " + + "stop at this point. If you would like to skip the row anyway and continue " + + "scrubbing, re-run the scrub with the --skip-corrupted option.", + keyString(key)); + throw new IOError(th); + } + } + + + public static class ScrubInfo extends CompactionInfo.Holder + { + private final RandomAccessReader dataFile; + private final SSTableReader sstable; + private final TimeUUID scrubCompactionId; + private final Lock fileReadLock; + + public ScrubInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock) + { + this.dataFile = dataFile; + this.sstable = sstable; + this.fileReadLock = fileReadLock; + scrubCompactionId = nextTimeUUID(); + } + + public CompactionInfo getCompactionInfo() + { + fileReadLock.lock(); + try + { + return new CompactionInfo(sstable.metadata(), + OperationType.SCRUB, + dataFile.getFilePointer(), + dataFile.length(), + scrubCompactionId, + ImmutableSet.of(sstable), + Paths.get(sstable.getFilename()).getParent().toString()); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + finally + { + fileReadLock.unlock(); + } + } + + public boolean isGlobal() + { + return false; + } + } + + /** + * In some case like CASSANDRA-12127 the cells might have been stored in the wrong order. This decorator check the + * cells order and collect the out-of-order cells to correct the problem. + */ + private static final class OrderCheckerIterator extends AbstractIterator implements WrappingUnfilteredRowIterator + { + private final UnfilteredRowIterator iterator; + private final ClusteringComparator comparator; + + private Unfiltered previous; + + /** + * The partition containing the rows which are out of order. + */ + private Partition rowsOutOfOrder; + + public OrderCheckerIterator(UnfilteredRowIterator iterator, ClusteringComparator comparator) + { + this.iterator = iterator; + this.comparator = comparator; + } + + @Override + public UnfilteredRowIterator wrapped() + { + return iterator; + } + + public boolean hasRowsOutOfOrder() + { + return rowsOutOfOrder != null; + } + + public Partition getRowsOutOfOrder() + { + return rowsOutOfOrder; + } + + @Override + protected Unfiltered computeNext() + { + if (!iterator.hasNext()) + return endOfData(); + + Unfiltered next = iterator.next(); + + // If we detect that some rows are out of order we will store and sort the remaining ones to insert them + // in a separate SSTable. + if (previous != null && comparator.compare(next, previous) < 0) + { + rowsOutOfOrder = ImmutableBTreePartition.create(UnfilteredRowIterators.concat(next, iterator), false); + return endOfData(); + } + previous = next; + return next; + } + } + + /** + * During 2.x migration, under some circumstances rows might have gotten duplicated. + * Merging iterator merges rows with same clustering. + *

+ * For more details, refer to CASSANDRA-12144. + */ + private static class RowMergingSSTableIterator implements WrappingUnfilteredRowIterator + { + Unfiltered nextToOffer = null; + private final OutputHandler output; + private final UnfilteredRowIterator wrapped; + + RowMergingSSTableIterator(UnfilteredRowIterator source, OutputHandler output) + { + this.wrapped = source; + this.output = output; + } + + @Override + public UnfilteredRowIterator wrapped() + { + return wrapped; + } + + @Override + public boolean hasNext() + { + return nextToOffer != null || wrapped.hasNext(); + } + + @Override + public Unfiltered next() + { + Unfiltered next = nextToOffer != null ? nextToOffer : wrapped.next(); + + if (next.isRow()) + { + boolean logged = false; + while (wrapped.hasNext()) + { + Unfiltered peek = wrapped.next(); + if (!peek.isRow() || !next.clustering().equals(peek.clustering())) + { + nextToOffer = peek; // Offer peek in next call + return next; + } + + // Duplicate row, merge it. + next = Rows.merge((Row) next, (Row) peek); + + if (!logged) + { + String partitionKey = metadata().partitionKeyType.getString(partitionKey().getKey()); + output.warn("Duplicate row detected in %s.%s: %s %s", metadata().keyspace, metadata().name, partitionKey, next.clustering().toString(metadata())); + logged = true; + } + } + } + + nextToOffer = null; + return next; + } + } + + /** + * This iterator converts negative {@link AbstractCell#localDeletionTime()} into {@link AbstractCell#MAX_DELETION_TIME} + *

+ * This is to recover entries with overflowed localExpirationTime due to CASSANDRA-14092 + */ + private static final class FixNegativeLocalDeletionTimeIterator extends AbstractIterator implements WrappingUnfilteredRowIterator + { + /** + * The decorated iterator. + */ + private final UnfilteredRowIterator iterator; + + private final OutputHandler outputHandler; + private final NegativeLocalDeletionInfoMetrics negativeLocalExpirationTimeMetrics; + + public FixNegativeLocalDeletionTimeIterator(UnfilteredRowIterator iterator, OutputHandler outputHandler, + NegativeLocalDeletionInfoMetrics negativeLocalDeletionInfoMetrics) + { + this.iterator = iterator; + this.outputHandler = outputHandler; + this.negativeLocalExpirationTimeMetrics = negativeLocalDeletionInfoMetrics; + } + + @Override + public UnfilteredRowIterator wrapped() + { + return iterator; + } + + @Override + protected Unfiltered computeNext() + { + if (!iterator.hasNext()) + return endOfData(); + + Unfiltered next = iterator.next(); + if (!next.isRow()) + return next; + + if (hasNegativeLocalExpirationTime((Row) next)) + { + outputHandler.debug("Found row with negative local expiration time: %s", next.toString(metadata(), false)); + negativeLocalExpirationTimeMetrics.fixedRows++; + return fixNegativeLocalExpirationTime((Row) next); + } + + return next; + } + + private boolean hasNegativeLocalExpirationTime(Row next) + { + Row row = next; + if (row.primaryKeyLivenessInfo().isExpiring() && row.primaryKeyLivenessInfo().localExpirationTime() < 0) + { + return true; + } + + for (ColumnData cd : row) + { + if (cd.column().isSimple()) + { + Cell cell = (Cell) cd; + if (cell.isExpiring() && cell.localDeletionTime() < 0) + return true; + } + else + { + ComplexColumnData complexData = (ComplexColumnData) cd; + for (Cell cell : complexData) + { + if (cell.isExpiring() && cell.localDeletionTime() < 0) + return true; + } + } + } + + return false; + } + + private Unfiltered fixNegativeLocalExpirationTime(Row row) + { + LivenessInfo livenessInfo = row.primaryKeyLivenessInfo(); + if (livenessInfo.isExpiring() && livenessInfo.localExpirationTime() < 0) + livenessInfo = livenessInfo.withUpdatedTimestampAndLocalDeletionTime(livenessInfo.timestamp() + 1, AbstractCell.MAX_DELETION_TIME); + + return row.transformAndFilter(livenessInfo, row.deletion(), cd -> { + if (cd.column().isSimple()) + { + Cell cell = (Cell) cd; + return cell.isExpiring() && cell.localDeletionTime() < 0 + ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME) + : cell; + } + else + { + ComplexColumnData complexData = (ComplexColumnData) cd; + return complexData.transformAndFilter(cell -> cell.isExpiring() && cell.localDeletionTime() < 0 + ? cell.withUpdatedTimestampAndLocalDeletionTime(cell.timestamp() + 1, AbstractCell.MAX_DELETION_TIME) + : cell); + } + }).clone(HeapCloner.instance); + } + } + + private static class NegativeLocalDeletionInfoMetrics + { + public volatile int fixedRows = 0; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java new file mode 100644 index 0000000000..f3cfdedd69 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOError; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; +import java.util.function.LongPredicate; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.compaction.CompactionController; +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.IVerifier; +import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.DataIntegrityMetadata; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.TimeUUID; + +public abstract class SortedTableVerifier implements IVerifier +{ + private final static Logger logger = LoggerFactory.getLogger(SortedTableVerifier.class); + + protected final ColumnFamilyStore cfs; + protected final R sstable; + + protected final ReadWriteLock fileAccessLock; + protected final RandomAccessReader dataFile; + protected final VerifyInfo verifyInfo; + protected final Options options; + protected final boolean isOffline; + + /** + * Given a keyspace, return the set of local and pending token ranges. By default {@link StorageService#getLocalAndPendingRanges(String)} + * is expected, but for the standalone verifier case we can't use that, so this is here to allow the CLI to provide + * the token ranges. + */ + protected final Function>> tokenLookup; + protected int goodRows; + + protected final OutputHandler outputHandler; + + public SortedTableVerifier(ColumnFamilyStore cfs, R sstable, OutputHandler outputHandler, boolean isOffline, Options options) + { + this.cfs = cfs; + this.sstable = sstable; + this.outputHandler = outputHandler; + + this.fileAccessLock = new ReentrantReadWriteLock(); + this.dataFile = isOffline + ? sstable.openDataReader() + : sstable.openDataReader(CompactionManager.instance.getRateLimiter()); + this.verifyInfo = new VerifyInfo(dataFile, sstable, fileAccessLock.readLock()); + this.options = options; + this.isOffline = isOffline; + this.tokenLookup = options.tokenLookup; + } + + protected void deserializeBloomFilter(SSTableReader sstable) throws IOException + { + try (IFilter filter = FilterComponent.load(sstable.descriptor)) { + if (filter != null) + logger.trace("Filter loaded for {}", sstable); + } + } + + public CompactionInfo.Holder getVerifyInfo() + { + return verifyInfo; + } + + protected static void throwIfFatal(Throwable th) + { + if (th instanceof Error && !(th instanceof AssertionError || th instanceof IOError)) + throw (Error) th; + } + + protected void markAndThrow(Throwable cause) + { + markAndThrow(cause, true); + } + + protected void markAndThrow(Throwable cause, boolean mutateRepaired) + { + if (mutateRepaired && options.mutateRepairStatus) // if we are able to mutate repaired flag, an incremental repair should be enough + { + try + { + sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient()); + cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); + } + catch (IOException ioe) + { + outputHandler.output("Error mutating repairedAt for SSTable %s, as part of markAndThrow", sstable.getFilename()); + } + } + Exception e = new Exception(String.format("Invalid SSTable %s, please force %srepair", sstable.getFilename(), (mutateRepaired && options.mutateRepairStatus) ? "" : "a full "), cause); + if (options.invokeDiskFailurePolicy) + throw new CorruptSSTableException(e, sstable.getFilename()); + else + throw new RuntimeException(e); + } + + protected void verifyBloomFilter() + { + try + { + outputHandler.debug("Deserializing bloom filter for %s", sstable); + deserializeBloomFilter(sstable); + } + catch (Throwable t) + { + outputHandler.warn(t); + markAndThrow(t); + } + } + + protected void verifySSTableMetadata() + { + outputHandler.output("Deserializing sstable metadata for %s ", sstable); + try + { + StatsComponent statsComponent = StatsComponent.load(sstable.descriptor, MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER); + if (statsComponent.validationMetadata() != null && + !statsComponent.validationMetadata().partitioner.equals(sstable.getPartitioner().getClass().getCanonicalName())) + throw new IOException("Partitioner does not match validation metadata"); + } + catch (Throwable t) + { + outputHandler.warn(t); + markAndThrow(t, false); + } + } + + protected void verifySSTableVersion() + { + outputHandler.output("Verifying %s (%s)", sstable, FBUtilities.prettyPrintMemory(dataFile.length())); + if (options.checkVersion && !sstable.descriptor.version.isLatestVersion()) + { + String msg = String.format("%s is not the latest version, run upgradesstables", sstable); + outputHandler.output(msg); + // don't use markAndThrow here because we don't want a CorruptSSTableException for this. + throw new RuntimeException(msg); + } + } + + protected int verifyOwnedRanges() + { + List> ownedRanges = Collections.emptyList(); + outputHandler.debug("Checking that all tokens are owned by the current node"); + try (KeyIterator iter = sstable.keyIterator()) + { + ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace)); + if (ownedRanges.isEmpty()) + return 0; + RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges); + while (iter.hasNext()) + { + DecoratedKey key = iter.next(); + rangeOwnHelper.validate(key); + } + } + catch (Throwable t) + { + outputHandler.warn(t); + markAndThrow(t); + } + + return ownedRanges.size(); + } + + protected boolean verifyDigest() + { + boolean passed = true; + // Verify will use the Digest files, which works for both compressed and uncompressed sstables + outputHandler.output("Checking computed hash of %s ", sstable); + try + { + DataIntegrityMetadata.FileDigestValidator validator = sstable.maybeGetDigestValidator(); + + if (validator != null) + { + validator.validate(); + } + else + { + outputHandler.output("Data digest missing, assuming extended verification of disk values"); + passed = false; + } + } + catch (IOException e) + { + outputHandler.warn(e); + markAndThrow(e); + } + return passed; + } + + @Override + public void close() + { + fileAccessLock.writeLock().lock(); + try + { + FileUtils.closeQuietly(dataFile); + } + finally + { + fileAccessLock.writeLock().unlock(); + } + } + + /** + * Use the fact that check(...) is called with sorted tokens - we keep a pointer in to the normalized ranges + * and only bump the pointer if the key given is out of range. This is done to avoid calling .contains(..) many + * times for each key (with vnodes for example) + */ + @VisibleForTesting + public static class RangeOwnHelper + { + private final List> normalizedRanges; + private int rangeIndex = 0; + private DecoratedKey lastKey; + + public RangeOwnHelper(List> normalizedRanges) + { + this.normalizedRanges = normalizedRanges; + Range.assertNormalized(normalizedRanges); + } + + /** + * check if the given key is contained in any of the given ranges + *

+ * Must be called in sorted order - key should be increasing + * + * @param key the key + * @throws RuntimeException if the key is not contained + */ + public void validate(DecoratedKey key) + { + if (!check(key)) + throw new RuntimeException("Key " + key + " is not contained in the given ranges"); + } + + /** + * check if the given key is contained in any of the given ranges + *

+ * Must be called in sorted order - key should be increasing + * + * @param key the key + * @return boolean + */ + public boolean check(DecoratedKey key) + { + assert lastKey == null || key.compareTo(lastKey) > 0; + lastKey = key; + + if (normalizedRanges.isEmpty()) // handle tests etc. where we don't have any ranges + return true; + + if (rangeIndex > normalizedRanges.size() - 1) + throw new IllegalStateException("RangeOwnHelper can only be used to find the first out-of-range-token"); + + while (!normalizedRanges.get(rangeIndex).contains(key.getToken())) + { + rangeIndex++; + if (rangeIndex > normalizedRanges.size() - 1) + return false; + } + + return true; + } + } + + protected static class VerifyInfo extends CompactionInfo.Holder + { + private final RandomAccessReader dataFile; + private final SSTableReader sstable; + private final TimeUUID verificationCompactionId; + private final Lock fileReadLock; + + public VerifyInfo(RandomAccessReader dataFile, SSTableReader sstable, Lock fileReadLock) + { + this.dataFile = dataFile; + this.sstable = sstable; + this.fileReadLock = fileReadLock; + verificationCompactionId = TimeUUID.Generator.nextTimeUUID(); + } + + public CompactionInfo getCompactionInfo() + { + fileReadLock.lock(); + try + { + return new CompactionInfo(sstable.metadata(), + OperationType.VERIFY, + dataFile.getFilePointer(), + dataFile.length(), + verificationCompactionId, + ImmutableSet.of(sstable)); + } + catch (Exception e) + { + throw new RuntimeException(); + } + finally + { + fileReadLock.unlock(); + } + } + + public boolean isGlobal() + { + return false; + } + } + + protected static class VerifyController extends CompactionController + { + public VerifyController(ColumnFamilyStore cfs) + { + super(cfs, Integer.MAX_VALUE); + } + + @Override + public LongPredicate getPurgeEvaluator(DecoratedKey key) + { + return time -> false; + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java new file mode 100644 index 0000000000..d8e512805f --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; +import java.nio.BufferOverflowException; +import java.util.Set; +import java.util.function.Consumer; + +import com.google.common.collect.ImmutableSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionPurger; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.guardrails.Guardrails; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.PartitionSerializationException; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.compress.CompressedSequentialWriter; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.FilterFactory; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.concurrent.Transactional; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * A generic implementation of a writer which assumes the existence of some partition index and bloom filter. + */ +public abstract class SortedTableWriter

extends SSTableWriter +{ + private final static Logger logger = LoggerFactory.getLogger(SortedTableWriter.class); + + // TODO dataWriter is not needed to be directly accessible - we can access everything we need for the dataWriter + // from a partition writer + protected final SequentialWriter dataWriter; + protected final P partitionWriter; + private final FileHandle.Builder dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.DATA)); + private DecoratedKey lastWrittenKey; + private DataPosition dataMark; + private long lastEarlyOpenLength; + + public SortedTableWriter(Builder builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner) + { + super(builder, lifecycleNewTracker, owner); + checkNotNull(builder.getDataWriter()); + checkNotNull(builder.getPartitionWriter()); + + this.dataWriter = builder.getDataWriter(); + this.partitionWriter = builder.getPartitionWriter(); + } + + /** + * Appends partition data to this writer. + * + * @param partition the partition to write + * @return the created index entry if something was written, that is if {@code iterator} wasn't empty, + * {@code null} otherwise. + * @throws FSWriteError if write to the dataFile fails + */ + @Override + public final AbstractRowIndexEntry append(UnfilteredRowIterator partition) + { + if (partition.isEmpty()) + return null; + + try + { + if (!verifyPartition(partition.partitionKey())) + return null; + + startPartition(partition.partitionKey(), partition.partitionLevelDeletion()); + + AbstractRowIndexEntry indexEntry; + if (header.hasStatic()) + addStaticRow(partition.partitionKey(), partition.staticRow()); + + while (partition.hasNext()) + addUnfiltered(partition.partitionKey(), partition.next()); + + indexEntry = endPartition(partition.partitionKey(), partition.partitionLevelDeletion()); + + return indexEntry; + } + catch (BufferOverflowException boe) + { + throw new PartitionSerializationException(partition, boe); + } + catch (IOException e) + { + throw new FSWriteError(e, getFilename()); + } + } + + private boolean verifyPartition(DecoratedKey key) + { + assert key != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values + + if (key.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT) + { + logger.error("Key size {} exceeds maximum of {}, skipping row", key.getKey().remaining(), FBUtilities.MAX_UNSIGNED_SHORT); + return false; + } + + if (lastWrittenKey != null && lastWrittenKey.compareTo(key) >= 0) + throw new RuntimeException(String.format("Last written key %s >= current key %s, writing into %s", lastWrittenKey, key, getFilename())); + + return true; + } + + private void startPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException + { + partitionWriter.start(key, partitionLevelDeletion); + metadataCollector.updatePartitionDeletion(partitionLevelDeletion); + + onStartPartition(key); + } + + private void addStaticRow(DecoratedKey key, Row row) throws IOException + { + guardCollectionSize(key, row); + + partitionWriter.addStaticRow(row); + if (!row.isEmpty()) + Rows.collectStats(row, metadataCollector); + + onStaticRow(row); + } + + private void addUnfiltered(DecoratedKey key, Unfiltered unfiltered) throws IOException + { + if (unfiltered.isRow()) + addRow(key, (Row) unfiltered); + else + addRangeTomstoneMarker((RangeTombstoneMarker) unfiltered); + } + + private void addRow(DecoratedKey key, Row row) throws IOException + { + guardCollectionSize(key, row); + + partitionWriter.addUnfiltered(row); + metadataCollector.updateClusteringValues(row.clustering()); + Rows.collectStats(row, metadataCollector); + + onRow(row); + } + + private void addRangeTomstoneMarker(RangeTombstoneMarker marker) throws IOException + { + partitionWriter.addUnfiltered(marker); + + metadataCollector.updateClusteringValuesByBoundOrBoundary(marker.clustering()); + if (marker.isBoundary()) + { + RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker) marker; + metadataCollector.update(bm.endDeletionTime()); + metadataCollector.update(bm.startDeletionTime()); + } + else + { + metadataCollector.update(((RangeTombstoneBoundMarker) marker).deletionTime()); + } + + onRangeTombstoneMarker(marker); + } + + private AbstractRowIndexEntry endPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException + { + long finishResult = partitionWriter.finish(); + + long endPosition = dataWriter.position(); + long rowSize = endPosition - partitionWriter.getInitialPosition(); + maybeLogLargePartitionWarning(key, rowSize); + maybeLogManyTombstonesWarning(key, metadataCollector.totalTombstones); + metadataCollector.addPartitionSizeInBytes(rowSize); + metadataCollector.addKey(key.getKey()); + metadataCollector.addCellPerPartitionCount(); + + lastWrittenKey = key; + last = lastWrittenKey; + if (first == null) + first = lastWrittenKey; + + if (logger.isTraceEnabled()) + logger.trace("wrote {} at {}", key, endPosition); + + return createRowIndexEntry(key, partitionLevelDeletion, finishResult); + } + + protected void onStartPartition(DecoratedKey key) + { + notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), partitionWriter.getInitialPosition())); + } + + protected void onStaticRow(Row row) + { + notifyObservers(o -> o.staticRow(row)); + } + + protected void onRow(Row row) + { + notifyObservers(o -> o.nextUnfilteredCluster(row)); + } + + protected void onRangeTombstoneMarker(RangeTombstoneMarker marker) + { + notifyObservers(o -> o.nextUnfilteredCluster(marker)); + } + + protected abstract AbstractRowIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException; + + protected final void notifyObservers(Consumer action) + { + if (observers != null && !observers.isEmpty()) + observers.forEach(action); + } + + @Override + public void mark() + { + dataMark = dataWriter.mark(); + } + + @Override + public void resetAndTruncate() + { + dataWriter.resetAndTruncate(dataMark); + partitionWriter.reset(); + } + + @Override + public long getFilePointer() + { + return dataWriter.position(); + } + + @Override + public long getOnDiskFilePointer() + { + return dataWriter.getOnDiskFilePointer(); + } + + @Override + public long getEstimatedOnDiskBytesWritten() + { + return dataWriter.getEstimatedOnDiskBytesWritten(); + } + + protected FileHandle openDataFile(long lengthOverride, StatsMetadata statsMetadata) + { + int dataBufferSize = ioOptions.diskOptimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(ioOptions.diskOptimizationEstimatePercentile)); + + + FileHandle dataFile; + try (CompressionMetadata compressionMetadata = compression ? ((CompressedSequentialWriter) dataWriter).open(lengthOverride) : null) + { + dataFile = dataFileBuilder.mmapped(ioOptions.defaultDiskAccessMode) + .withMmappedRegionsCache(mmappedRegionsCache) + .withChunkCache(chunkCache) + .withCompressionMetadata(compressionMetadata) + .bufferSize(dataBufferSize) + .withLengthOverride(lengthOverride) + .complete(); + } + + try + { + if (chunkCache != null) + { + if (lastEarlyOpenLength != 0 && dataFile.dataLength() > lastEarlyOpenLength) + chunkCache.invalidatePosition(dataFile, lastEarlyOpenLength); + } + lastEarlyOpenLength = dataFile.dataLength(); + } + catch (RuntimeException | Error ex) + { + Throwables.closeNonNullAndAddSuppressed(ex, dataFile); + throw ex; + } + + return dataFile; + } + + private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize) + { + if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold()) + { + String keyString = metadata().partitionKeyType.getString(key.getKey()); + logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.keyspace, metadata.name, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename()); + } + } + + private void maybeLogManyTombstonesWarning(DecoratedKey key, int tombstoneCount) + { + if (tombstoneCount > DatabaseDescriptor.getCompactionTombstoneWarningThreshold()) + { + String keyString = metadata().partitionKeyType.getString(key.getKey()); + logger.warn("Writing {} tombstones to {}/{}:{} in sstable {}", tombstoneCount, metadata.keyspace, metadata.name, keyString, getFilename()); + } + } + + private void guardCollectionSize(DecoratedKey partitionKey, Row row) + { + if (!Guardrails.collectionSize.enabled() && !Guardrails.itemsPerCollection.enabled()) + return; + + if (row.isEmpty() || SchemaConstants.isSystemKeyspace(metadata.keyspace)) + return; + + for (ColumnMetadata column : row.columns()) + { + if (!column.type.isCollection() || !column.type.isMultiCell()) + continue; + + ComplexColumnData cells = row.getComplexColumnData(column); + if (cells == null) + continue; + + ComplexColumnData liveCells = cells.purge(DeletionPurger.PURGE_ALL, FBUtilities.nowInSeconds()); + if (liveCells == null) + continue; + + int cellsSize = liveCells.dataSize(); + int cellsCount = liveCells.cellsCount(); + + if (!Guardrails.collectionSize.triggersOn(cellsSize, null) && + !Guardrails.itemsPerCollection.triggersOn(cellsCount, null)) + continue; + + String keyString = metadata.getLocal().primaryKeyAsCQLLiteral(partitionKey.getKey(), row.clustering()); + String msg = String.format("%s in row %s in table %s", + column.name.toString(), + keyString, + metadata); + Guardrails.collectionSize.guard(cellsSize, msg, true, null); + Guardrails.itemsPerCollection.guard(cellsCount, msg, true, null); + } + } + + protected static abstract class AbstractIndexWriter extends AbstractTransactional implements Transactional + { + protected final Descriptor descriptor; + protected final TableMetadataRef metadata; + protected final Set components; + + protected final IFilter bf; + + protected AbstractIndexWriter(Builder b) + { + this.descriptor = b.descriptor; + this.metadata = b.getTableMetadataRef(); + this.components = b.getComponents(); + + bf = FilterFactory.getFilter(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.bloomFilterFpChance); + } + + protected void flushBf() + { + if (components.contains(Components.FILTER)) + { + try + { + FilterComponent.save(bf, descriptor, true); + } + catch (IOException ex) + { + throw new FSWriteError(ex, descriptor.fileFor(Components.FILTER)); + } + } + } + + protected void doPrepare() + { + flushBf(); + } + + @Override + protected Throwable doPostCleanup(Throwable accumulate) + { + accumulate = bf.close(accumulate); + return accumulate; + } + + public IFilter getFilterCopy() + { + return bf.sharedCopy(); + } + } + + public abstract static class Builder

, + B extends Builder> extends SSTableWriter.Builder + { + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + @Override + public B addDefaultComponents() + { + super.addDefaultComponents(); + + if (FilterComponent.shouldUseBloomFilter(getTableMetadataRef().getLocal().params.bloomFilterFpChance)) + { + addComponents(ImmutableSet.of(SSTableFormat.Components.FILTER)); + } + + return (B) this; + } + + public abstract SequentialWriter getDataWriter(); + + public abstract P getPartitionWriter(); + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java new file mode 100644 index 0000000000..25042e331f --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/StatsComponent.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.IOException; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.exceptions.UnknownColumnException; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.schema.TableMetadata; + +public class StatsComponent +{ + public final Map metadata; + + public StatsComponent(Map metadata) + { + this.metadata = ImmutableMap.copyOf(metadata); + } + + public static StatsComponent load(Descriptor descriptor) throws IOException + { + return load(descriptor, MetadataType.values()); + } + + public static StatsComponent load(Descriptor descriptor, MetadataType... types) throws IOException + { + Map metadata; + try + { + metadata = descriptor.getMetadataSerializer().deserialize(descriptor, EnumSet.copyOf(Arrays.asList(types))); + } + catch (IOException e) + { + throw new CorruptSSTableException(e, descriptor.fileFor(Components.STATS)); + } + + return new StatsComponent(metadata); + } + + public SerializationHeader.Component serializationHeader() + { + return (SerializationHeader.Component) metadata.get(MetadataType.HEADER); + } + + public SerializationHeader serializationHeader(TableMetadata metadata) + { + SerializationHeader.Component header = serializationHeader(); + if (header != null) + { + try + { + return header.toHeader(metadata); + } + catch (UnknownColumnException ex) + { + throw new IllegalArgumentException(ex); + } + } + + return null; + } + + public CompactionMetadata compactionMetadata() + { + return (CompactionMetadata) metadata.get(MetadataType.COMPACTION); + } + + public ValidationMetadata validationMetadata() + { + return (ValidationMetadata) metadata.get(MetadataType.VALIDATION); + } + + public StatsMetadata statsMetadata() + { + return (StatsMetadata) metadata.get(MetadataType.STATS); + } + + public void save(Descriptor desc) + { + File file = desc.fileFor(Components.STATS); + try (SequentialWriter out = new SequentialWriter(file, SequentialWriterOption.DEFAULT)) + { + desc.getMetadataSerializer().serialize(metadata, out, desc.version); + out.finish(); + } + catch (IOException e) + { + throw new FSWriteError(e, file.path()); + } + } + +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java new file mode 100644 index 0000000000..677d809877 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format; + +import java.io.FileNotFoundException; +import java.io.IOError; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileOutputStreamPlus; + +import static org.apache.cassandra.io.util.File.WriteMode.APPEND; + +public class TOCComponent +{ + private static final Logger logger = LoggerFactory.getLogger(TOCComponent.class); + + /** + * Reads the list of components from the TOC component. + * + * @return set of components found in the TOC + */ + public static Set loadTOC(Descriptor descriptor) throws IOException + { + return loadTOC(descriptor, true); + } + + /** + * Reads the list of components from the TOC component. + * + * @param skipMissing, skip adding the component to the returned set if the corresponding file is missing. + * @return set of components found in the TOC + */ + public static Set loadTOC(Descriptor descriptor, boolean skipMissing) throws IOException + { + File tocFile = descriptor.fileFor(Components.TOC); + List componentNames = Files.readAllLines(tocFile.toPath()); + Set components = Sets.newHashSetWithExpectedSize(componentNames.size()); + for (String componentName : componentNames) + { + Component component = Component.parse(componentName, descriptor.formatType); + if (skipMissing && !descriptor.fileFor(component).exists()) + logger.error("Missing component: {}", descriptor.fileFor(component)); + else + components.add(component); + } + return components; + } + + /** + * Appends new component names to the TOC component. + */ + public static void appendTOC(Descriptor descriptor, Collection components) + { + File tocFile = descriptor.fileFor(Components.TOC); + try (FileOutputStreamPlus out = tocFile.newOutputStream(APPEND); + PrintWriter w = new PrintWriter(out)) + { + for (Component component : components) + w.println(component.name); + w.flush(); + out.sync(); + } + catch (IOException e) + { + throw new FSWriteError(e, tocFile); + } + } + + public static Set loadOrCreate(Descriptor descriptor) + { + try + { + try + { + return TOCComponent.loadTOC(descriptor); + } + catch (FileNotFoundException | NoSuchFileException e) + { + Set components = descriptor.discoverComponents(); + if (components.isEmpty()) + return components; // sstable doesn't exist yet + + components.add(Components.TOC); + TOCComponent.appendTOC(descriptor, components); + return components; + } + } + catch (IOException e) + { + throw new IOError(e); + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index 781c2791d8..ae5feedd1a 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -40,8 +40,8 @@ public abstract class Version protected Version(SSTableFormat format, String version) { - this.format = format; - this.version = version; + this.format = Objects.requireNonNull(format); + this.version = Objects.requireNonNull(version); } public abstract boolean isLatestVersion(); diff --git a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java b/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java index 7d698a9980..25dc1c5c60 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java +++ b/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java @@ -31,6 +31,7 @@ import org.apache.cassandra.io.sstable.Descriptor; *

Note that both information are currently necessary to identify the exact "format" of an sstable (without having * its {@link Descriptor}). In particular, while {@link Version} contains its {{@link SSTableFormat}}, you cannot get * the {{@link SSTableFormat.Type}} from that. + * @deprecated TODO remove this class - Version contains SSTableFormat and you _can_ get its type from it */ public final class VersionAndType { @@ -80,7 +81,7 @@ public final class VersionAndType if (components.size() != 2) throw new IllegalArgumentException("Invalid VersionAndType string: " + versionAndType + " (should be of the form 'big-bc')"); - SSTableFormat.Type formatType = SSTableFormat.Type.validate(components.get(0)); + SSTableFormat.Type formatType = SSTableFormat.Type.getByName(components.get(0)); Version version = formatType.info.getVersion(components.get(1)); return new VersionAndType(version, formatType); } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index 3510998b64..64fb1b0353 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -17,35 +17,132 @@ */ package org.apache.cassandra.io.sstable.format.big; -import java.util.Collection; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Set; -import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.db.RowIndexEntry; -import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.*; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.MetricsProviders; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics; +import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.SortedTableScrubber; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryMetrics; +import org.apache.cassandra.io.sstable.keycache.KeyCacheMetrics; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.service.CacheService; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Pair; + +import static org.apache.cassandra.io.sstable.format.SSTableFormat.Components.DATA; /** * Legacy bigtable format */ -public class BigFormat implements SSTableFormat +public class BigFormat extends AbstractSSTableFormat { + private final static Logger logger = LoggerFactory.getLogger(BigFormat.class); + public static final BigFormat instance = new BigFormat(); - public static final Version latestVersion = new BigVersion(BigVersion.current_version); - private static final SSTableReader.Factory readerFactory = new ReaderFactory(); - private static final SSTableWriter.Factory writerFactory = new WriterFactory(); + + private final Version latestVersion = new BigVersion(this, BigVersion.current_version); + private final BigTableReaderFactory readerFactory = new BigTableReaderFactory(); + private final BigTableWriterFactory writerFactory = new BigTableWriterFactory(); + + public static class Components extends AbstractSSTableFormat.Components + { + public static class Types extends SSTableFormat.Components.Types + { + // index of the row keys with pointers to their positions in the data file + public static final Component.Type PRIMARY_INDEX = Component.Type.createSingleton("PRIMARY_INDEX", "Index.db", BigFormat.class); + // holds SSTable Index Summary (sampling of Index component) + public static final Component.Type SUMMARY = Component.Type.createSingleton("SUMMARY", "Summary.db", BigFormat.class); + } + + public final static Component PRIMARY_INDEX = Types.PRIMARY_INDEX.getSingleton(); + public final static Component SUMMARY = Types.SUMMARY.getSingleton(); + + private static final Set BATCH_COMPONENTS = ImmutableSet.of(DATA, + PRIMARY_INDEX, + COMPRESSION_INFO, + FILTER, + STATS); + + private static final Set PRIMARY_COMPONENTS = ImmutableSet.of(DATA, + PRIMARY_INDEX); + + private static final Set GENERATED_ON_LOAD_COMPONENTS = ImmutableSet.of(FILTER, SUMMARY); + + private static final Set MUTABLE_COMPONENTS = ImmutableSet.of(STATS, + SUMMARY); + + private static final Set UPLOAD_COMPONENTS = ImmutableSet.of(DATA, + PRIMARY_INDEX, + SUMMARY, + COMPRESSION_INFO, + STATS); + + private static final Set STREAM_COMPONENTS = ImmutableSet.of(DATA, + PRIMARY_INDEX, + STATS, + COMPRESSION_INFO, + FILTER, + SUMMARY, + DIGEST, + CRC); + + private static final Set ALL_COMPONENTS = ImmutableSet.of(DATA, + PRIMARY_INDEX, + STATS, + COMPRESSION_INFO, + FILTER, + SUMMARY, + DIGEST, + CRC, + TOC); + } private BigFormat() { } + public static BigFormat getInstance() + { + return instance; + } + + public static boolean isDefault() + { + return getInstance().getType() == Type.current(); + } + @Override public Version getLatestVersion() { @@ -55,28 +152,192 @@ public class BigFormat implements SSTableFormat @Override public Version getVersion(String version) { - return new BigVersion(version); + return new BigVersion(this, version); } @Override - public SSTableWriter.Factory getWriterFactory() + public BigTableWriterFactory getWriterFactory() { return writerFactory; } @Override - public SSTableReader.Factory getReaderFactory() + public BigTableReaderFactory getReaderFactory() { return readerFactory; } @Override - public RowIndexEntry.IndexSerializer getIndexSerializer(TableMetadata metadata, Version version, SerializationHeader header) + public Set allComponents() { - return new RowIndexEntry.Serializer(version, header); + return Components.ALL_COMPONENTS; } - static class WriterFactory extends SSTableWriter.Factory + @Override + public Set streamingComponents() + { + return Components.STREAM_COMPONENTS; + } + + @Override + public Set primaryComponents() + { + return Components.PRIMARY_COMPONENTS; + } + + @Override + public Set batchComponents() + { + return Components.BATCH_COMPONENTS; + } + + @Override + public Set uploadComponents() + { + return Components.UPLOAD_COMPONENTS; + } + + @Override + public Set mutableComponents() + { + return Components.MUTABLE_COMPONENTS; + } + + @Override + public Set generatedOnLoadComponents() + { + return Components.GENERATED_ON_LOAD_COMPONENTS; + } + + @Override + public SSTableFormat.KeyCacheValueSerializer getKeyCacheValueSerializer() + { + return KeyCacheValueSerializer.instance; + } + + @Override + public IScrubber getScrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, OutputHandler outputHandler, IScrubber.Options options) + { + Preconditions.checkArgument(cfs.metadata().equals(transaction.onlyOne().metadata())); + return new BigTableScrubber(cfs, transaction, outputHandler, options); + } + + @Override + public BigTableReader cast(SSTableReader sstr) + { + return (BigTableReader) sstr; + } + + @Override + public BigTableWriter cast(SSTableWriter sstw) + { + return (BigTableWriter) sstw; + } + + @Override + public MetricsProviders getFormatSpecificMetricsProviders() + { + return BigTableSpecificMetricsProviders.instance; + } + + @Override + public void deleteOrphanedComponents(Descriptor descriptor, Set components) + { + SortedTableScrubber.deleteOrphanedComponents(descriptor, components); + } + + private void delete(Descriptor desc, List components) + { + logger.info("Deleting sstable: {}", desc); + + if (components.remove(DATA)) + components.add(0, DATA); // DATA component should be first + if (components.remove(Components.SUMMARY)) + components.add(Components.SUMMARY); // SUMMARY component should be last (IDK why) + + for (Component component : components) + { + logger.trace("Deleting component {} of {}", component, desc); + desc.fileFor(component).deleteIfExists(); + } + } + + @Override + public void delete(Descriptor desc) + { + try + { + // remove key cache entries for the sstable being deleted + Iterator it = CacheService.instance.keyCache.keyIterator(); + while (it.hasNext()) + { + KeyCacheKey key = it.next(); + if (key.desc.equals(desc)) + it.remove(); + } + + delete(desc, Lists.newArrayList(Sets.intersection(allComponents(), desc.discoverComponents()))); + } + catch (Throwable t) + { + JVMStabilityInspector.inspectThrowable(t); + } + } + + static class KeyCacheValueSerializer implements SSTableFormat.KeyCacheValueSerializer + { + private final static KeyCacheValueSerializer instance = new KeyCacheValueSerializer(); + + @Override + public void skip(DataInputPlus input) throws IOException + { + RowIndexEntry.Serializer.skipForCache(input); + } + + @Override + public RowIndexEntry deserialize(BigTableReader reader, DataInputPlus input) throws IOException + { + return reader.deserializeKeyCacheValue(input); + } + + @Override + public void serialize(RowIndexEntry entry, DataOutputPlus output) throws IOException + { + entry.serializeForCache(output); + } + } + + static class BigTableReaderFactory implements SSTableReaderFactory + { + @Override + public BigTableReader.Builder builder(Descriptor descriptor) + { + return new BigTableReader.Builder(descriptor); + } + + @Override + public SSTableReaderLoadingBuilder loadingBuilder(Descriptor descriptor, + TableMetadataRef tableMetadataRef, + Set components) + { + return new BigSSTableReaderLoadingBuilder(new SSTable.Builder<>(descriptor).setTableMetadataRef(tableMetadataRef) + .setComponents(components)); + } + + @Override + public Pair readKeyRange(Descriptor descriptor, IPartitioner partitioner) throws IOException + { + return IndexSummaryComponent.loadFirstAndLastKey(descriptor.fileFor(Components.SUMMARY), partitioner); + } + + @Override + public Class getReaderClass() + { + return BigTableReader.class; + } + } + + static class BigTableWriterFactory implements SSTableWriterFactory { @Override public long estimateSize(SSTableWriter.SSTableSizeParameters parameters) @@ -88,28 +349,9 @@ public class BigFormat implements SSTableFormat } @Override - public SSTableWriter open(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - TableMetadataRef metadata, - MetadataCollector metadataCollector, - SerializationHeader header, - Collection observers, - LifecycleNewTracker lifecycleNewTracker) + public BigTableWriter.Builder builder(Descriptor descriptor) { - SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient); - return new BigTableWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers, lifecycleNewTracker); - } - } - - static class ReaderFactory extends SSTableReader.Factory - { - @Override - public SSTableReader open(SSTableReaderBuilder builder) - { - return new BigTableReader(builder); + return new BigTableWriter.Builder(descriptor); } } @@ -158,9 +400,9 @@ public class BigFormat implements SSTableFormat */ private final boolean hasOldBfFormat; - BigVersion(String version) + BigVersion(BigFormat format, String version) { - super(instance, version); + super(format, version); isLatestVersion = version.compareTo(current_version) == 0; correspondingMessagingVersion = MessagingService.VERSION_30; @@ -282,4 +524,19 @@ public class BigFormat implements SSTableFormat return isCompatible() && version.charAt(0) == current_version.charAt(0); } } + + private static class BigTableSpecificMetricsProviders implements MetricsProviders + { + private final static BigTableSpecificMetricsProviders instance = new BigTableSpecificMetricsProviders(); + + private final Iterable> gaugeProviders = Iterables.concat(BloomFilterMetrics.instance.getGaugeProviders(), + IndexSummaryMetrics.instance.getGaugeProviders(), + KeyCacheMetrics.instance.getGaugeProviders()); + + @Override + public Iterable> getGaugeProviders() + { + return gaugeProviders; + } + } } diff --git a/src/java/org/apache/cassandra/db/ColumnIndex.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java similarity index 60% rename from src/java/org/apache/cassandra/db/ColumnIndex.java rename to src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java index f7860df1e6..c2c11f9976 100644 --- a/src/java/org/apache/cassandra/db/ColumnIndex.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java @@ -16,31 +16,33 @@ * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import com.google.common.primitives.Ints; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.sstable.IndexInfo; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; -import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.SortedTablePartitionWriter; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.SequentialWriter; -import org.apache.cassandra.utils.ByteBufferUtil; /** * Column index builder used by {@link org.apache.cassandra.io.sstable.format.big.BigTableWriter}. * For index entries that exceed {@link org.apache.cassandra.config.Config#column_index_cache_size}, * this uses the serialization logic as in {@link RowIndexEntry}. */ -public class ColumnIndex +public class BigFormatPartitionWriter extends SortedTablePartitionWriter { // used, if the row-index-entry reaches config column_index_cache_size private DataOutputBuffer buffer; @@ -51,97 +53,50 @@ public class ColumnIndex private DataOutputBuffer reusableBuffer; - public int columnIndexCount; + private int columnIndexCount; private int[] indexOffsets; - private final SerializationHelper helper; - private final SerializationHeader header; - private final int version; - private final SequentialWriter writer; - private long initialPosition; - private final ISerializer idxSerializer; - public long headerLength; - private long startPosition; + private final ISerializer idxSerializer; - private int written; - private long previousRowStart; + private final int cacheSizeThreshold; + private final int indexSize; - private ClusteringPrefix firstClustering; - private ClusteringPrefix lastClustering; - - private DeletionTime openMarker; - - private int cacheSizeThreshold; - - private final Collection observers; - - public ColumnIndex(SerializationHeader header, - SequentialWriter writer, - Version version, - Collection observers, - ISerializer indexInfoSerializer) + BigFormatPartitionWriter(SerializationHeader header, + SequentialWriter writer, + Version version, + ISerializer indexInfoSerializer) { - this.helper = new SerializationHelper(header); - this.header = header; - this.writer = writer; - this.version = version.correspondingMessagingVersion(); - this.observers = observers; + this(header, writer, version, indexInfoSerializer, DatabaseDescriptor.getColumnIndexCacheSize(), DatabaseDescriptor.getColumnIndexSize()); + } + + BigFormatPartitionWriter(SerializationHeader header, + SequentialWriter writer, + Version version, + ISerializer indexInfoSerializer, + int cacheSizeThreshold, + int indexSize) + { + super(header, writer, version); this.idxSerializer = indexInfoSerializer; + this.cacheSizeThreshold = cacheSizeThreshold; + this.indexSize = indexSize; } public void reset() { - this.initialPosition = writer.position(); - this.headerLength = -1; - this.startPosition = -1; - this.previousRowStart = 0; + super.reset(); this.columnIndexCount = 0; - this.written = 0; this.indexSamplesSerializedSize = 0; this.indexSamples.clear(); - this.firstClustering = null; - this.lastClustering = null; - this.openMarker = null; - int newCacheSizeThreshold = DatabaseDescriptor.getColumnIndexCacheSize(); - if (this.buffer != null && this.cacheSizeThreshold == newCacheSizeThreshold) + if (this.buffer != null) this.reusableBuffer = this.buffer; this.buffer = null; - this.cacheSizeThreshold = newCacheSizeThreshold; } - public void buildRowIndex(UnfilteredRowIterator iterator) throws IOException + public int getColumnIndexCount() { - writePartitionHeader(iterator); - this.headerLength = writer.position() - initialPosition; - - while (iterator.hasNext()) - { - Unfiltered unfiltered = iterator.next(); - SSTableWriter.guardCollectionSize(iterator.metadata(), iterator.partitionKey(), unfiltered); - add(unfiltered); - } - - finish(); - } - - private void writePartitionHeader(UnfilteredRowIterator iterator) throws IOException - { - ByteBufferUtil.writeWithShortLength(iterator.partitionKey().getKey(), writer); - DeletionTime.serializer.serialize(iterator.partitionLevelDeletion(), writer); - if (header.hasStatic()) - { - Row staticRow = iterator.staticRow(); - - UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version); - if (!observers.isEmpty()) - observers.forEach((o) -> o.nextUnfilteredCluster(staticRow)); - } - } - - private long currentPosition() - { - return writer.position() - initialPosition; + return columnIndexCount; } public ByteBuffer buffer() @@ -172,7 +127,7 @@ public class ColumnIndex lastClustering, startPosition, currentPosition() - startPosition, - openMarker); + !openMarker.isLive() ? openMarker : null); // indexOffsets is used for both shallow (ShallowIndexedEntry) and non-shallow IndexedEntry. // For shallow ones, we need it to serialize the offsts in finish(). @@ -235,7 +190,8 @@ public class ColumnIndex { // Check whether a reusable DataOutputBuffer already exists for this // ColumnIndex instance and return it. - if (reusableBuffer != null) { + if (reusableBuffer != null) + { DataOutputBuffer buffer = reusableBuffer; buffer.clear(); return buffer; @@ -244,59 +200,43 @@ public class ColumnIndex return new DataOutputBuffer(cacheSizeThreshold * 2); } - private void add(Unfiltered unfiltered) throws IOException + @Override + public void addUnfiltered(Unfiltered unfiltered) throws IOException { - long pos = currentPosition(); - - if (firstClustering == null) - { - // Beginning of an index block. Remember the start and position - firstClustering = unfiltered.clustering(); - startPosition = pos; - } - - UnfilteredSerializer.serializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version); - - // notify observers about each new row - if (!observers.isEmpty()) - observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered)); - - lastClustering = unfiltered.clustering(); - previousRowStart = pos; - ++written; - - if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) - { - RangeTombstoneMarker marker = (RangeTombstoneMarker) unfiltered; - openMarker = marker.isOpen(false) ? marker.openDeletionTime(false) : null; - } + super.addUnfiltered(unfiltered); // if we hit the column index size that we have to index after, go ahead and index it. - if (currentPosition() - startPosition >= DatabaseDescriptor.getColumnIndexSize()) + if (currentPosition() - startPosition >= indexSize) addIndexBlock(); } - private void finish() throws IOException + @Override + public long finish() throws IOException { - UnfilteredSerializer.serializer.writeEndOfPartition(writer); + long endPosition = super.finish(); // It's possible we add no rows, just a top level deletion if (written == 0) - return; + return endPosition; // the last column may have fallen on an index boundary already. if not, index it explicitly. if (firstClustering != null) addIndexBlock(); // If we serialize the IndexInfo objects directly in the code above into 'buffer', - // we have to write the offsts to these here. The offsets have already been are collected + // we have to write the offsts to these here. The offsets have already been collected // in indexOffsets[]. buffer is != null, if it exceeds Config.column_index_cache_size. // In the other case, when buffer==null, the offsets are serialized in RowIndexEntry.IndexedEntry.serialize(). if (buffer != null) - RowIndexEntry.Serializer.serializeOffsets(buffer, indexOffsets, columnIndexCount); + { + for (int i = 0; i < columnIndexCount; i++) + buffer.writeInt(indexOffsets[i]); + } // we should always have at least one computed index block, but we only write it out if there is more than that. - assert columnIndexCount > 0 && headerLength >= 0; + assert columnIndexCount > 0 && getHeaderLength() >= 0; + + return endPosition; } public int indexInfoSerializedSize() @@ -305,4 +245,10 @@ public class ColumnIndex ? buffer.buffer().limit() : indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0); } -} + + @Override + public void close() + { + // no-op + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java new file mode 100644 index 0000000000..307fbdf921 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; +import java.util.OptionalInt; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.Downsampling; +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; +import org.apache.cassandra.io.sstable.format.FilterComponent; +import org.apache.cassandra.io.sstable.format.IndexComponent; +import org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder; +import org.apache.cassandra.io.sstable.format.StatsComponent; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder; +import org.apache.cassandra.io.sstable.keycache.KeyCache; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; +import org.apache.cassandra.io.util.DiskOptimizationStrategy; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.service.CacheService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.FilterFactory; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.Throwables; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +public class BigSSTableReaderLoadingBuilder extends SSTableReaderLoadingBuilder +{ + private final static Logger logger = LoggerFactory.getLogger(BigSSTableReaderLoadingBuilder.class); + + private FileHandle.Builder dataFileBuilder; + private FileHandle.Builder indexFileBuilder; + + public BigSSTableReaderLoadingBuilder(SSTable.Builder descriptor) + { + super(descriptor); + } + + @Override + protected void openComponents(BigTableReader.Builder builder, SSTable.Owner owner, boolean validate, boolean online) throws IOException + { + try + { + if (online && builder.getTableMetadataRef().getLocal().params.caching.cacheKeys()) + builder.setKeyCache(new KeyCache(CacheService.instance.keyCache)); + + StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION); + builder.setSerializationHeader(statsComponent.serializationHeader(builder.getTableMetadataRef().getLocal())); + checkArgument(!online || builder.getSerializationHeader() != null); + + builder.setStatsMetadata(statsComponent.statsMetadata()); + if (descriptor.version.hasKeyRange() && statsComponent.statsMetadata() != null) + { + builder.setFirst(tableMetadataRef.getLocal().partitioner.decorateKey(statsComponent.statsMetadata().firstKey)); + builder.setLast(tableMetadataRef.getLocal().partitioner.decorateKey(statsComponent.statsMetadata().lastKey)); + } + + ValidationMetadata validationMetadata = statsComponent.validationMetadata(); + validatePartitioner(builder.getTableMetadataRef().getLocal(), validationMetadata); + + boolean filterNeeded = online; + if (filterNeeded) + builder.setFilter(loadFilter(validationMetadata)); + boolean rebuildFilter = filterNeeded && builder.getFilter() == null; + + boolean summaryNeeded = true; + if (summaryNeeded) + { + IndexSummaryComponent summaryComponent = loadSummary(); + if (summaryComponent != null) + { + if (builder.getFirst() == null || builder.getLast() == null) + { + builder.setFirst(summaryComponent.first); + builder.setLast(summaryComponent.last); + } + builder.setIndexSummary(summaryComponent.indexSummary); + } + } + boolean rebuildSummary = summaryNeeded && builder.getIndexSummary() == null; + + if (builder.getComponents().contains(Components.PRIMARY_INDEX) && (rebuildFilter || rebuildSummary)) + { + try (FileHandle indexFile = indexFileBuilder(builder.getIndexSummary()).complete()) + { + Pair filterAndSummary = buildSummaryAndBloomFilter(indexFile, builder.getSerializationHeader(), rebuildFilter, rebuildSummary, owner != null ? owner.getMetrics() : null); + IFilter filter = filterAndSummary.left; + IndexSummaryComponent summaryComponent = filterAndSummary.right; + + if (summaryComponent != null) + { + builder.setFirst(summaryComponent.first); + builder.setLast(summaryComponent.last); + builder.setIndexSummary(summaryComponent.indexSummary); + + if (online) + summaryComponent.save(descriptor.fileFor(Components.SUMMARY), false); + } + + if (filter != null) + { + builder.setFilter(filter); + + if (online) + FilterComponent.save(filter, descriptor, false); + } + } + } + + try (CompressionMetadata compressionMetadata = CompressionInfoComponent.maybeLoad(descriptor, components)) + { + builder.setDataFile(dataFileBuilder(builder.getStatsMetadata()).withCompressionMetadata(compressionMetadata).complete()); + } + + if (builder.getFilter() == null) + builder.setFilter(FilterFactory.AlwaysPresent); + + if (builder.getComponents().contains(Components.PRIMARY_INDEX)) + builder.setIndexFile(indexFileBuilder(builder.getIndexSummary()).complete()); + } + catch (IOException | RuntimeException | Error ex) + { + Throwables.closeNonNullAndAddSuppressed(ex, builder.getDataFile(), builder.getIndexFile(), builder.getFilter(), builder.getIndexSummary()); + throw ex; + } + } + + @Override + public KeyReader buildKeyReader(TableMetrics tableMetrics) throws IOException + { + StatsComponent statsComponent = StatsComponent.load(descriptor, MetadataType.STATS, MetadataType.HEADER, MetadataType.VALIDATION); + SerializationHeader header = statsComponent.serializationHeader(tableMetadataRef.getLocal()); + try (FileHandle indexFile = indexFileBuilder(null).complete()) + { + return createKeyReader(indexFile, header, tableMetrics); + } + } + + private KeyReader createKeyReader(FileHandle indexFile, SerializationHeader serializationHeader, TableMetrics tableMetrics) throws IOException + { + checkNotNull(indexFile); + checkNotNull(serializationHeader); + + RowIndexEntry.IndexSerializer serializer = new RowIndexEntry.Serializer(descriptor.version, serializationHeader, tableMetrics); + return BigTableKeyReader.create(indexFile, serializer); + } + + /** + * Go through the index and optionally rebuild the index summary and Bloom filter. + * + * @param rebuildFilter true if Bloom filter should be rebuilt + * @param rebuildSummary true if index summary, first and last keys should be rebuilt + * @return a pair of created filter and index summary component (or nulls if some of them were not created) + */ + @SuppressWarnings("resource") + private Pair buildSummaryAndBloomFilter(FileHandle indexFile, + SerializationHeader serializationHeader, + boolean rebuildFilter, + boolean rebuildSummary, + TableMetrics tableMetrics) throws IOException + { + checkNotNull(indexFile); + checkNotNull(serializationHeader); + + DecoratedKey first = null, key = null; + IFilter bf = null; + IndexSummary indexSummary = null; + + // we read the positions in a BRAF, so we don't have to worry about an entry spanning a mmap boundary. + try (KeyReader keyReader = createKeyReader(indexFile, serializationHeader, tableMetrics)) + { + long estimatedRowsNumber = rebuildFilter || rebuildSummary ? estimateRowsFromIndex(indexFile) : 0; + + if (rebuildFilter) + bf = FilterFactory.getFilter(estimatedRowsNumber, tableMetadataRef.getLocal().params.bloomFilterFpChance); + + try (IndexSummaryBuilder summaryBuilder = !rebuildSummary ? null : new IndexSummaryBuilder(estimatedRowsNumber, + tableMetadataRef.getLocal().params.minIndexInterval, + Downsampling.BASE_SAMPLING_LEVEL)) + { + while (!keyReader.isExhausted()) + { + key = tableMetadataRef.getLocal().partitioner.decorateKey(keyReader.key()); + if (rebuildSummary) + { + if (first == null) + first = key; + summaryBuilder.maybeAddEntry(key, keyReader.keyPositionForSecondaryIndex()); + } + + if (rebuildFilter) + bf.add(key); + + keyReader.advance(); + } + + if (rebuildSummary) + indexSummary = summaryBuilder.build(tableMetadataRef.getLocal().partitioner); + } + } + catch (IOException | RuntimeException | Error ex) + { + Throwables.closeNonNullAndAddSuppressed(ex, indexSummary, bf); + throw ex; + } + + assert rebuildSummary || indexSummary == null; + return Pair.create(bf, rebuildSummary ? new IndexSummaryComponent(indexSummary, first, key) : null); + } + + private IFilter loadFilter(ValidationMetadata validationMetadata) + { + return FilterComponent.maybeLoadBloomFilter(descriptor, + components, + tableMetadataRef.getLocal(), + validationMetadata); + } + + /** + * Load index summary, first key and last key from Summary.db file if it exists. + *

+ * if loaded index summary has different index interval from current value stored in schema, + * then Summary.db file will be deleted and need to be rebuilt. + */ + private IndexSummaryComponent loadSummary() + { + IndexSummaryComponent summaryComponent = null; + try + { + if (components.contains(Components.SUMMARY)) + summaryComponent = IndexSummaryComponent.loadOrDeleteCorrupted(descriptor.fileFor(Components.SUMMARY), tableMetadataRef.get()); + + if (summaryComponent == null) + logger.debug("Index summary file is missing: {}", descriptor.fileFor(Components.SUMMARY)); + } + catch (IOException ex) + { + logger.debug("Index summary file is corrupted: " + descriptor.fileFor(Components.SUMMARY), ex); + } + + return summaryComponent; + } + + /** + * @return An estimate of the number of keys contained in the given index file. + */ + public long estimateRowsFromIndex(FileHandle indexFile) throws IOException + { + checkNotNull(indexFile); + + try (RandomAccessReader indexReader = indexFile.createReader()) + { + // collect sizes for the first 10000 keys, or first 10 mebibytes of data + final int samplesCap = 10000; + final int bytesCap = (int) Math.min(10000000, indexReader.length()); + int keys = 0; + while (indexReader.getFilePointer() < bytesCap && keys < samplesCap) + { + ByteBufferUtil.skipShortLength(indexReader); + RowIndexEntry.Serializer.skip(indexReader, descriptor.version); + keys++; + } + assert keys > 0 && indexReader.getFilePointer() > 0 && indexReader.length() > 0 : "Unexpected empty index file: " + indexReader; + long estimatedRows = indexReader.length() / (indexReader.getFilePointer() / keys); + indexReader.seek(0); + return estimatedRows; + } + } + + private FileHandle.Builder dataFileBuilder(StatsMetadata statsMetadata) + { + assert this.dataFileBuilder == null || this.dataFileBuilder.file.equals(descriptor.fileFor(Components.DATA)); + + logger.info("Opening {} ({})", descriptor, FBUtilities.prettyPrintMemory(descriptor.fileFor(Components.DATA).length())); + + long recordSize = statsMetadata.estimatedPartitionSize.percentile(ioOptions.diskOptimizationEstimatePercentile); + int bufferSize = ioOptions.diskOptimizationStrategy.bufferSize(recordSize); + + if (dataFileBuilder == null) + dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.DATA)); + + dataFileBuilder.bufferSize(bufferSize); + dataFileBuilder.withChunkCache(chunkCache); + dataFileBuilder.mmapped(ioOptions.defaultDiskAccessMode); + + return dataFileBuilder; + } + + private FileHandle.Builder indexFileBuilder(IndexSummary indexSummary) + { + assert this.indexFileBuilder == null || this.indexFileBuilder.file.equals(descriptor.fileFor(Components.PRIMARY_INDEX)); + + long indexFileLength = descriptor.fileFor(Components.PRIMARY_INDEX).length(); + OptionalInt indexBufferSize = indexSummary != null ? OptionalInt.of(ioOptions.diskOptimizationStrategy.bufferSize(indexFileLength / indexSummary.size())) + : OptionalInt.empty(); + + if (indexFileBuilder == null) + indexFileBuilder = IndexComponent.fileBuilder(descriptor.fileFor(Components.PRIMARY_INDEX), ioOptions, chunkCache) + .bufferSize(indexBufferSize.orElse(DiskOptimizationStrategy.MAX_BUFFER_SIZE)); + + indexBufferSize.ifPresent(indexFileBuilder::bufferSize); + + return indexFileBuilder; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java new file mode 100644 index 0000000000..04058d7af2 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; +import java.nio.ByteBuffer; +import javax.annotation.concurrent.NotThreadSafe; + +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry.IndexSerializer; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.ByteBufferUtil; + +@NotThreadSafe +public class BigTableKeyReader implements KeyReader +{ + private final FileHandle indexFile; + private final RandomAccessReader indexFileReader; + private final IndexSerializer rowIndexEntrySerializer; + private final long initialPosition; + + private ByteBuffer key; + private long dataPosition; + private long keyPosition; + + private BigTableKeyReader(FileHandle indexFile, + RandomAccessReader indexFileReader, + IndexSerializer rowIndexEntrySerializer) + { + this.indexFile = indexFile; + this.indexFileReader = indexFileReader; + this.rowIndexEntrySerializer = rowIndexEntrySerializer; + this.initialPosition = indexFileReader.getFilePointer(); + } + + public static BigTableKeyReader create(RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException + { + BigTableKeyReader iterator = new BigTableKeyReader(null, indexFileReader, serializer); + try + { + iterator.advance(); + return iterator; + } + catch (IOException | RuntimeException ex) + { + iterator.close(); + throw ex; + } + } + + @SuppressWarnings({ "resource" }) + public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer) throws IOException + { + FileHandle iFile = null; + RandomAccessReader reader = null; + BigTableKeyReader iterator = null; + try + { + iFile = indexFile.sharedCopy(); + reader = iFile.createReader(); + iterator = new BigTableKeyReader(iFile, reader, serializer); + iterator.advance(); + return iterator; + } + catch (IOException | RuntimeException ex) + { + if (iterator != null) + { + iterator.close(); + } + else + { + FileUtils.closeQuietly(reader); + FileUtils.closeQuietly(iFile); + } + throw ex; + } + } + + @Override + public void close() + { + key = null; + dataPosition = -1; + keyPosition = -1; + FileUtils.closeQuietly(indexFileReader); + FileUtils.closeQuietly(indexFile); + } + + @Override + public boolean advance() throws IOException + { + if (!indexFileReader.isEOF()) + { + keyPosition = indexFileReader.getFilePointer(); + key = ByteBufferUtil.readWithShortLength(indexFileReader); + dataPosition = rowIndexEntrySerializer.deserializePositionAndSkip(indexFileReader); + return true; + } + else + { + keyPosition = -1; + dataPosition = -1; + key = null; + return false; + } + } + + @Override + public boolean isExhausted() + { + return key == null && dataPosition < 0; + } + + @Override + public ByteBuffer key() + { + return key; + } + + @Override + public long keyPositionForSecondaryIndex() + { + return keyPosition; + } + + @Override + public long dataPosition() + { + return dataPosition; + } + + public long indexPosition() + { + return indexFileReader.getFilePointer(); + } + + public void indexPosition(long position) throws IOException + { + if (position > indexLength()) + throw new IndexOutOfBoundsException("The requested position exceeds the index length"); + indexFileReader.seek(position); + key = null; + keyPosition = 0; + dataPosition = 0; + advance(); + } + + public long indexLength() + { + return indexFileReader.length(); + } + + @Override + public void reset() throws IOException + { + indexFileReader.seek(initialPosition); + key = null; + keyPosition = 0; + dataPosition = 0; + advance(); + } + + @Override + public String toString() + { + return String.format("BigTable-PartitionIndexIterator(%s)", indexFile.path()); + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java index e0edd7af06..34126ce177 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java @@ -19,42 +19,110 @@ package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; -import org.apache.cassandra.io.sstable.format.SSTableReaderBuilder; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.SSTableIterator; -import org.apache.cassandra.db.columniterator.SSTableReversedIterator; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIteratorWithLowerBound; import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.Downsampling; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.IVerifier; +import org.apache.cassandra.io.sstable.IndexInfo; +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener.SelectionReason; +import org.apache.cassandra.io.sstable.SSTableReadsListener.SkippingReason; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SelectionReason; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener.SkippingReason; +import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport; +import org.apache.cassandra.io.sstable.keycache.KeyCache; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.OutputHandler; + +import static org.apache.cassandra.utils.concurrent.SharedCloseable.sharedCopyOrNull; /** * SSTableReaders are open()ed by Keyspace.onStart; after that they are created by SSTableWriter.renameAndOpen. * Do not re-call open() on existing SSTable files; use the references kept by ColumnFamilyStore post-start instead. */ -public class BigTableReader extends SSTableReader +public class BigTableReader extends SSTableReaderWithFilter implements IndexSummarySupport, + KeyCacheSupport { private static final Logger logger = LoggerFactory.getLogger(BigTableReader.class); - BigTableReader(SSTableReaderBuilder builder) + private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + private final IndexSummary indexSummary; + private final FileHandle ifile; + + private final KeyCache keyCache; + + public BigTableReader(Builder builder, SSTable.Owner owner) { - super(builder); + super(builder, owner); + this.ifile = builder.getIndexFile(); + this.indexSummary = builder.getIndexSummary(); + this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(descriptor.version, header, owner != null ? owner.getMetrics() : null); + this.keyCache = Objects.requireNonNull(builder.getKeyCache()); + } + + @Override + protected List setupInstance(boolean trackHotness) + { + ArrayList closeables = Lists.newArrayList(indexSummary, ifile); + closeables.addAll(super.setupInstance(trackHotness)); + return closeables; + } + + @Override + public void releaseInMemoryComponents() + { + closeInternalComponent(indexSummary); + assert indexSummary.isCleanedUp(); + } + + @Override + public IndexSummary getIndexSummary() + { + return indexSummary; } public UnfilteredRowIterator rowIterator(DecoratedKey key, @@ -63,18 +131,18 @@ public class BigTableReader extends SSTableReader boolean reversed, SSTableReadsListener listener) { - RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ, listener); + RowIndexEntry rie = getRowIndexEntry(key, SSTableReader.Operator.EQ, true, false, listener); return rowIterator(null, key, rie, slices, selectedColumns, reversed); } - @SuppressWarnings("resource") public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) { if (indexEntry == null) return UnfilteredRowIterators.noRowsIterator(metadata(), key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed); - return reversed - ? new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, ifile) - : new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, ifile); + else if (reversed) + return new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, ifile); + else + return new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, ifile); } @Override @@ -83,6 +151,12 @@ public class BigTableReader extends SSTableReader return BigTableScanner.getScanner(this, columns, dataRange, listener); } + @Override + public KeyReader keyReader() throws IOException + { + return BigTableKeyReader.create(ifile, rowIndexEntrySerializer); + } + /** * Direct I/O SSTableScanner over an iterator of bounds. * @@ -118,97 +192,134 @@ public class BigTableReader extends SSTableReader return getScanner(); } - - @SuppressWarnings("resource") // caller to close + /** + * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. + */ @Override - public UnfilteredRowIterator simpleIterator(FileDataInput dfile, DecoratedKey key, RowIndexEntry position, boolean tombstoneOnly) + public DecoratedKey firstKeyBeyond(PartitionPosition token) { - return SSTableIdentityIterator.create(this, dfile, position, key, tombstoneOnly); + if (token.compareTo(first) < 0) + return first; + + long sampledPosition = getIndexScanPosition(token); + + if (ifile == null) + return null; + + String path = null; + try (FileDataInput in = ifile.createReader(sampledPosition)) + { + path = in.getPath(); + while (!in.isEOF()) + { + ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(in); + DecoratedKey indexDecoratedKey = decorateKey(indexKey); + if (indexDecoratedKey.compareTo(token) > 0) + return indexDecoratedKey; + + RowIndexEntry.Serializer.skip(in, descriptor.version); + } + } + catch (IOException e) + { + markSuspect(); + throw new CorruptSSTableException(e, path); + } + + return null; } /** + * Retrieves the position while updating the key cache and the stats. + * * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to - * allow key selection by token bounds but only if op != * EQ - * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. - * @param updateCacheAndStats true if updating stats and cache + * allow key selection by token bounds but only if op != * EQ + * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. + */ + public final RowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op) + { + return getRowIndexEntry(key, op, true, false, SSTableReadsListener.NOOP_LISTENER); + } + + /** + * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to + * allow key selection by token bounds but only if op != * EQ + * @param operator The Operator defining matching keys: the nearest key to the target matching the operator wins. + * @param updateStats true if updating stats and cache * @return The index entry corresponding to the key, or null if the key is not present */ - protected RowIndexEntry getPosition(PartitionPosition key, - Operator op, - boolean updateCacheAndStats, - boolean permitMatchPastLast, - SSTableReadsListener listener) + @Override + public RowIndexEntry getRowIndexEntry(PartitionPosition key, + Operator operator, + boolean updateStats, + boolean permitMatchPastLast, + SSTableReadsListener listener) { // Having no index file is impossible in a normal operation. The only way it might happen is running - // Scrubber that does not really rely onto this method. + // Scrubber that does not really rely on this method. if (ifile == null) - { return null; - } - if (op == Operator.EQ) - { - assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key - if (!bf.isPresent((DecoratedKey)key)) - { - listener.onSSTableSkipped(this, SkippingReason.BLOOM_FILTER); - Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.id); - bloomFilterTracker.addTrueNegative(); - return null; - } - } - - // next, the key cache (only make sense for valid row key) - if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey)) - { - DecoratedKey decoratedKey = (DecoratedKey) key; - RowIndexEntry cachedPosition = getCachedPosition(decoratedKey, updateCacheAndStats); - if (cachedPosition != null) - { - // we do not need to track "true positive" for Bloom Filter here because it has been already tracked - // inside getCachedPosition method - listener.onSSTableSelected(this, cachedPosition, SelectionReason.KEY_CACHE_HIT); - Tracing.trace("Key cache hit for sstable {}", descriptor.id); - return cachedPosition; - } - } + Operator searchOp = operator; // check the smallest and greatest keys in the sstable to see if it can't be present boolean skip = false; if (key.compareTo(first) < 0) { - if (op == Operator.EQ) + if (searchOp == Operator.EQ) + { skip = true; + } else + { key = first; - - op = Operator.EQ; + searchOp = Operator.GE; // since op != EQ, bloom filter will be skipped; first key is included so no reason to check bloom filter + } } else { int l = last.compareTo(key); - // l <= 0 => we may be looking past the end of the file; we then narrow our behaviour to: - // 1) skipping if strictly greater for GE and EQ; - // 2) skipping if equal and searching GT, and we aren't permitting matching past last - skip = l <= 0 && (l < 0 || (!permitMatchPastLast && op == Operator.GT)); + skip = l < 0 // out of range, skip + || l == 0 && searchOp == Operator.GT && !permitMatchPastLast; // search entry > key, but key is the last in range, so if permitMatchPastLast == false skip + if (l == 0) + searchOp = Operator.GE; // since op != EQ, bloom filter will be skipped, last key is included so no reason to check bloom filter } if (skip) { - if (op == Operator.EQ && updateCacheAndStats) - bloomFilterTracker.addFalsePositive(); - listener.onSSTableSkipped(this, SkippingReason.MIN_MAX_KEYS); - Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.id); + notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, operator, updateStats); return null; } + if (searchOp == Operator.EQ) + { + assert key instanceof DecoratedKey; // EQ only make sense if the key is a valid row key + if (!isPresentInFilter((IFilter.FilterKey) key)) + { + notifySkipped(SkippingReason.BLOOM_FILTER, listener, operator, updateStats); + return null; + } + } + + // next, the key cache (only make sense for valid row key) + if ((searchOp == Operator.EQ || searchOp == Operator.GE) && (key instanceof DecoratedKey)) + { + DecoratedKey decoratedKey = (DecoratedKey) key; + AbstractRowIndexEntry cachedPosition = getCachedPosition(decoratedKey, updateStats); + if (cachedPosition != null && cachedPosition.getSSTableFormat() == descriptor.getFormat()) + { + notifySelected(SelectionReason.KEY_CACHE_HIT, listener, operator, updateStats, cachedPosition); + return (RowIndexEntry) cachedPosition; + } + } + int binarySearchResult = indexSummary.binarySearch(key); - long sampledPosition = getIndexScanPositionFromBinarySearchResult(binarySearchResult, indexSummary); - int sampledIndex = getIndexSummaryIndexFromBinarySearchResult(binarySearchResult); + long sampledPosition = indexSummary.getScanPositionFromBinarySearchResult(binarySearchResult); + int sampledIndex = IndexSummary.getIndexFromBinarySearchResult(binarySearchResult); int effectiveInterval = indexSummary.getEffectiveIndexIntervalAfterIndex(sampledIndex); // scan the on-disk index, starting at the nearest sampled position. - // The check against IndexInterval is to be exit the loop in the EQ case when the key looked for is not present + // The check against IndexInterval is to be exited the loop in the EQ case when the key looked for is not present // (bloom filter false positive). But note that for non-EQ cases, we might need to check the first key of the // next index position because the searched key can be greater the last key of the index interval checked if it // is lesser than the first key of next interval (and in that case we must return the position of the first key @@ -228,7 +339,7 @@ public class BigTableReader extends SSTableReader boolean exactMatch; // is the current position an exact match for the key, suitable for caching // Compare raw keys if possible for performance, otherwise compare decorated keys. - if (op == Operator.EQ && i <= effectiveInterval) + if (searchOp == Operator.EQ && i <= effectiveInterval) { opSatisfied = exactMatch = indexKey.equals(((DecoratedKey) key).getKey()); } @@ -236,15 +347,12 @@ public class BigTableReader extends SSTableReader { DecoratedKey indexDecoratedKey = decorateKey(indexKey); int comparison = indexDecoratedKey.compareTo(key); - int v = op.apply(comparison); + int v = searchOp.apply(comparison); opSatisfied = (v == 0); exactMatch = (comparison == 0); if (v < 0) { - if (op == SSTableReader.Operator.EQ && updateCacheAndStats) - bloomFilterTracker.addFalsePositive(); - listener.onSSTableSkipped(this, SkippingReason.PARTITION_INDEX_LOOKUP); - Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.id); + notifySkipped(SkippingReason.PARTITION_INDEX_LOOKUP, listener, operator, updateStats); return null; } } @@ -253,10 +361,10 @@ public class BigTableReader extends SSTableReader { // read data position from index entry RowIndexEntry indexEntry = rowIndexEntrySerializer.deserialize(in); - if (exactMatch && updateCacheAndStats) + if (exactMatch && updateStats) { assert key instanceof DecoratedKey; // key can be == to the index key only if it's a true row key - DecoratedKey decoratedKey = (DecoratedKey)key; + DecoratedKey decoratedKey = (DecoratedKey) key; if (logger.isTraceEnabled()) { @@ -272,10 +380,7 @@ public class BigTableReader extends SSTableReader // store exact match for the key cacheKey(decoratedKey, indexEntry); } - if (op == Operator.EQ && updateCacheAndStats) - bloomFilterTracker.addTruePositive(); - listener.onSSTableSelected(this, indexEntry, SelectionReason.INDEX_ENTRY_FOUND); - Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndexCount(), descriptor.id); + notifySelected(SelectionReason.INDEX_ENTRY_FOUND, listener, operator, updateStats, indexEntry); return indexEntry; } @@ -288,12 +393,355 @@ public class BigTableReader extends SSTableReader throw new CorruptSSTableException(e, path); } - if (op == SSTableReader.Operator.EQ && updateCacheAndStats) - bloomFilterTracker.addFalsePositive(); - listener.onSSTableSkipped(this, SkippingReason.INDEX_ENTRY_NOT_FOUND); - Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.id); + notifySkipped(SkippingReason.INDEX_ENTRY_NOT_FOUND, listener, operator, updateStats); return null; } + /** + * @param key The key to apply as the rhs to the given Operator. A 'fake' key is allowed to + * allow key selection by token bounds but only if op != * EQ + * @param op The Operator defining matching keys: the nearest key to the target matching the operator wins. + * @param updateCacheAndStats true if updating stats and cache + * @return The index entry corresponding to the key, or null if the key is not present + */ + @Override + protected long getPosition(PartitionPosition key, + Operator op, + boolean updateCacheAndStats, + boolean permitMatchPastLast, + SSTableReadsListener listener) + { + RowIndexEntry rowIndexEntry = getRowIndexEntry(key, op, updateCacheAndStats, permitMatchPastLast, listener); + return rowIndexEntry != null ? rowIndexEntry.position : -1; + } + @Override + public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException + { + DecoratedKey key; + try (FileDataInput in = ifile.createReader(keyPositionFromSecondaryIndex)) + { + if (in.isEOF()) + return null; + + key = decorateKey(ByteBufferUtil.readWithShortLength(in)); + + // hint read path about key location if caching is enabled + // this saves index summary lookup and index file iteration which whould be pretty costly + // especially in presence of promoted column indexes + cacheKey(key, rowIndexEntrySerializer.deserialize(in)); + } + + return key; + } + + @Override + public RowIndexEntry deserializeKeyCacheValue(DataInputPlus input) throws IOException + { + return rowIndexEntrySerializer.deserializeForCache(input); + } + + @Override + public ClusteringBound getLowerBoundPrefixFromCache(DecoratedKey partitionKey, boolean isReversed) + { + AbstractRowIndexEntry rie = getCachedPosition(partitionKey, false); + if (!(rie instanceof RowIndexEntry)) + return null; + + RowIndexEntry rowIndexEntry = (RowIndexEntry) rie; + if (!rowIndexEntry.indexOnHeap()) + return null; + + try (RowIndexEntry.IndexInfoRetriever onHeapRetriever = rowIndexEntry.openWithIndex(null)) + { + IndexInfo columns = onHeapRetriever.columnsIndex(isReversed ? rowIndexEntry.columnsIndexCount() - 1 : 0); + ClusteringBound bound = isReversed ? columns.lastName.asEndBound() : columns.firstName.asStartBound(); + UnfilteredRowIteratorWithLowerBound.assertBoundSize(bound, this); + return bound.artificialLowerBound(isReversed); + } + catch (IOException e) + { + throw new RuntimeException("should never occur", e); + } + } + + /** + * @return An estimate of the number of keys in this SSTable based on the index summary. + */ + @Override + public long estimatedKeys() + { + return indexSummary.getEstimatedKeyCount(); + } + + /** + * @return An estimate of the number of keys for given ranges in this SSTable. + */ + @Override + public long estimatedKeysForRanges(Collection> ranges) + { + long sampleKeyCount = 0; + List sampleIndexes = indexSummary.getSampleIndexesForRanges(ranges); + for (IndexesBounds sampleIndexRange : sampleIndexes) + sampleKeyCount += (sampleIndexRange.upperPosition - sampleIndexRange.lowerPosition + 1); + + // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling + long estimatedKeys = sampleKeyCount * ((long) Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel(); + return Math.max(1, estimatedKeys); + } + + /** + * Returns whether the number of entries in the IndexSummary > 2. At full sampling, this is approximately + * 1/INDEX_INTERVALth of the keys in this SSTable. + */ + @Override + public boolean isEstimationInformative() + { + return indexSummary.size() > 2; + } + + @Override + public Iterable getKeySamples(final Range range) + { + return Iterables.transform(indexSummary.getKeySamples(range), bytes -> decorateKey(ByteBuffer.wrap(bytes))); + } + + public RandomAccessReader openIndexReader() + { + if (ifile != null) + return ifile.createReader(); + return null; + } + + public FileHandle getIndexFile() + { + return ifile; + } + + @Override + public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options) + { + Preconditions.checkArgument(cfs.metadata().equals(metadata())); + return new BigTableVerifier(cfs, this, outputHandler, isOffline, options); + } + + /** + * Gets the position with the index file to start scanning to find the given key (at most indexInterval keys away, + * modulo downsampling of the index summary). Always returns a {@code value >= 0} + */ + long getIndexScanPosition(PartitionPosition key) + { + if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0) + key = first; + + return indexSummary.getScanPosition(key); + } + + protected final Builder unbuildTo(Builder builder, boolean sharedCopy) + { + Builder b = super.unbuildTo(builder, sharedCopy); + if (builder.getIndexFile() == null) + b.setIndexFile(sharedCopy ? sharedCopyOrNull(ifile) : ifile); + if (builder.getIndexSummary() == null) + b.setIndexSummary(sharedCopy ? sharedCopyOrNull(indexSummary) : indexSummary); + + b.setKeyCache(keyCache); + + return b; + } + + @VisibleForTesting + @Override + public SSTableReaderWithFilter cloneAndReplace(IFilter filter) + { + return unbuildTo(new Builder(descriptor).setFilter(filter), true).build(owner().orElse(null), true, true); + } + + /** + * Clone this reader with the provided start and open reason, and set the clone as replacement. + * + * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive + * opening of compaction results). + * @param reason the {@code OpenReason} for the replacement. + * @return the cloned reader. That reader is set as a replacement by the method. + */ + private SSTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason) + { + return unbuildTo(new Builder(descriptor), true) + .setFirst(newFirst) + .setOpenReason(reason) + .build(owner().orElse(null), true, true); + } + + /** + * Clone this reader with the new values and set the clone as replacement. + * + * @param newFirst the first key for the replacement (which can be different from the original due to the pre-emptive + * opening of compaction results). + * @param reason the {@code OpenReason} for the replacement. + * @param newSummary the index summary for the replacement. + * @return the cloned reader. That reader is set as a replacement by the method. + */ + private BigTableReader cloneAndReplace(DecoratedKey newFirst, OpenReason reason, IndexSummary newSummary) + { + return unbuildTo(new Builder(descriptor).setIndexSummary(newSummary), true) + .setIndexSummary(newSummary) + .setFirst(newFirst) + .setOpenReason(reason) + .build(owner().orElse(null), true, true); + } + + public SSTableReader cloneWithRestoredStart(DecoratedKey restoredStart) + { + return runWithLock(ignored -> cloneAndReplace(restoredStart, OpenReason.NORMAL)); + } + + public SSTableReader cloneWithNewStart(DecoratedKey newStart) + { + return runWithLock(ignored -> { + assert openReason != OpenReason.EARLY; + // TODO: merge with caller's firstKeyBeyond() work,to save time + if (newStart.compareTo(first) > 0) + { + Map handleAndPositions = new LinkedHashMap<>(2); + if (dfile != null) + handleAndPositions.put(dfile, getPosition(newStart, Operator.EQ)); + if (ifile != null) + handleAndPositions.put(ifile, getIndexScanPosition(newStart)); + runOnClose(() -> handleAndPositions.forEach(FileHandle::dropPageCache)); + } + + return cloneAndReplace(newStart, OpenReason.MOVED_START); + }); + } + + /** + * Returns a new SSTableReader with the same properties as this SSTableReader except that a new IndexSummary will + * be built at the target samplingLevel. This (original) SSTableReader instance will be marked as replaced, have + * its DeletingTask removed, and have its periodic read-meter sync task cancelled. + * + * @param samplingLevel the desired sampling level for the index summary on the new SSTableReader + * @return a new SSTableReader + */ + @SuppressWarnings("resource") + public BigTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException + { + assert openReason != OpenReason.EARLY; + + int minIndexInterval = metadata().params.minIndexInterval; + int maxIndexInterval = metadata().params.maxIndexInterval; + double effectiveInterval = indexSummary.getEffectiveIndexInterval(); + + IndexSummary newSummary; + + // We have to rebuild the summary from the on-disk primary index in three cases: + // 1. The sampling level went up, so we need to read more entries off disk + // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary + // at full sampling (and consequently at any other sampling level) + // 3. The max_index_interval was lowered, forcing us to raise the sampling level + if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval) + { + newSummary = buildSummaryAtLevel(samplingLevel); + } + else if (samplingLevel < indexSummary.getSamplingLevel()) + { + // we can use the existing index summary to make a smaller one + newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, getPartitioner()); + } + else + { + throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " + + "no adjustments to min/max_index_interval"); + } + + // Always save the resampled index with lock to avoid racing with entire-sstable streaming + return runWithLock(ignored -> { + new IndexSummaryComponent(newSummary, first, last).save(descriptor.fileFor(Components.SUMMARY), true); + return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary); + }); + } + + private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException + { + // we read the positions in a BRAF, so we don't have to worry about an entry spanning a mmap boundary. + RandomAccessReader primaryIndex = RandomAccessReader.open(descriptor.fileFor(Components.PRIMARY_INDEX)); + try + { + long indexSize = primaryIndex.length(); + try (IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata().params.minIndexInterval, newSamplingLevel)) + { + long indexPosition; + while ((indexPosition = primaryIndex.getFilePointer()) != indexSize) + { + summaryBuilder.maybeAddEntry(decorateKey(ByteBufferUtil.readWithShortLength(primaryIndex)), indexPosition); + RowIndexEntry.Serializer.skip(primaryIndex, descriptor.version); + } + + return summaryBuilder.build(getPartitioner()); + } + } + finally + { + FileUtils.closeQuietly(primaryIndex); + } + } + + @Override + public KeyCache getKeyCache() + { + return this.keyCache; + } + + public static class Builder extends SSTableReaderWithFilter.Builder + { + private static final Logger logger = LoggerFactory.getLogger(Builder.class); + + private IndexSummary indexSummary; + private FileHandle indexFile; + private KeyCache keyCache = KeyCache.NO_CACHE; + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + public Builder setIndexFile(FileHandle indexFile) + { + this.indexFile = indexFile; + return this; + } + + public Builder setIndexSummary(IndexSummary indexSummary) + { + this.indexSummary = indexSummary; + return this; + } + + public Builder setKeyCache(KeyCache keyCache) + { + this.keyCache = keyCache; + return this; + } + + public IndexSummary getIndexSummary() + { + return indexSummary; + } + + public FileHandle getIndexFile() + { + return indexFile; + } + + public KeyCache getKeyCache() + { + return keyCache; + } + + @Override + protected BigTableReader buildInternal(Owner owner) + { + return new BigTableReader(this, owner); + } + } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java index 235b9b1a71..1de9a33645 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java @@ -18,31 +18,39 @@ package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.utils.AbstractIterator; - import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterators; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.rows.LazilyInitializedUnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.AbstractBounds.Boundary; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.EmptySSTableScanner; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableIdentityIterator; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.dht.AbstractBounds.isEmpty; @@ -54,7 +62,7 @@ public class BigTableScanner implements ISSTableScanner private final AtomicBoolean isClosed = new AtomicBoolean(false); protected final RandomAccessReader dfile; protected final RandomAccessReader ifile; - public final SSTableReader sstable; + public final BigTableReader sstable; private final Iterator> rangeIterator; private AbstractBounds currentRange; @@ -69,12 +77,12 @@ public class BigTableScanner implements ISSTableScanner protected Iterator iterator; // Full scan of the sstables - public static ISSTableScanner getScanner(SSTableReader sstable) + public static ISSTableScanner getScanner(BigTableReader sstable) { return getScanner(sstable, Iterators.singletonIterator(fullRange(sstable))); } - public static ISSTableScanner getScanner(SSTableReader sstable, + public static ISSTableScanner getScanner(BigTableReader sstable, ColumnFilter columns, DataRange dataRange, SSTableReadsListener listener) @@ -82,7 +90,7 @@ public class BigTableScanner implements ISSTableScanner return new BigTableScanner(sstable, columns, dataRange, makeBounds(sstable, dataRange).iterator(), listener); } - public static ISSTableScanner getScanner(SSTableReader sstable, Collection> tokenRanges) + public static ISSTableScanner getScanner(BigTableReader sstable, Collection> tokenRanges) { // We want to avoid allocating a SSTableScanner if the range don't overlap the sstable (#5249) List positions = sstable.getPositionsForRanges(tokenRanges); @@ -92,12 +100,12 @@ public class BigTableScanner implements ISSTableScanner return getScanner(sstable, makeBounds(sstable, tokenRanges).iterator()); } - public static ISSTableScanner getScanner(SSTableReader sstable, Iterator> rangeIterator) + public static ISSTableScanner getScanner(BigTableReader sstable, Iterator> rangeIterator) { return new BigTableScanner(sstable, ColumnFilter.all(sstable.metadata()), null, rangeIterator, SSTableReadsListener.NOOP_LISTENER); } - private BigTableScanner(SSTableReader sstable, + private BigTableScanner(BigTableReader sstable, ColumnFilter columns, DataRange dataRange, Iterator> rangeIterator, @@ -110,9 +118,7 @@ public class BigTableScanner implements ISSTableScanner this.sstable = sstable; this.columns = columns; this.dataRange = dataRange; - this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata(), - sstable.descriptor.version, - sstable.header); + this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, sstable.owner().map(SSTable.Owner::getMetrics).orElse(null)); this.rangeIterator = rangeIterator; this.listener = listener; } @@ -139,7 +145,7 @@ public class BigTableScanner implements ISSTableScanner private static void addRange(SSTableReader sstable, AbstractBounds requested, List> boundsList) { - if (requested instanceof Range && ((Range)requested).isWrapAround()) + if (requested instanceof Range && ((Range) requested).isWrapAround()) { if (requested.right.compareTo(sstable.first) >= 0) { @@ -171,7 +177,7 @@ public class BigTableScanner implements ISSTableScanner left = maxLeft(left, sstable.first, true); // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping right = requested.right.isMinimum() ? new Boundary(sstable.last, true) - : minRight(right, sstable.last, true); + : minRight(right, sstable.last, true); if (!isEmpty(left, right)) boundsList.add(AbstractBounds.bounds(left, right)); } @@ -392,59 +398,4 @@ public class BigTableScanner implements ISSTableScanner " sstable=" + sstable + ")"; } - - public static class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator implements ISSTableScanner - { - private final SSTableReader sstable; - - public EmptySSTableScanner(SSTableReader sstable) - { - this.sstable = sstable; - } - - public long getLengthInBytes() - { - return 0; - } - - public long getCurrentPosition() - { - return 0; - } - - public long getBytesScanned() - { - return 0; - } - - public long getCompressedLengthInBytes() - { - return 0; - } - - public Set getBackingSSTables() - { - return ImmutableSet.of(sstable); - } - - public TableMetadata metadata() - { - return sstable.metadata(); - } - - public boolean hasNext() - { - return false; - } - - public UnfilteredRowIterator next() - { - return null; - } - - public int getMinLocalDeletionTime() - { - return DeletionTime.LIVE.localDeletionTime(); - } - } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java new file mode 100644 index 0000000000..dc991f491f --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScrubber.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOError; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.compaction.CompactionInterruptedException; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.SSTableRewriter; +import org.apache.cassandra.io.sstable.format.SortedTableScrubber; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.OutputHandler; + +public class BigTableScrubber extends SortedTableScrubber implements IScrubber +{ + private final boolean isIndex; + + private final RandomAccessReader indexFile; + private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + + private ByteBuffer currentIndexKey; + private ByteBuffer nextIndexKey; + private long currentPartitionPositionFromIndex; + private long nextPartitionPositionFromIndex; + + public BigTableScrubber(ColumnFamilyStore cfs, + LifecycleTransaction transaction, + OutputHandler outputHandler, + Options options) + { + super(cfs, transaction, outputHandler, options); + + this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, cfs.getMetrics()); + + boolean hasIndexFile = sstable.descriptor.fileFor(Components.PRIMARY_INDEX).exists(); + this.isIndex = cfs.isIndex(); + if (!hasIndexFile) + { + // if there's any corruption in the -Data.db then partitions can't be skipped over. but it's worth a shot. + outputHandler.warn("Missing component: %s", sstable.descriptor.fileFor(Components.PRIMARY_INDEX)); + } + + this.indexFile = hasIndexFile + ? RandomAccessReader.open(sstable.descriptor.fileFor(Components.PRIMARY_INDEX)) + : null; + + this.currentPartitionPositionFromIndex = 0; + this.nextPartitionPositionFromIndex = 0; + } + + @Override + protected UnfilteredRowIterator withValidation(UnfilteredRowIterator iter, String filename) + { + return options.checkData && !isIndex ? UnfilteredRowIterators.withValidation(iter, filename) : iter; + } + + @Override + protected void scrubInternal(SSTableRewriter writer) throws IOException + { + try + { + nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null; + if (indexAvailable()) + { + // throw away variable, so we don't have a side effect in the assertion + long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); + assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex; + } + } + catch (Throwable ex) + { + throwIfFatal(ex); + nextIndexKey = null; + nextPartitionPositionFromIndex = dataFile.length(); + if (indexFile != null) + indexFile.seek(indexFile.length()); + } + + DecoratedKey prevKey = null; + + while (!dataFile.isEOF()) + { + if (scrubInfo.isStopRequested()) + throw new CompactionInterruptedException(scrubInfo.getCompactionInfo()); + + long partitionStart = dataFile.getFilePointer(); + outputHandler.debug("Reading row at %d", partitionStart); + + DecoratedKey key = null; + try + { + ByteBuffer raw = ByteBufferUtil.readWithShortLength(dataFile); + if (!cfs.metadata.getLocal().isIndex()) + cfs.metadata.getLocal().partitionKeyType.validate(raw); + key = sstable.decorateKey(raw); + } + catch (Throwable th) + { + throwIfFatal(th); + // check for null key below + } + + long dataStartFromIndex = -1; + long dataSizeFromIndex = -1; + updateIndexKey(); + + if (indexAvailable()) + { + if (currentIndexKey != null) + { + dataStartFromIndex = currentPartitionPositionFromIndex + 2 + currentIndexKey.remaining(); + dataSizeFromIndex = nextPartitionPositionFromIndex - dataStartFromIndex; + } + } + + long dataStart = dataFile.getFilePointer(); + + String keyName = key == null ? "(unreadable key)" : keyString(key); + outputHandler.debug("partition %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSizeFromIndex)); + assert currentIndexKey != null || !indexAvailable(); + + try + { + if (key == null) + throw new IOError(new IOException("Unable to read partition key from data file")); + + if (currentIndexKey != null && !key.getKey().equals(currentIndexKey)) + { + throw new IOError(new IOException(String.format("Key from data file (%s) does not match key from index file (%s)", + //ByteBufferUtil.bytesToHex(key.getKey()), ByteBufferUtil.bytesToHex(currentIndexKey)))); + "_too big_", ByteBufferUtil.bytesToHex(currentIndexKey)))); + } + + if (indexFile != null && dataSizeFromIndex > dataFile.length()) + throw new IOError(new IOException("Impossible partition size (greater than file length): " + dataSizeFromIndex)); + + if (indexFile != null && dataStart != dataStartFromIndex) + outputHandler.warn("Data file partition position %d differs from index file row position %d", dataStart, dataStartFromIndex); + + if (tryAppend(prevKey, key, writer)) + prevKey = key; + } + catch (Throwable th) + { + throwIfFatal(th); + outputHandler.warn(th, "Error reading partition %s (stacktrace follows):", keyName); + + if (currentIndexKey != null + && (key == null || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex)) + { + + outputHandler.output("Retrying from partition index; data is %s bytes starting at %s", + dataSizeFromIndex, dataStartFromIndex); + key = sstable.decorateKey(currentIndexKey); + try + { + if (!cfs.metadata.getLocal().isIndex()) + cfs.metadata.getLocal().partitionKeyType.validate(key.getKey()); + dataFile.seek(dataStartFromIndex); + + if (tryAppend(prevKey, key, writer)) + prevKey = key; + } + catch (Throwable th2) + { + throwIfFatal(th2); + throwIfCannotContinue(key, th2); + + outputHandler.warn(th2, "Retry failed too. Skipping to next partition (retry's stacktrace follows)"); + badPartitions++; + if (!seekToNextPartition()) + break; + } + } + else + { + throwIfCannotContinue(key, th); + + outputHandler.warn("Partition starting at position %d is unreadable; skipping to next", dataStart); + badPartitions++; + if (currentIndexKey != null) + if (!seekToNextPartition()) + break; + } + } + } + } + + private void updateIndexKey() + { + currentIndexKey = nextIndexKey; + currentPartitionPositionFromIndex = nextPartitionPositionFromIndex; + try + { + nextIndexKey = !indexAvailable() ? null : ByteBufferUtil.readWithShortLength(indexFile); + + nextPartitionPositionFromIndex = !indexAvailable() + ? dataFile.length() + : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); + } + catch (Throwable th) + { + JVMStabilityInspector.inspectThrowable(th); + outputHandler.warn(th, "Error reading index file"); + nextIndexKey = null; + nextPartitionPositionFromIndex = dataFile.length(); + } + } + + private boolean indexAvailable() + { + return indexFile != null && !indexFile.isEOF(); + } + + private boolean seekToNextPartition() + { + while (nextPartitionPositionFromIndex < dataFile.length()) + { + try + { + dataFile.seek(nextPartitionPositionFromIndex); + return true; + } + catch (Throwable th) + { + throwIfFatal(th); + outputHandler.warn(th, "Failed to seek to next partition position %d", nextPartitionPositionFromIndex); + badPartitions++; + } + + updateIndexKey(); + } + + return false; + } + + @Override + protected void throwIfCannotContinue(DecoratedKey key, Throwable th) + { + if (isIndex) + { + outputHandler.warn("An error occurred while scrubbing the partition with key '%s' for an index table. " + + "Scrubbing will abort for this table and the index will be rebuilt.", keyString(key)); + throw new IOError(th); + } + + super.throwIfCannotContinue(key, th); + } + + @Override + public void close() + { + fileAccessLock.writeLock().lock(); + try + { + FileUtils.closeQuietly(dataFile); + FileUtils.closeQuietly(indexFile); + } + finally + { + fileAccessLock.writeLock().unlock(); + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java new file mode 100644 index 0000000000..bb412bfaab --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.NoSuchFileException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Throwables; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.compaction.CompactionInterruptedException; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.LocalPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.IVerifier; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.SSTableIdentityIterator; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SortedTableVerifier; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.OutputHandler; + +public class BigTableVerifier extends SortedTableVerifier implements IVerifier +{ + private final RandomAccessReader indexFile; + private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + + public BigTableVerifier(ColumnFamilyStore cfs, BigTableReader sstable, OutputHandler outputHandler, boolean isOffline, Options options) + { + super(cfs, sstable, outputHandler, isOffline, options); + + this.rowIndexEntrySerializer = new RowIndexEntry.Serializer(sstable.descriptor.version, sstable.header, sstable.owner().map(SSTable.Owner::getMetrics).orElse(null)); + this.indexFile = RandomAccessReader.open(sstable.descriptor.fileFor(Components.PRIMARY_INDEX)); + } + + @Override + public void verify() + { + verifySSTableVersion(); + + verifySSTableMetadata(); + + verifyIndex(); + + verifyIndexSummary(); + + verifyBloomFilter(); + + if (options.checkOwnsTokens && !isOffline && !(cfs.getPartitioner() instanceof LocalPartitioner)) + { + if (verifyOwnedRanges() == 0) + return; + } + + if (options.quick) + return; + + if (verifyDigest() && !options.extendedVerification) + return; + + verifySSTable(); + + outputHandler.output("Verify of %s succeeded. All %d rows read successfully", sstable, goodRows); + } + + private void verifySSTable() + { + long rowStart; + outputHandler.output("Extended Verify requested, proceeding to inspect values"); + + try (VerifyController verifyController = new VerifyController(cfs)) + { + ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); + { + long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); + if (firstRowPositionFromIndex != 0) + markAndThrow(new RuntimeException("firstRowPositionFromIndex != 0: " + firstRowPositionFromIndex)); + } + + List> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace)); + RangeOwnHelper rangeOwnHelper = new RangeOwnHelper(ownedRanges); + DecoratedKey prevKey = null; + + while (!dataFile.isEOF()) + { + + if (verifyInfo.isStopRequested()) + throw new CompactionInterruptedException(verifyInfo.getCompactionInfo()); + + rowStart = dataFile.getFilePointer(); + outputHandler.debug("Reading row at %d", rowStart); + + DecoratedKey key = null; + try + { + key = sstable.decorateKey(ByteBufferUtil.readWithShortLength(dataFile)); + } + catch (Throwable th) + { + throwIfFatal(th); + // check for null key below + } + + if (options.checkOwnsTokens && ownedRanges.size() > 0 && !(cfs.getPartitioner() instanceof LocalPartitioner)) + { + try + { + rangeOwnHelper.validate(key); + } + catch (Throwable t) + { + outputHandler.warn(t, "Key %s in sstable %s not owned by local ranges %s", key, sstable, ownedRanges); + markAndThrow(t); + } + } + + ByteBuffer currentIndexKey = nextIndexKey; + long nextRowPositionFromIndex = 0; + try + { + nextIndexKey = indexFile.isEOF() ? null : ByteBufferUtil.readWithShortLength(indexFile); + nextRowPositionFromIndex = indexFile.isEOF() + ? dataFile.length() + : rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); + } + catch (Throwable th) + { + markAndThrow(th); + } + + long dataStart = dataFile.getFilePointer(); + long dataStartFromIndex = currentIndexKey == null + ? -1 + : rowStart + 2 + currentIndexKey.remaining(); + + long dataSize = nextRowPositionFromIndex - dataStartFromIndex; + // avoid an NPE if key is null + String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey()); + outputHandler.debug("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize)); + + assert currentIndexKey != null || indexFile.isEOF(); + + try + { + if (key == null || dataSize > dataFile.length()) + markAndThrow(new RuntimeException(String.format("key = %s, dataSize=%d, dataFile.length() = %d", key, dataSize, dataFile.length()))); + + try (UnfilteredRowIterator iterator = SSTableIdentityIterator.create(sstable, dataFile, key)) + { + Row first = null; + int duplicateRows = 0; + long minTimestamp = Long.MAX_VALUE; + long maxTimestamp = Long.MIN_VALUE; + while (iterator.hasNext()) + { + Unfiltered uf = iterator.next(); + if (uf.isRow()) + { + Row row = (Row) uf; + if (first != null && first.clustering().equals(row.clustering())) + { + duplicateRows++; + for (Cell cell : row.cells()) + { + maxTimestamp = Math.max(cell.timestamp(), maxTimestamp); + minTimestamp = Math.min(cell.timestamp(), minTimestamp); + } + } + else + { + if (duplicateRows > 0) + logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp); + duplicateRows = 0; + first = row; + maxTimestamp = Long.MIN_VALUE; + minTimestamp = Long.MAX_VALUE; + } + } + } + if (duplicateRows > 0) + logDuplicates(key, first, duplicateRows, minTimestamp, maxTimestamp); + } + + if ((prevKey != null && prevKey.compareTo(key) > 0) || !key.getKey().equals(currentIndexKey) || dataStart != dataStartFromIndex) + markAndThrow(new RuntimeException("Key out of order: previous = " + prevKey + " : current = " + key)); + goodRows++; + prevKey = key; + + + outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex); + dataFile.seek(nextRowPositionFromIndex); + } + catch (Throwable th) + { + markAndThrow(th); + } + } + } + catch (Throwable t) + { + Throwables.throwIfUnchecked(t); + throw new RuntimeException(t); + } + } + + private void verifyIndexSummary() + { + try + { + outputHandler.debug("Deserializing index summary for %s", sstable); + deserializeIndexSummary(sstable); + } + catch (Throwable t) + { + outputHandler.output("Index summary is corrupt - if it is removed it will get rebuilt on startup %s", sstable.descriptor.fileFor(Components.SUMMARY)); + outputHandler.warn(t); + markAndThrow(t, false); + } + } + + private void verifyIndex() + { + try + { + outputHandler.debug("Deserializing index for %s", sstable); + deserializeIndex(sstable); + } + catch (Throwable t) + { + outputHandler.warn(t); + markAndThrow(t); + } + } + + private void logDuplicates(DecoratedKey key, Row first, int duplicateRows, long minTimestamp, long maxTimestamp) + { + String keyString = sstable.metadata().partitionKeyType.getString(key.getKey()); + long firstMaxTs = Long.MIN_VALUE; + long firstMinTs = Long.MAX_VALUE; + for (Cell cell : first.cells()) + { + firstMaxTs = Math.max(firstMaxTs, cell.timestamp()); + firstMinTs = Math.min(firstMinTs, cell.timestamp()); + } + outputHandler.output("%d duplicate rows found for [%s %s] in %s.%s (%s), timestamps: [first row (%s, %s)], [duplicates (%s, %s, eq:%b)]", + duplicateRows, + keyString, first.clustering().toString(sstable.metadata()), + sstable.metadata().keyspace, + sstable.metadata().name, + sstable, + dateString(firstMinTs), dateString(firstMaxTs), + dateString(minTimestamp), dateString(maxTimestamp), minTimestamp == maxTimestamp); + } + + private String dateString(long time) + { + return Instant.ofEpochMilli(TimeUnit.MICROSECONDS.toMillis(time)).toString(); + } + + + private void deserializeIndex(SSTableReader sstable) throws IOException + { + try (RandomAccessReader primaryIndex = RandomAccessReader.open(sstable.descriptor.fileFor(Components.PRIMARY_INDEX))) + { + long indexSize = primaryIndex.length(); + + while ((primaryIndex.getFilePointer()) != indexSize) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(primaryIndex); + RowIndexEntry.Serializer.skip(primaryIndex, sstable.descriptor.version); + } + } + } + + private void deserializeIndexSummary(SSTableReader sstable) throws IOException + { + IndexSummaryComponent summaryComponent = IndexSummaryComponent.load(sstable.descriptor.fileFor(Components.SUMMARY), cfs.metadata()); + if (summaryComponent == null) + throw new NoSuchFileException("Index summary component of sstable " + sstable.descriptor + " is missing"); + FileUtils.closeQuietly(summaryComponent.indexSummary); + } + + @Override + public void close() + { + fileAccessLock.writeLock().lock(); + try + { + FileUtils.closeQuietly(indexFile); + super.close(); + } + finally + { + fileAccessLock.writeLock().unlock(); + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 6bb0f6eff1..5ae52cfb6c 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -17,551 +17,268 @@ */ package org.apache.cassandra.io.sstable.format.big; - import java.io.IOException; -import java.nio.BufferOverflowException; import java.nio.ByteBuffer; -import java.util.*; -import java.util.stream.Stream; - -import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.cache.ChunkCache; -import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.lifecycle.ILifecycleTransaction; +import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.compress.CompressedSequentialWriter; -import org.apache.cassandra.io.compress.ICompressor; -import org.apache.cassandra.io.sstable.*; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.Downsampling; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.DataComponent; +import org.apache.cassandra.io.sstable.format.IndexComponent; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; -import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.*; -import org.apache.cassandra.schema.CompressionParams; -import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.utils.*; -import org.apache.cassandra.utils.concurrent.SharedCloseableImpl; -import org.apache.cassandra.utils.concurrent.Transactional; +import org.apache.cassandra.io.sstable.format.SortedTableWriter; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder; +import org.apache.cassandra.io.sstable.keycache.KeyCache; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.MmappedRegionsCache; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.service.CacheService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.Throwables; +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.cassandra.io.util.FileHandle.Builder.NO_LENGTH_OVERRIDE; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -public class BigTableWriter extends SSTableWriter +public class BigTableWriter extends SortedTableWriter { private static final Logger logger = LoggerFactory.getLogger(BigTableWriter.class); - private final ColumnIndex columnIndexWriter; - private final IndexWriter iwriter; - private final FileHandle.Builder dbuilder; - protected final SequentialWriter dataFile; - private DecoratedKey lastWrittenKey; - private DataPosition dataMark; - private long lastEarlyOpenLength = 0; - private final Optional chunkCache = Optional.ofNullable(ChunkCache.instance); + private final IndexWriter indexWriter; + private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + private final Map cachedKeys = new HashMap<>(); + private final boolean shouldMigrateKeyCache; - private final SequentialWriterOption writerOption = SequentialWriterOption.newBuilder() - .trickleFsync(DatabaseDescriptor.getTrickleFsync()) - .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024) - .build(); - - public BigTableWriter(Descriptor descriptor, - long keyCount, - long repairedAt, - TimeUUID pendingRepair, - boolean isTransient, - TableMetadataRef metadata, - MetadataCollector metadataCollector, - SerializationHeader header, - Collection observers, - LifecycleNewTracker lifecycleNewTracker) + public BigTableWriter(Builder builder, LifecycleNewTracker lifecycleNewTracker, SSTable.Owner owner) { - super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, metadataCollector, header, observers); - lifecycleNewTracker.trackNew(this); // must track before any files are created + super(builder, lifecycleNewTracker, owner); + checkNotNull(builder.getRowIndexEntrySerializer()); + checkNotNull(builder.getIndexWriter()); - if (compression) - { - final CompressionParams compressionParams = compressionFor(lifecycleNewTracker.opType()); - - dataFile = new CompressedSequentialWriter(new File(getFilename()), - descriptor.filenameFor(Component.COMPRESSION_INFO), - new File(descriptor.filenameFor(Component.DIGEST)), - writerOption, - compressionParams, - metadataCollector); - } - else - { - dataFile = new ChecksummedSequentialWriter(new File(getFilename()), - new File(descriptor.filenameFor(Component.CRC)), - new File(descriptor.filenameFor(Component.DIGEST)), - writerOption); - } - dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression) - .mmapped(DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap); - chunkCache.ifPresent(dbuilder::withChunkCache); - iwriter = new IndexWriter(keyCount); - - columnIndexWriter = new ColumnIndex(this.header, dataFile, descriptor.version, this.observers, getRowIndexEntrySerializer().indexInfoSerializer()); - } - - /** - * Given an OpType, determine the correct Compression Parameters - * @param opType - * @return {@link org.apache.cassandra.schema.CompressionParams} - */ - private CompressionParams compressionFor(final OperationType opType) - { - CompressionParams compressionParams = metadata.getLocal().params.compression; - final ICompressor compressor = compressionParams.getSstableCompressor(); - - if (null != compressor && opType == OperationType.FLUSH) - { - // When we are flushing out of the memtable throughput of the compressor is critical as flushes, - // especially of large tables, can queue up and potentially block writes. - // This optimization allows us to fall back to a faster compressor if a particular - // compression algorithm indicates we should. See CASSANDRA-15379 for more details. - switch (DatabaseDescriptor.getFlushCompression()) - { - // It is relatively easier to insert a Noop compressor than to disable compressed writing - // entirely as the "compression" member field is provided outside the scope of this class. - // It may make sense in the future to refactor the ownership of the compression flag so that - // We can bypass the CompressedSequentialWriter in this case entirely. - case none: - compressionParams = CompressionParams.NOOP; - break; - case fast: - if (!compressor.recommendedUses().contains(ICompressor.Uses.FAST_COMPRESSION)) - { - // The default compressor is generally fast (LZ4 with 16KiB block size) - compressionParams = CompressionParams.DEFAULT; - break; - } - case table: - default: - } - } - return compressionParams; + this.rowIndexEntrySerializer = builder.getRowIndexEntrySerializer(); + this.indexWriter = builder.getIndexWriter(); + this.shouldMigrateKeyCache = DatabaseDescriptor.shouldMigrateKeycacheOnCompaction() + && lifecycleNewTracker instanceof ILifecycleTransaction + && !((ILifecycleTransaction) lifecycleNewTracker).isOffline(); } + @Override public void mark() { - dataMark = dataFile.mark(); - iwriter.mark(); + super.mark(); + indexWriter.mark(); } + @Override public void resetAndTruncate() { - dataFile.resetAndTruncate(dataMark); - iwriter.resetAndTruncate(); + super.resetAndTruncate(); + indexWriter.resetAndTruncate(); } - /** - * Perform sanity checks on @param decoratedKey and @return the position in the data file before any data is written - */ - protected long beforeAppend(DecoratedKey decoratedKey) + @Override + protected void onStartPartition(DecoratedKey key) { - assert decoratedKey != null : "Keys must not be null"; // empty keys ARE allowed b/c of indexed column values - if (lastWrittenKey != null && lastWrittenKey.compareTo(decoratedKey) >= 0) - throw new RuntimeException("Last written key " + lastWrittenKey + " >= current key " + decoratedKey + " writing into " + getFilename()); - return (lastWrittenKey == null) ? 0 : dataFile.position(); + notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), indexWriter.writer.position())); } - private void afterAppend(DecoratedKey decoratedKey, long dataEnd, RowIndexEntry index, ByteBuffer indexInfo) throws IOException + @Override + protected RowIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException { - metadataCollector.addKey(decoratedKey.getKey()); - lastWrittenKey = decoratedKey; - last = lastWrittenKey; - if (first == null) - first = lastWrittenKey; + // afterAppend() writes the partition key before the first RowIndexEntry - so we have to add it's + // serialized size to the index-writer position + long indexFilePosition = ByteBufferUtil.serializedSizeWithShortLength(key.getKey()) + indexWriter.writer.position(); - if (logger.isTraceEnabled()) - logger.trace("wrote {} at {}", decoratedKey, dataEnd); - iwriter.append(decoratedKey, index, dataEnd, indexInfo); - } + RowIndexEntry entry = RowIndexEntry.create(partitionWriter.getInitialPosition(), + indexFilePosition, + partitionLevelDeletion, + partitionWriter.getHeaderLength(), + partitionWriter.getColumnIndexCount(), + partitionWriter.indexInfoSerializedSize(), + partitionWriter.indexSamples(), + partitionWriter.offsets(), + rowIndexEntrySerializer.indexInfoSerializer()); - /** - * Appends partition data to this writer. - * - * @param iterator the partition to write - * @return the created index entry if something was written, that is if {@code iterator} - * wasn't empty, {@code null} otherwise. - * - * @throws FSWriteError if a write to the dataFile fails - */ - public RowIndexEntry append(UnfilteredRowIterator iterator) - { - DecoratedKey key = iterator.partitionKey(); + indexWriter.append(key, entry, dataWriter.position(), partitionWriter.buffer()); - if (key.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT) + if (shouldMigrateKeyCache) { - logger.error("Key size {} exceeds maximum of {}, skipping row", key.getKey().remaining(), FBUtilities.MAX_UNSIGNED_SHORT); - return null; - } - - if (iterator.isEmpty()) - return null; - - long startPosition = beforeAppend(key); - observers.forEach((o) -> o.startPartition(key, iwriter.indexFile.position())); - - //Reuse the writer for each row - columnIndexWriter.reset(); - - try (UnfilteredRowIterator collecting = Transformation.apply(iterator, new StatsCollector(metadataCollector))) - { - columnIndexWriter.buildRowIndex(collecting); - - // afterAppend() writes the partition key before the first RowIndexEntry - so we have to add it's - // serialized size to the index-writer position - long indexFilePosition = ByteBufferUtil.serializedSizeWithShortLength(key.getKey()) + iwriter.indexFile.position(); - - RowIndexEntry entry = RowIndexEntry.create(startPosition, indexFilePosition, - collecting.partitionLevelDeletion(), - columnIndexWriter.headerLength, - columnIndexWriter.columnIndexCount, - columnIndexWriter.indexInfoSerializedSize(), - columnIndexWriter.indexSamples(), - columnIndexWriter.offsets(), - getRowIndexEntrySerializer().indexInfoSerializer()); - - long endPosition = dataFile.position(); - long rowSize = endPosition - startPosition; - maybeLogLargePartitionWarning(key, rowSize); - maybeLogManyTombstonesWarning(key, metadataCollector.totalTombstones); - metadataCollector.addPartitionSizeInBytes(rowSize); - afterAppend(key, endPosition, entry, columnIndexWriter.buffer()); - return entry; - } - catch (BufferOverflowException boe) - { - throw new PartitionSerializationException(iterator, boe); - } - catch (IOException e) - { - throw new FSWriteError(e, dataFile.getPath()); - } - } - - private RowIndexEntry.IndexSerializer getRowIndexEntrySerializer() - { - return (RowIndexEntry.IndexSerializer) rowIndexEntrySerializer; - } - - private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize) - { - if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold()) - { - String keyString = metadata().partitionKeyType.getString(key.getKey()); - logger.warn("Writing large partition {}/{}:{} ({}) to sstable {}", metadata.keyspace, metadata.name, keyString, FBUtilities.prettyPrintMemory(rowSize), getFilename()); - } - } - - private void maybeLogManyTombstonesWarning(DecoratedKey key, int tombstoneCount) - { - if (tombstoneCount > DatabaseDescriptor.getCompactionTombstoneWarningThreshold()) - { - String keyString = metadata().partitionKeyType.getString(key.getKey()); - logger.warn("Writing {} tombstones to {}/{}:{} in sstable {}", tombstoneCount, metadata.keyspace, metadata.name, keyString, getFilename()); - } - } - - private static class StatsCollector extends Transformation - { - private final MetadataCollector collector; - private int cellCount; - - StatsCollector(MetadataCollector collector) - { - this.collector = collector; - } - - @Override - public Row applyToStatic(Row row) - { - if (!row.isEmpty()) - cellCount += Rows.collectStats(row, collector); - return row; - } - - @Override - public Row applyToRow(Row row) - { - collector.updateClusteringValues(row.clustering()); - cellCount += Rows.collectStats(row, collector); - return row; - } - - @Override - public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker) - { - collector.updateClusteringValuesByBoundOrBoundary(marker.clustering()); - if (marker.isBoundary()) + for (SSTableReader reader : ((ILifecycleTransaction) lifecycleNewTracker).originals()) { - RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker; - collector.update(bm.endDeletionTime()); - collector.update(bm.startDeletionTime()); + if (reader instanceof KeyCacheSupport && ((KeyCacheSupport) reader).getCachedPosition(key, false) != null) + { + cachedKeys.put(key, entry); + break; + } } - else - { - collector.update(((RangeTombstoneBoundMarker)marker).deletionTime()); - } - return marker; } - @Override - public void onPartitionClose() - { - collector.addCellPerPartitionCount(cellCount); - } - - @Override - public DeletionTime applyToDeletion(DeletionTime deletionTime) - { - collector.updatePartitionDeletion(deletionTime); - return deletionTime; - } + return entry; } @SuppressWarnings("resource") - public SSTableReader openEarly() + private BigTableReader openInternal(IndexSummaryBuilder.ReadableBoundary boundary, SSTableReader.OpenReason openReason) { - // find the max (exclusive) readable key - IndexSummaryBuilder.ReadableBoundary boundary = iwriter.getMaxReadable(); - if (boundary == null) - return null; + assert boundary == null || (boundary.indexLength > 0 && boundary.dataLength > 0); + IFilter filter = null; IndexSummary indexSummary = null; - FileHandle ifile = null; - FileHandle dfile = null; - SSTableReader sstable = null; + FileHandle dataFile = null; + FileHandle indexFile = null; + BigTableReader.Builder builder = unbuildTo(new BigTableReader.Builder(descriptor), true).setMaxDataAge(maxDataAge) + .setSerializationHeader(header) + .setOpenReason(openReason) + .setFirst(first) + .setLast(boundary != null ? boundary.lastKey : last); + + BigTableReader reader; try { - StatsMetadata stats = statsMetadata(); - assert boundary.indexLength > 0 && boundary.dataLength > 0; - // open the reader early - indexSummary = iwriter.summary.build(metadata().partitioner, boundary); - long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); - int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); - ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength); - if (compression) - dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(boundary.dataLength)); - EstimatedHistogram partitionSizeHistogram = stats.estimatedPartitionSize; + builder.setStatsMetadata(statsMetadata()); - if (partitionSizeHistogram.isOverflowed()) + EstimatedHistogram partitionSizeHistogram = builder.getStatsMetadata().estimatedPartitionSize; + if (boundary != null) { - logger.warn("Estimated partition size histogram for '{}' is overflowed ({} values greater than {}). " + - "Clearing the overflow bucket to allow for degraded mean and percentile calculations...", - descriptor, partitionSizeHistogram.overflowCount(), partitionSizeHistogram.getLargestBucketOffset()); - - partitionSizeHistogram.clearOverflow(); + if (partitionSizeHistogram.isOverflowed()) + { + logger.warn("Estimated partition size histogram for '{}' is overflowed ({} values greater than {}). " + + "Clearing the overflow bucket to allow for degraded mean and percentile calculations...", + descriptor, partitionSizeHistogram.overflowCount(), partitionSizeHistogram.getLargestBucketOffset()); + partitionSizeHistogram.clearOverflow(); + } } - int dataBufferSize = optimizationStrategy.bufferSize(partitionSizeHistogram.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); - dfile = dbuilder.bufferSize(dataBufferSize).complete(boundary.dataLength); - invalidateCacheAtBoundary(dfile); - sstable = SSTableReader.internalOpen(descriptor, - components, metadata, - ifile, dfile, - indexSummary, - iwriter.bf.sharedCopy(), - maxDataAge, - stats, - SSTableReader.OpenReason.EARLY, - header); + filter = indexWriter.getFilterCopy(); + builder.setFilter(filter); + indexSummary = indexWriter.summary.build(metadata().partitioner, boundary); + builder.setIndexSummary(indexSummary); - // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed) - sstable.first = getMinimalKey(first); - sstable.last = getMinimalKey(boundary.lastKey); - return sstable; + long indexFileLength = descriptor.fileFor(Components.PRIMARY_INDEX).length(); + int indexBufferSize = ioOptions.diskOptimizationStrategy.bufferSize(indexFileLength / builder.getIndexSummary().size()); + FileHandle.Builder indexFileBuilder = indexWriter.builder; + indexFile = indexFileBuilder.bufferSize(indexBufferSize) + .withLengthOverride(boundary != null ? boundary.indexLength : NO_LENGTH_OVERRIDE) + .complete(); + builder.setIndexFile(indexFile); + dataFile = openDataFile(boundary != null ? boundary.dataLength : NO_LENGTH_OVERRIDE, builder.getStatsMetadata()); + builder.setDataFile(dataFile); + builder.setKeyCache(metadata().params.caching.cacheKeys() ? new KeyCache(CacheService.instance.keyCache) : KeyCache.NO_CACHE); + + reader = builder.build(owner().orElse(null), true, true); } catch (Throwable t) { JVMStabilityInspector.inspectThrowable(t); - // If we successfully created our sstable, we can rely on its InstanceTidier to clean things up for us - if (sstable != null) - sstable.selfRef().release(); - else - Stream.of(indexSummary, ifile, dfile).filter(Objects::nonNull).forEach(SharedCloseableImpl::close); + Throwables.closeNonNullAndAddSuppressed(t, dataFile, indexFile, indexSummary, filter); throw t; } + + try + { + for (Map.Entry cachedKey : cachedKeys.entrySet()) + reader.cacheKey(cachedKey.getKey(), cachedKey.getValue()); + + // clearing the collected cache keys so that we will not have to cache them again when opening partial or + // final later - cache key refer only to the descriptor, not to the particular SSTableReader instance. + cachedKeys.clear(); + } + catch (Throwable t) + { + JVMStabilityInspector.inspectThrowable(t); + } + + return reader; } - void invalidateCacheAtBoundary(FileHandle dfile) + @Override + public void openEarly(Consumer doWhenReady) { - chunkCache.ifPresent(cache -> { - if (lastEarlyOpenLength != 0 && dfile.dataLength() > lastEarlyOpenLength) - cache.invalidatePosition(dfile, lastEarlyOpenLength); - }); - lastEarlyOpenLength = dfile.dataLength(); + // find the max (exclusive) readable key + IndexSummaryBuilder.ReadableBoundary boundary = indexWriter.getMaxReadable(); + if (boundary == null) + return; + + doWhenReady.accept(openInternal(boundary, SSTableReader.OpenReason.EARLY)); } + @Override public SSTableReader openFinalEarly() { // we must ensure the data is completely flushed to disk - dataFile.sync(); - iwriter.indexFile.sync(); + dataWriter.sync(); + indexWriter.writer.sync(); return openFinal(SSTableReader.OpenReason.EARLY); } - @SuppressWarnings("resource") - private SSTableReader openFinal(SSTableReader.OpenReason openReason) + @Override + public SSTableReader openFinal(SSTableReader.OpenReason openReason) { if (maxDataAge < 0) maxDataAge = currentTimeMillis(); - IndexSummary indexSummary = null; - FileHandle ifile = null; - FileHandle dfile = null; - SSTableReader sstable = null; - - try - { - StatsMetadata stats = statsMetadata(); - // finalize in-memory state for the reader - indexSummary = iwriter.summary.build(metadata().partitioner); - long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); - int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); - int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); - ifile = iwriter.builder.bufferSize(indexBufferSize).complete(); - if (compression) - dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(0)); - dfile = dbuilder.bufferSize(dataBufferSize).complete(); - invalidateCacheAtBoundary(dfile); - sstable = SSTableReader.internalOpen(descriptor, - components, - metadata, - ifile, - dfile, - indexSummary, - iwriter.bf.sharedCopy(), - maxDataAge, - stats, - openReason, - header); - sstable.first = getMinimalKey(first); - sstable.last = getMinimalKey(last); - return sstable; - } - catch (Throwable t) - { - JVMStabilityInspector.inspectThrowable(t); - // If we successfully created our sstable, we can rely on its InstanceTidier to clean things up for us - if (sstable != null) - sstable.selfRef().release(); - else - Stream.of(indexSummary, ifile, dfile).filter(Objects::nonNull).forEach(SharedCloseableImpl::close); - throw t; - } + return openInternal(null, openReason); } + @Override protected SSTableWriter.TransactionalProxy txnProxy() { - return new TransactionalProxy(); - } - - class TransactionalProxy extends SSTableWriter.TransactionalProxy - { - // finalise our state on disk, including renaming - protected void doPrepare() - { - iwriter.prepareToCommit(); - - // write sstable statistics - dataFile.prepareToCommit(); - writeMetadata(descriptor, finalizeMetadata()); - - // save the table of components - SSTable.appendTOC(descriptor, components); - - if (openResult) - finalReader = openFinal(SSTableReader.OpenReason.NORMAL); - } - - protected Throwable doCommit(Throwable accumulate) - { - accumulate = dataFile.commit(accumulate); - accumulate = iwriter.commit(accumulate); - return accumulate; - } - - @Override - protected Throwable doPostCleanup(Throwable accumulate) - { - accumulate = dbuilder.close(accumulate); - return accumulate; - } - - protected Throwable doAbort(Throwable accumulate) - { - accumulate = iwriter.abort(accumulate); - accumulate = dataFile.abort(accumulate); - return accumulate; - } - } - - private void writeMetadata(Descriptor desc, Map components) - { - File file = new File(desc.filenameFor(Component.STATS)); - try (SequentialWriter out = new SequentialWriter(file, writerOption)) - { - desc.getMetadataSerializer().serialize(components, out, desc.version); - out.finish(); - } - catch (IOException e) - { - throw new FSWriteError(e, file.path()); - } - } - - public long getFilePointer() - { - return dataFile.position(); - } - - public long getOnDiskFilePointer() - { - return dataFile.getOnDiskFilePointer(); - } - - public long getEstimatedOnDiskBytesWritten() - { - return dataFile.getEstimatedOnDiskBytesWritten(); + return new SSTableWriter.TransactionalProxy(() -> FBUtilities.immutableListWithFilteredNulls(indexWriter, dataWriter)); } /** * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed. */ - class IndexWriter extends AbstractTransactional implements Transactional + static class IndexWriter extends SortedTableWriter.AbstractIndexWriter { - private final SequentialWriter indexFile; - public final FileHandle.Builder builder; - public final IndexSummaryBuilder summary; - public final IFilter bf; - private DataPosition mark; + private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; - IndexWriter(long keyCount) + final SequentialWriter writer; + final FileHandle.Builder builder; + final IndexSummaryBuilder summary; + private DataPosition mark; + private DecoratedKey first; + private DecoratedKey last; + + protected IndexWriter(Builder b) { - indexFile = new SequentialWriter(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)), writerOption); - builder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap); - chunkCache.ifPresent(builder::withChunkCache); - summary = new IndexSummaryBuilder(keyCount, metadata().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL); - bf = FilterFactory.getFilter(keyCount, metadata().params.bloomFilterFpChance); + super(b); + this.rowIndexEntrySerializer = b.getRowIndexEntrySerializer(); + writer = new SequentialWriter(b.descriptor.fileFor(Components.PRIMARY_INDEX), b.getIOOptions().writerOptions); + builder = IndexComponent.fileBuilder(Components.PRIMARY_INDEX, b).withMmappedRegionsCache(b.getMmappedRegionsCache()); + summary = new IndexSummaryBuilder(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL); // register listeners to be alerted when the data files are flushed - indexFile.setPostFlushListener(() -> summary.markIndexSynced(indexFile.getLastFlushOffset())); - dataFile.setPostFlushListener(() -> summary.markDataSynced(dataFile.getLastFlushOffset())); + writer.setPostFlushListener(() -> summary.markIndexSynced(writer.getLastFlushOffset())); + @SuppressWarnings("resource") + SequentialWriter dataWriter = b.getDataWriter(); + dataWriter.setPostFlushListener(() -> summary.markDataSynced(dataWriter.getLastFlushOffset())); } // finds the last (-offset) decorated key that can be guaranteed to occur fully in the flushed portion of the index file @@ -573,17 +290,21 @@ public class BigTableWriter extends SSTableWriter public void append(DecoratedKey key, RowIndexEntry indexEntry, long dataEnd, ByteBuffer indexInfo) throws IOException { bf.add(key); - long indexStart = indexFile.position(); + if (first == null) + first = key; + last = key; + + long indexStart = writer.position(); try { - ByteBufferUtil.writeWithShortLength(key.getKey(), indexFile); - rowIndexEntrySerializer.serialize(indexEntry, indexFile, indexInfo); + ByteBufferUtil.writeWithShortLength(key.getKey(), writer); + rowIndexEntrySerializer.serialize(indexEntry, writer, indexInfo); } catch (IOException e) { - throw new FSWriteError(e, indexFile.getPath()); + throw new FSWriteError(e, writer.getPath()); } - long indexEnd = indexFile.position(); + long indexEnd = writer.position(); if (logger.isTraceEnabled()) logger.trace("wrote index entry: {} at {}", indexEntry, indexStart); @@ -591,75 +312,152 @@ public class BigTableWriter extends SSTableWriter summary.maybeAddEntry(key, indexStart, indexEnd, dataEnd); } - /** - * Closes the index and bloomfilter, making the public state of this writer valid for consumption. - */ - void flushBf() - { - if (components.contains(Component.FILTER)) - { - String path = descriptor.filenameFor(Component.FILTER); - try (FileOutputStreamPlus stream = new FileOutputStreamPlus(path)) - { - // bloom filter - BloomFilterSerializer.serialize((BloomFilter) bf, stream); - stream.flush(); - stream.sync(); - } - catch (IOException e) - { - throw new FSWriteError(e, path); - } - } - } - public void mark() { - mark = indexFile.mark(); + mark = writer.mark(); } public void resetAndTruncate() { // we can't un-set the bloom filter addition, but extra keys in there are harmless. - // we can't reset dbuilder either, but that is the last thing called in afterappend so + // we can't reset dbuilder either, but that is the last thing called in afterappend, so // we assume that if that worked then we won't be trying to reset. - indexFile.resetAndTruncate(mark); + writer.resetAndTruncate(mark); } protected void doPrepare() { - flushBf(); + checkNotNull(first); + checkNotNull(last); + + super.doPrepare(); // truncate index file - long position = indexFile.position(); - indexFile.prepareToCommit(); - FileUtils.truncate(indexFile.getPath(), position); + long position = writer.position(); + writer.prepareToCommit(); + FileUtils.truncate(writer.getPath(), position); // save summary summary.prepareToCommit(); - try (IndexSummary indexSummary = summary.build(getPartitioner())) + try (IndexSummary indexSummary = summary.build(metadata.getLocal().partitioner)) { - SSTableReader.saveSummary(descriptor, first, last, indexSummary); + new IndexSummaryComponent(indexSummary, first, last).save(descriptor.fileFor(Components.SUMMARY), true); + } + catch (IOException ex) + { + logger.warn("Failed to save index summary", ex); } } protected Throwable doCommit(Throwable accumulate) { - return indexFile.commit(accumulate); + return writer.commit(accumulate); } protected Throwable doAbort(Throwable accumulate) { - return summary.close(indexFile.abort(accumulate)); + return summary.close(writer.abort(accumulate)); } @Override protected Throwable doPostCleanup(Throwable accumulate) { + accumulate = super.doPostCleanup(accumulate); accumulate = summary.close(accumulate); - accumulate = bf.close(accumulate); - accumulate = builder.close(accumulate); return accumulate; } } -} \ No newline at end of file + + public static class Builder extends SortedTableWriter.Builder + { + private RowIndexEntry.IndexSerializer rowIndexEntrySerializer; + private IndexWriter indexWriter; + private SequentialWriter dataWriter; + private BigFormatPartitionWriter partitionWriter; + private MmappedRegionsCache mmappedRegionsCache; + + public Builder(Descriptor descriptor) + { + super(descriptor); + } + + @Override + public Builder addDefaultComponents() + { + super.addDefaultComponents(); + + addComponents(ImmutableSet.of(Components.PRIMARY_INDEX, Components.SUMMARY)); + + return this; + } + + // The following getters for the resources opened by buildInternal method can be only used during the lifetime of + // that method - that is, during the construction of the sstable. + + @Override + public MmappedRegionsCache getMmappedRegionsCache() + { + return ensuringInBuildInternalContext(mmappedRegionsCache); + } + + @Override + public SequentialWriter getDataWriter() + { + return ensuringInBuildInternalContext(dataWriter); + } + + @Override + public BigFormatPartitionWriter getPartitionWriter() + { + return ensuringInBuildInternalContext(partitionWriter); + } + + public RowIndexEntry.IndexSerializer getRowIndexEntrySerializer() + { + return ensuringInBuildInternalContext(rowIndexEntrySerializer); + } + + public IndexWriter getIndexWriter() + { + return ensuringInBuildInternalContext(indexWriter); + } + + private T ensuringInBuildInternalContext(T value) + { + Preconditions.checkState(value != null, "This getter can be used only during the lifetime of the sstable constructor. Do not use it directly."); + return value; + } + + @Override + protected BigTableWriter buildInternal(LifecycleNewTracker lifecycleNewTracker, Owner owner) + { + try + { + mmappedRegionsCache = new MmappedRegionsCache(); + rowIndexEntrySerializer = new RowIndexEntry.Serializer(descriptor.version, getSerializationHeader(), owner != null ? owner.getMetrics() : null); + dataWriter = DataComponent.buildWriter(descriptor, + getTableMetadataRef().getLocal(), + getIOOptions().writerOptions, + getMetadataCollector(), + lifecycleNewTracker.opType(), + getIOOptions().flushCompression); + indexWriter = new IndexWriter(this); + partitionWriter = new BigFormatPartitionWriter(getSerializationHeader(), dataWriter, descriptor.version, rowIndexEntrySerializer.indexInfoSerializer()); + return new BigTableWriter(this, lifecycleNewTracker, owner); + } + catch (RuntimeException | Error ex) + { + Throwables.closeAndAddSuppressed(ex, partitionWriter, indexWriter, dataWriter, mmappedRegionsCache); + throw ex; + } + finally + { + rowIndexEntrySerializer = null; + indexWriter = null; + dataWriter = null; + partitionWriter = null; + mmappedRegionsCache = null; + } + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java new file mode 100644 index 0000000000..47da54d4ce --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexState.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.io.sstable.AbstractSSTableIterator; +import org.apache.cassandra.io.sstable.IndexInfo; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.FileHandle; + +// Used by indexed readers to store where they are of the index. +public class IndexState implements AutoCloseable +{ + private final AbstractSSTableIterator.AbstractReader reader; + private final ClusteringComparator comparator; + + private final RowIndexEntry indexEntry; + private final RowIndexEntry.IndexInfoRetriever indexInfoRetriever; + private final boolean reversed; + + private int currentIndexIdx; + + // Marks the beginning of the block corresponding to currentIndexIdx. + private DataPosition mark; + + public IndexState(AbstractSSTableIterator.AbstractReader reader, ClusteringComparator comparator, RowIndexEntry indexEntry, boolean reversed, FileHandle indexFile) + { + this.reader = reader; + this.comparator = comparator; + this.indexEntry = indexEntry; + this.indexInfoRetriever = indexEntry.openWithIndex(indexFile); + this.reversed = reversed; + this.currentIndexIdx = reversed ? indexEntry.columnsIndexCount() : -1; + } + + public boolean isDone() + { + return reversed ? currentIndexIdx < 0 : currentIndexIdx >= indexEntry.columnsIndexCount(); + } + + // Sets the reader to the beginning of blockIdx. + public void setToBlock(int blockIdx) throws IOException + { + if (blockIdx >= 0 && blockIdx < indexEntry.columnsIndexCount()) + { + reader.seekToPosition(columnOffset(blockIdx)); + mark = reader.file.mark(); + reader.deserializer.clearState(); + } + + currentIndexIdx = blockIdx; + reader.openMarker = blockIdx > 0 ? index(blockIdx - 1).endOpenMarker : null; + } + + private long columnOffset(int i) throws IOException + { + return indexEntry.position + index(i).offset; + } + + public int blocksCount() + { + return indexEntry.columnsIndexCount(); + } + + // Update the block idx based on the current reader position if we're past the current block. + // This only makes sense for forward iteration (for reverse ones, when we reach the end of a block we + // should seek to the previous one, not update the index state and continue). + public void updateBlock() throws IOException + { + assert !reversed; + + // If we get here with currentBlockIdx < 0, it means setToBlock() has never been called, so it means + // we're about to read from the beginning of the partition, but haven't "prepared" the IndexState yet. + // Do so by setting us on the first block. + if (currentIndexIdx < 0) + { + setToBlock(0); + return; + } + + while (currentIndexIdx + 1 < indexEntry.columnsIndexCount() && isPastCurrentBlock()) + { + reader.openMarker = currentIndex().endOpenMarker; + ++currentIndexIdx; + + // We have to set the mark, and we have to set it at the beginning of the block. So if we're not at the beginning of the block, this forces us to a weird seek dance. + // This can only happen when reading old file however. + long startOfBlock = columnOffset(currentIndexIdx); + long currentFilePointer = reader.file.getFilePointer(); + if (startOfBlock == currentFilePointer) + { + mark = reader.file.mark(); + } + else + { + reader.seekToPosition(startOfBlock); + mark = reader.file.mark(); + reader.seekToPosition(currentFilePointer); + } + } + } + + // Check if we've crossed an index boundary (based on the mark on the beginning of the index block). + public boolean isPastCurrentBlock() throws IOException + { + assert reader.deserializer != null; + return reader.file.bytesPastMark(mark) >= currentIndex().width; + } + + public int currentBlockIdx() + { + return currentIndexIdx; + } + + public IndexInfo currentIndex() throws IOException + { + return index(currentIndexIdx); + } + + public IndexInfo index(int i) throws IOException + { + return indexInfoRetriever.columnsIndex(i); + } + + // Finds the index of the first block containing the provided bound, starting at the provided index. + // Will be -1 if the bound is before any block, and blocksCount() if it is after every block. + public int findBlockIndex(ClusteringBound bound, int fromIdx) throws IOException + { + if (bound.isBottom()) + return -1; + if (bound.isTop()) + return blocksCount(); + + return indexFor(bound, fromIdx); + } + + public int indexFor(ClusteringPrefix name, int lastIndex) throws IOException + { + IndexInfo target = new IndexInfo(name, name, 0, 0, null); + /* + Take the example from the unit test, and say your index looks like this: + [0..5][10..15][20..25] + and you look for the slice [13..17]. + + When doing forward slice, we are doing a binary search comparing 13 (the start of the query) + to the lastName part of the index slot. You'll end up with the "first" slot, going from left to right, + that may contain the start. + + When doing a reverse slice, we do the same thing, only using as a start column the end of the query, + i.e. 17 in this example, compared to the firstName part of the index slots. bsearch will give us the + first slot where firstName > start ([20..25] here), so we subtract an extra one to get the slot just before. + */ + int startIdx = 0; + int endIdx = indexEntry.columnsIndexCount() - 1; + + if (reversed) + { + if (lastIndex < endIdx) + { + endIdx = lastIndex; + } + } + else + { + if (lastIndex > 0) + { + startIdx = lastIndex; + } + } + + int index = binarySearch(target, comparator.indexComparator(reversed), startIdx, endIdx); + return (index < 0 ? -index - (reversed ? 2 : 1) : index); + } + + private int binarySearch(IndexInfo key, Comparator c, int low, int high) throws IOException + { + while (low <= high) + { + int mid = (low + high) >>> 1; + IndexInfo midVal = index(mid); + int cmp = c.compare(midVal, key); + + if (cmp < 0) + low = mid + 1; + else if (cmp > 0) + high = mid - 1; + else + return mid; + } + return -(low + 1); + } + + @Override + public String toString() + { + return String.format("IndexState(indexSize=%d, currentBlock=%d, reversed=%b)", indexEntry.columnsIndexCount(), currentIndexIdx, reversed); + } + + @Override + public void close() throws IOException + { + indexInfoRetriever.close(); + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java b/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java new file mode 100644 index 0000000000..b3a40cbafe --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/format/big/IndexSummaryComponent.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Pair; + +public class IndexSummaryComponent +{ + private static final Logger logger = LoggerFactory.getLogger(IndexSummaryComponent.class); + + public final IndexSummary indexSummary; + public final DecoratedKey first; + public final DecoratedKey last; + + public IndexSummaryComponent(IndexSummary indexSummary, DecoratedKey first, DecoratedKey last) + { + this.indexSummary = indexSummary; + this.first = first; + this.last = last; + } + + public static Pair loadFirstAndLastKey(File summaryFile, IPartitioner partitioner) throws IOException + { + if (!summaryFile.exists()) + { + if (logger.isDebugEnabled()) + logger.debug("Index summary {} does not exist", summaryFile.absolutePath()); + return null; + } + + try (FileInputStreamPlus iStream = summaryFile.newInputStream()) + { + return new IndexSummary.IndexSummarySerializer().deserializeFirstLastKey(iStream, partitioner); + } + } + + /** + * Load index summary, first key and last key from Summary.db file if it exists. + *

+ * if loaded index summary has different index interval from current value stored in schema, + * then Summary.db file will be deleted and need to be rebuilt. + */ + @SuppressWarnings("resource") + public static IndexSummaryComponent load(File summaryFile, TableMetadata metadata) throws IOException + { + if (!summaryFile.exists()) + { + if (logger.isDebugEnabled()) + logger.debug("Index summary {} does not exist", summaryFile.absolutePath()); + return null; + } + + IndexSummary summary = null; + try (FileInputStreamPlus iStream = summaryFile.newInputStream()) + { + summary = IndexSummary.serializer.deserialize(iStream, + metadata.partitioner, + metadata.params.minIndexInterval, + metadata.params.maxIndexInterval); + DecoratedKey first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); + DecoratedKey last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); + + return new IndexSummaryComponent(summary, first, last); + } + catch (IOException ex) + { + if (summary != null) + summary.close(); + + throw new IOException(String.format("Cannot deserialize index summary from %s", summaryFile), ex); + } + } + + public static IndexSummaryComponent loadOrDeleteCorrupted(File summaryFile, TableMetadata metadata) throws IOException + { + try + { + return load(summaryFile, metadata); + } + catch (IOException ex) + { + summaryFile.deleteIfExists(); + throw ex; + } + } + + /** + * Save index summary to Summary.db file. + */ + public void save(File summaryFile, boolean deleteOnFailure) throws IOException + { + if (summaryFile.exists()) + summaryFile.delete(); + + try (DataOutputStreamPlus oStream = summaryFile.newOutputStream(File.WriteMode.OVERWRITE)) + { + IndexSummary.serializer.serialize(indexSummary, oStream); + ByteBufferUtil.writeWithLength(first.getKey(), oStream); + ByteBufferUtil.writeWithLength(last.getKey(), oStream); + } + catch (IOException ex) + { + if (deleteOnFailure) + summaryFile.deleteIfExists(); + throw new IOException("Failed to save index summary to " + summaryFile, ex); + } + } +} diff --git a/src/java/org/apache/cassandra/db/RowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java similarity index 90% rename from src/java/org/apache/cassandra/db/RowIndexEntry.java rename to src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java index cf6ae7e7a2..d8989dbb27 100644 --- a/src/java/org/apache/cassandra/db/RowIndexEntry.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java @@ -15,25 +15,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import com.codahale.metrics.Histogram; -import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.filter.RowIndexEntryReadSizeTooLargeException; +import org.apache.cassandra.db.ArrayClustering; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.MessageParams; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RejectException; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.ISerializer; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.IndexInfo; import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.TrackedDataInputPlus; import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.metrics.MetricNameFactory; +import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.net.ParamType; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.vint.VIntCoding; @@ -123,7 +135,7 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; *

* */ -public class RowIndexEntry implements IMeasurableMemory +public class RowIndexEntry extends AbstractRowIndexEntry { private static final long EMPTY_SIZE = ObjectSizes.measure(new RowIndexEntry(0)); @@ -145,17 +157,16 @@ public class RowIndexEntry implements IMeasurableMemory indexInfoReadsHistogram = Metrics.histogram(factory.createMetricName("IndexInfoReads"), false); } - public final long position; - public RowIndexEntry(long position) { - this.position = position; + super(position); } /** * @return true if this index entry contains the row-level tombstone and column summary. Otherwise, * caller should fetch these from the row header. */ + @Override public boolean isIndexed() { return columnsIndexCount() > 1; @@ -166,24 +177,33 @@ public class RowIndexEntry implements IMeasurableMemory return false; } + @Override public DeletionTime deletionTime() { throw new UnsupportedOperationException(); } + @Override public int columnsIndexCount() { return 0; } + @Override + public BigFormat getSSTableFormat() + { + return BigFormat.getInstance(); + } + + @Override public long unsharedHeapSize() { return EMPTY_SIZE; } /** - * @param dataFilePosition position of the partition in the {@link org.apache.cassandra.io.sstable.Component.Type#DATA} file - * @param indexFilePosition position in the {@link org.apache.cassandra.io.sstable.Component.Type#PRIMARY_INDEX} of the {@link RowIndexEntry} + * @param dataFilePosition position of the partition in the {@link Components.Types#DATA} file + * @param indexFilePosition position in the {@link Components.Types#PRIMARY_INDEX} of the {@link RowIndexEntry} * @param deletionTime deletion time of {@link RowIndexEntry} * @param headerLength deletion time of {@link RowIndexEntry} * @param columnIndexCount number of {@link IndexInfo} entries in the {@link RowIndexEntry} @@ -192,7 +212,7 @@ public class RowIndexEntry implements IMeasurableMemory * @param offsets offsets of IndexInfo offsets * @param idxInfoSerializer the {@link IndexInfo} serializer */ - public static RowIndexEntry create(long dataFilePosition, long indexFilePosition, + public static RowIndexEntry create(long dataFilePosition, long indexFilePosition, DeletionTime deletionTime, long headerLength, int columnIndexCount, int indexedPartSize, List indexSamples, int[] offsets, @@ -214,7 +234,7 @@ public class RowIndexEntry implements IMeasurableMemory deletionTime, headerLength, columnIndexCount, indexedPartSize, idxInfoSerializer); // Last case is that there are no index samples. - return new RowIndexEntry<>(dataFilePosition); + return new RowIndexEntry(dataFilePosition); } public IndexInfoRetriever openWithIndex(FileHandle indexFile) @@ -222,65 +242,72 @@ public class RowIndexEntry implements IMeasurableMemory return null; } - public interface IndexSerializer + public interface IndexSerializer { - void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException; + void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException; - RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException; - default RowIndexEntry deserialize(RandomAccessReader reader) throws IOException + RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException; + default RowIndexEntry deserialize(RandomAccessReader reader) throws IOException { return deserialize(reader, reader.getFilePointer()); } - default RowIndexEntry deserialize(FileDataInput input) throws IOException + default RowIndexEntry deserialize(FileDataInput input) throws IOException { return deserialize(input, input.getFilePointer()); } - void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException; - RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException; + void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException; + + RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException; long deserializePositionAndSkip(DataInputPlus in) throws IOException; - ISerializer indexInfoSerializer(); + ISerializer indexInfoSerializer(); } - public static final class Serializer implements IndexSerializer + public static final class Serializer implements IndexSerializer { private final IndexInfo.Serializer idxInfoSerializer; private final Version version; + private final TableMetrics tableMetrics; - public Serializer(Version version, SerializationHeader header) + public Serializer(Version version, SerializationHeader header, TableMetrics tableMetrics) { this.idxInfoSerializer = IndexInfo.serializer(version, header); this.version = version; + this.tableMetrics = tableMetrics; } + @Override public IndexInfo.Serializer indexInfoSerializer() { return idxInfoSerializer; } - public void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException + @Override + public void serialize(RowIndexEntry rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException { rie.serialize(out, indexInfo); } - public void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException + @Override + public void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException { rie.serializeForCache(out); } - public RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException + @Override + public RowIndexEntry deserializeForCache(DataInputPlus in) throws IOException { long position = in.readUnsignedVInt(); switch (in.readByte()) { case CACHE_NOT_INDEXED: - return new RowIndexEntry<>(position); + return new RowIndexEntry(position); case CACHE_INDEXED: return new IndexedEntry(position, in, idxInfoSerializer); case CACHE_INDEXED_SHALLOW: @@ -308,14 +335,15 @@ public class RowIndexEntry implements IMeasurableMemory } } - public RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException + @Override + public RowIndexEntry deserialize(DataInputPlus in, long indexFilePosition) throws IOException { long position = in.readUnsignedVInt(); int size = in.readUnsignedVInt32(); if (size == 0) { - return new RowIndexEntry<>(position); + return new RowIndexEntry(position); } else { @@ -356,9 +384,8 @@ public class RowIndexEntry implements IMeasurableMemory return; long estimatedMemory = estimateMaterializedIndexSize(entries, bytes); - ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(command.metadata().id); - if (cfs != null) - cfs.metric.rowIndexSize.update(estimatedMemory); + if (tableMetrics != null) + tableMetrics.rowIndexSize.update(estimatedMemory); if (failThreshold != null && estimatedMemory > failThreshold.toBytes()) { @@ -389,6 +416,7 @@ public class RowIndexEntry implements IMeasurableMemory return (overhead * entries) + bytes; } + @Override public long deserializePositionAndSkip(DataInputPlus in) throws IOException { long position = in.readUnsignedVInt(); @@ -424,12 +452,6 @@ public class RowIndexEntry implements IMeasurableMemory in.skipBytesFully(size); } - - public static void serializeOffsets(DataOutputBuffer out, int[] indexOffsets, int columnIndexCount) throws IOException - { - for (int i = 0; i < columnIndexCount; i++) - out.writeInt(indexOffsets[i]); - } } private static int serializedSize(DeletionTime deletionTime, long headerLength, int columnIndexCount) @@ -456,7 +478,7 @@ public class RowIndexEntry implements IMeasurableMemory /** * An entry in the row index for a row whose columns are indexed - used for both legacy and current formats. */ - private static final class IndexedEntry extends RowIndexEntry + private static final class IndexedEntry extends RowIndexEntry { private static final long BASE_SIZE; @@ -570,6 +592,7 @@ public class RowIndexEntry implements IMeasurableMemory return columnsIndex[index]; } + @Override public void close() { indexInfoGetsHistogram.update(retrievals); @@ -588,6 +611,7 @@ public class RowIndexEntry implements IMeasurableMemory + ObjectSizes.sizeOfReferenceArray(columnsIndex.length); } + @Override public void serialize(DataOutputPlus out, ByteBuffer indexInfo) throws IOException { assert indexedPartSize != Integer.MIN_VALUE; @@ -633,7 +657,7 @@ public class RowIndexEntry implements IMeasurableMemory * An entry in the row index for a row whose columns are indexed and the {@link IndexInfo} objects * are not read into the key cache. */ - private static final class ShallowIndexedEntry extends RowIndexEntry + private static final class ShallowIndexedEntry extends RowIndexEntry { private static final long BASE_SIZE; @@ -782,6 +806,7 @@ public class RowIndexEntry implements IMeasurableMemory this.offsetsOffset = offsetsOffset; } + @Override IndexInfo fetchIndex(int index) throws IOException { // seek to position in "offsets to IndexInfo" table @@ -832,6 +857,7 @@ public class RowIndexEntry implements IMeasurableMemory this.indexReader = indexReader; } + @Override public final IndexInfo columnsIndex(int index) throws IOException { retrievals++; @@ -840,6 +866,7 @@ public class RowIndexEntry implements IMeasurableMemory abstract IndexInfo fetchIndex(int index) throws IOException; + @Override public void close() throws IOException { indexReader.close(); @@ -848,4 +875,12 @@ public class RowIndexEntry implements IMeasurableMemory indexInfoReadsHistogram.update(retrievals); } } + + public static class RowIndexEntryReadSizeTooLargeException extends RejectException + { + public RowIndexEntryReadSizeTooLargeException(String message) + { + super(message); + } + } } diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java similarity index 53% rename from src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java rename to src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java index ddcbcf929b..0415d0a7eb 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableIterator.java @@ -15,14 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.columniterator; +package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; -import java.util.NoSuchElementException; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.UnfilteredValidation; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.io.sstable.AbstractSSTableIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileHandle; @@ -30,7 +34,7 @@ import org.apache.cassandra.io.util.FileHandle; /** * A Cell Iterator over SSTable */ -public class SSTableIterator extends AbstractSSTableIterator +public class SSTableIterator extends AbstractSSTableIterator { /** * The index of the slice being processed. @@ -73,139 +77,6 @@ public class SSTableIterator extends AbstractSSTableIterator return false; } - private class ForwardReader extends Reader - { - // The start of the current slice. This will be null as soon as we know we've passed that bound. - protected ClusteringBound start; - // The end of the current slice. Will never be null. - protected ClusteringBound end = BufferClusteringBound.TOP; - - protected Unfiltered next; // the next element to return: this is computed by hasNextInternal(). - - protected boolean sliceDone; // set to true once we know we have no more result for the slice. This is in particular - // used by the indexed reader when we know we can't have results based on the index. - - private ForwardReader(FileDataInput file, boolean shouldCloseFile) - { - super(file, shouldCloseFile); - } - - public void setForSlice(Slice slice) throws IOException - { - start = slice.start().isBottom() ? null : slice.start(); - end = slice.end(); - - sliceDone = false; - next = null; - } - - // Skip all data that comes before the currently set slice. - // Return what should be returned at the end of this, or null if nothing should. - private Unfiltered handlePreSliceData() throws IOException - { - assert deserializer != null; - - // Note that the following comparison is not strict. The reason is that the only cases - // where it can be == is if the "next" is a RT start marker (either a '[' of a ')[' boundary), - // and if we had a strict inequality and an open RT marker before this, we would issue - // the open marker first, and then return then next later, which would send in the - // stream both '[' (or '(') and then ')[' for the same clustering value, which is wrong. - // By using a non-strict inequality, we avoid that problem (if we do get ')[' for the same - // clustering value than the slice, we'll simply record it in 'openMarker'). - while (deserializer.hasNext() && deserializer.compareNextTo(start) <= 0) - { - if (deserializer.nextIsRow()) - deserializer.skipNext(); - else - updateOpenMarker((RangeTombstoneMarker)deserializer.readNext()); - } - - ClusteringBound sliceStart = start; - start = null; - - // We've reached the beginning of our queried slice. If we have an open marker - // we should return that first. - if (openMarker != null) - return new RangeTombstoneBoundMarker(sliceStart, openMarker); - - return null; - } - - // Compute the next element to return, assuming we're in the middle to the slice - // and the next element is either in the slice, or just after it. Returns null - // if we're done with the slice. - protected Unfiltered computeNext() throws IOException - { - assert deserializer != null; - - while (true) - { - // We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below. - // We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those - // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound - // (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because - // it's fundamentally excluded. And if the bound is a end (for a range tombstone), it means it's exactly - // our slice end, but in that case we will properly close the range tombstone anyway as part of our "close - // an open marker" code in hasNextInterna - if (!deserializer.hasNext() || deserializer.compareNextTo(end) >= 0) - return null; - - Unfiltered next = deserializer.readNext(); - UnfilteredValidation.maybeValidateUnfiltered(next, metadata(), key, sstable); - // We may get empty row for the same reason expressed on UnfilteredSerializer.deserializeOne. - if (next.isEmpty()) - continue; - - if (next.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER) - updateOpenMarker((RangeTombstoneMarker) next); - return next; - } - } - - protected boolean hasNextInternal() throws IOException - { - if (next != null) - return true; - - if (sliceDone) - return false; - - if (start != null) - { - Unfiltered unfiltered = handlePreSliceData(); - if (unfiltered != null) - { - next = unfiltered; - return true; - } - } - - next = computeNext(); - if (next != null) - return true; - - // for current slice, no data read from deserialization - sliceDone = true; - // If we have an open marker, we should not close it, there could be more slices - if (openMarker != null) - { - next = new RangeTombstoneBoundMarker(end, openMarker); - return true; - } - return false; - } - - protected Unfiltered nextInternal() throws IOException - { - if (!hasNextInternal()) - throw new NoSuchElementException(); - - Unfiltered toReturn = next; - next = null; - return toReturn; - } - } - private class ForwardIndexedReader extends ForwardReader { private final IndexState indexState; diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java similarity index 95% rename from src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java rename to src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java index 37db6d9754..14e25383d9 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/SSTableReversedIterator.java @@ -15,15 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db.columniterator; +package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.BufferClusteringBound; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.MutableDeletionInfo; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.UnfilteredValidation; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.ImmutableBTreePartition; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.io.sstable.AbstractSSTableIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileHandle; @@ -34,7 +49,7 @@ import org.apache.cassandra.utils.btree.BTree; /** * A Cell Iterator in reversed clustering order over SSTable */ -public class SSTableReversedIterator extends AbstractSSTableIterator +public class SSTableReversedIterator extends AbstractSSTableIterator { /** * The index of the slice being processed. @@ -77,7 +92,7 @@ public class SSTableReversedIterator extends AbstractSSTableIterator return slice < slices.size(); } - private class ReverseReader extends Reader + private class ReverseReader extends AbstractReader { protected ReusablePartitionData buffer; protected Iterator iterator; diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java similarity index 73% rename from src/java/org/apache/cassandra/io/sstable/IndexSummary.java rename to src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java index 303adfd75c..105d235373 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java @@ -15,20 +15,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; -import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.Downsampling; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.Memory; +import org.apache.cassandra.io.util.MemoryOutputStream; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -261,6 +275,120 @@ public class IndexSummary extends WrappedSharedCloseable return Downsampling.getEffectiveIndexIntervalAfterIndex(index, samplingLevel, minIndexInterval); } + public List getSampleIndexesForRanges(Collection> ranges) + { + // use the index to determine a minimal section for each range + List positions = new ArrayList<>(); + + for (Range range : Range.normalize(ranges)) + { + PartitionPosition leftPosition = range.left.maxKeyBound(); + PartitionPosition rightPosition = range.right.maxKeyBound(); + + int left = binarySearch(leftPosition); + if (left < 0) + left = (left + 1) * -1; + else + // left range are start exclusive + left = left + 1; + if (left == size()) + // left is past the end of the sampling + continue; + + int right = Range.isWrapAround(range.left, range.right) + ? size() - 1 + : binarySearch(rightPosition); + if (right < 0) + { + // range are end inclusive so we use the previous index from what binarySearch give us + // since that will be the last index we will return + right = (right + 1) * -1; + if (right == 0) + // Means the first key is already stricly greater that the right bound + continue; + right--; + } + + if (left > right) + // empty range + continue; + positions.add(new SSTableReader.IndexesBounds(left, right)); + } + return positions; + } + + public Iterable getKeySamples(final Range range) + { + final List indexRanges = getSampleIndexesForRanges(Collections.singletonList(range)); + + if (indexRanges.isEmpty()) + return Collections.emptyList(); + + return () -> new Iterator() + { + private Iterator rangeIter = indexRanges.iterator(); + private SSTableReader.IndexesBounds current; + private int idx; + + public boolean hasNext() + { + if (current == null || idx > current.upperPosition) + { + if (rangeIter.hasNext()) + { + current = rangeIter.next(); + idx = current.lowerPosition; + return true; + } + return false; + } + + return true; + } + + public byte[] next() + { + return getKey(idx++); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + }; + } + + public long getScanPosition(PartitionPosition key) + { + return getScanPositionFromBinarySearchResult(binarySearch(key)); + } + + @VisibleForTesting + public long getScanPositionFromBinarySearchResult(int binarySearchResult) + { + if (binarySearchResult == -1) + return 0; + else + return getPosition(getIndexFromBinarySearchResult(binarySearchResult)); + } + + public static int getIndexFromBinarySearchResult(int binarySearchResult) + { + if (binarySearchResult < 0) + { + // binary search gives us the first index _greater_ than the key searched for, + // i.e., its insertion position + int greaterThan = (binarySearchResult + 1) * -1; + if (greaterThan == 0) + return -1; + return greaterThan - 1; + } + else + { + return binarySearchResult; + } + } + public IndexSummary sharedCopy() { return new IndexSummary(this); @@ -294,7 +422,7 @@ public class IndexSummary extends WrappedSharedCloseable } @SuppressWarnings("resource") - public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException + public IndexSummary deserialize(T in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException { int minIndexInterval = in.readInt(); if (minIndexInterval != expectedMinIndexInterval) @@ -340,18 +468,18 @@ public class IndexSummary extends WrappedSharedCloseable /** * Deserializes the first and last key stored in the summary - * + *

* Only for use by offline tools like SSTableMetadataViewer, otherwise SSTable.first/last should be used. */ - public Pair deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner) throws IOException + public Pair deserializeFirstLastKey(DataInputStreamPlus in, IPartitioner partitioner) throws IOException { in.skipBytes(4); // minIndexInterval int offsetCount = in.readInt(); long offheapSize = in.readLong(); in.skipBytes(8); // samplingLevel, fullSamplingSummarySize - in.skip(offsetCount * 4); - in.skip(offheapSize - offsetCount * 4); + in.skipBytes(offsetCount * 4); + in.skipBytes((int) (offheapSize - offsetCount * 4)); DecoratedKey first = partitioner.decorateKey(ByteBufferUtil.readWithLength(in)); DecoratedKey last = partitioner.decorateKey(ByteBufferUtil.readWithLength(in)); diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java similarity index 99% rename from src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java rename to src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java index 75cca84b97..19c5b5b42a 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.nio.ByteOrder; @@ -29,6 +29,7 @@ import org.apache.cassandra.config.Config; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.sstable.Downsampling; import org.apache.cassandra.io.util.Memory; import org.apache.cassandra.io.util.SafeMemoryWriter; diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java similarity index 79% rename from src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java rename to src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java index b11ad2b354..82dc3b8cc2 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManager.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.util.ArrayList; @@ -26,16 +26,16 @@ import java.util.Set; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; - -import com.codahale.metrics.Timer; -import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.codahale.metrics.Timer; +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; @@ -58,11 +58,11 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFac * Manages the fixed-size memory pool for index summaries, periodically resizing them * in order to give more memory to hot sstables and less memory to cold sstables. */ -public class IndexSummaryManager implements IndexSummaryManagerMBean +public class IndexSummaryManager> implements IndexSummaryManagerMBean { private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManager.class); public static final String MBEAN_NAME = "org.apache.cassandra.db:type=IndexSummaries"; - public static final IndexSummaryManager instance; + public static final IndexSummaryManager instance; private long memoryPoolBytes; @@ -71,14 +71,28 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean // our next scheduled resizing run private ScheduledFuture future; + private final Supplier> indexSummariesProvider; + + private static List getAllSupportedReaders() { + List readers = new ArrayList<>(); + for (Keyspace keyspace : Keyspace.all()) + for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores()) + for (SSTableReader sstr : cfs.getLiveSSTables()) + if (sstr instanceof IndexSummarySupport) + readers.add(((T) sstr)); + return readers; + } + static { - instance = new IndexSummaryManager(); + instance = new IndexSummaryManager<>(IndexSummaryManager::getAllSupportedReaders); MBeanWrapper.instance.registerMBean(instance, MBEAN_NAME); } - private IndexSummaryManager() + private IndexSummaryManager(Supplier> indexSummariesProvider) { + this.indexSummariesProvider = indexSummariesProvider; + executor = executorFactory().scheduled(false, "IndexSummaryManager", Thread.MIN_PRIORITY); long indexSummarySizeInMB = DatabaseDescriptor.getIndexSummaryCapacityInMiB(); @@ -145,21 +159,21 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean public Map getIndexIntervals() { - List sstables = getAllSSTables(); - Map intervals = new HashMap<>(sstables.size()); - for (SSTableReader sstable : sstables) - intervals.put(sstable.getFilename(), (int) Math.round(sstable.getEffectiveIndexInterval())); + List summaryProviders = indexSummariesProvider.get(); + Map intervals = new HashMap<>(summaryProviders.size()); + for (T summaryProvider : summaryProviders) + intervals.put(summaryProvider.getFilename(), (int) Math.round(summaryProvider.getIndexSummary().getEffectiveIndexInterval())); return intervals; } public double getAverageIndexInterval() { - List sstables = getAllSSTables(); + List summaryProviders = indexSummariesProvider.get(); double total = 0.0; - for (SSTableReader sstable : sstables) - total += sstable.getEffectiveIndexInterval(); - return total / sstables.size(); + for (IndexSummarySupport summaryProvider : summaryProviders) + total += summaryProvider.getIndexSummary().getEffectiveIndexInterval(); + return total / summaryProviders.size(); } public void setMemoryPoolCapacityInMB(long memoryPoolCapacityInMB) @@ -174,23 +188,11 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean public double getMemoryPoolSizeInMB() { long total = 0; - for (SSTableReader sstable : getAllSSTables()) - total += sstable.getIndexSummaryOffHeapSize(); + for (IndexSummarySupport summaryProvider : indexSummariesProvider.get()) + total += summaryProvider.getIndexSummary().getOffHeapSize(); return total / 1024.0 / 1024.0; } - private List getAllSSTables() - { - List result = new ArrayList<>(); - for (Keyspace ks : Keyspace.all()) - { - for (ColumnFamilyStore cfStore: ks.getColumnFamilyStores()) - result.addAll(cfStore.getLiveSSTables()); - } - - return result; - } - /** * Marks the non-compacting sstables as compacting for index summary redistribution for all keyspaces/tables. * @@ -221,7 +223,12 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean allCompacting.addAll(Sets.difference(allSSTables, nonCompacting)); } } - long nonRedistributingOffHeapSize = allCompacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum(); + long nonRedistributingOffHeapSize = allCompacting.stream() + .filter(IndexSummarySupport.class::isInstance) + .map(IndexSummarySupport.class::cast) + .map(IndexSummarySupport::getIndexSummary) + .mapToLong(IndexSummary::getOffHeapSize) + .sum(); return Pair.create(nonRedistributingOffHeapSize, allNonCompacting); } @@ -272,9 +279,9 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean * @return a list of new SSTableReader instances */ @VisibleForTesting - public static List redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException + public static List redistributeSummaries(IndexSummaryRedistribution redistribution) throws IOException { - return CompactionManager.instance.runIndexSummaryRedistribution(redistribution); + return CompactionManager.instance.runAsActiveCompaction(redistribution, redistribution::redistributeSummaries); } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java similarity index 97% rename from src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java rename to src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java index 9ba3d40044..2179544cf0 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerMBean.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.util.Map; diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java new file mode 100644 index 0000000000..8855e413f4 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryMetrics.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.indexsummary; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cassandra.io.sstable.AbstractMetricsProviders; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +public class IndexSummaryMetrics> extends AbstractMetricsProviders +{ + public final static IndexSummaryMetrics instance = new IndexSummaryMetrics<>(); + + @Override + protected R map(SSTableReader r) + { + if (r instanceof IndexSummarySupport) + return (R) r; + return null; + } + + private final GaugeProvider indexSummaryOffHeapMemoryUsed = newGaugeProvider("IndexSummaryOffHeapMemoryUsed", + 0L, + r -> r.getIndexSummary().getOffHeapSize(), + Long::sum); + + private final List> gaugeProviders = Arrays.asList(indexSummaryOffHeapMemoryUsed); + + public List> getGaugeProviders() + { + return gaugeProviders; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java similarity index 76% rename from src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java rename to src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java index 74ae43daaf..17d1f96888 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.util.ArrayList; @@ -28,16 +28,17 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionInfo.Unit; import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.compaction.CompactionInfo.Unit; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.io.sstable.Downsampling; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.schema.TableId; @@ -81,26 +82,42 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder this.compactionId = nextTimeUUID(); } - public List redistributeSummaries() throws IOException + private static > List getIndexSummarySupportingAndCloseOthers(LifecycleTransaction txn) + { + List filtered = new ArrayList<>(); + List cancels = new ArrayList<>(); + for (SSTableReader sstable : txn.originals()) + { + if (sstable instanceof IndexSummarySupport) + filtered.add((T) sstable); + else + cancels.add(sstable); + } + txn.cancel(cancels); + return filtered; + } + + public > List redistributeSummaries() throws IOException { long start = nanoTime(); logger.info("Redistributing index summaries"); - List redistribute = new ArrayList<>(); + List redistribute = new ArrayList<>(); + for (LifecycleTransaction txn : transactions.values()) { - redistribute.addAll(txn.originals()); + redistribute.addAll(getIndexSummarySupportingAndCloseOthers(txn)); } long total = nonRedistributingOffHeapSize; - for (SSTableReader sstable : redistribute) - total += sstable.getIndexSummaryOffHeapSize(); + for (T sstable : redistribute) + total += sstable.getIndexSummary().getOffHeapSize(); logger.info("Beginning redistribution of index summaries for {} sstables with memory pool size {} MiB; current spaced used is {} MiB", redistribute.size(), memoryPoolBytes / 1024L / 1024L, total / 1024.0 / 1024.0); - final Map readRates = new HashMap<>(redistribute.size()); + final Map readRates = new HashMap<>(redistribute.size()); double totalReadsPerSec = 0.0; - for (SSTableReader sstable : redistribute) + for (T sstable : redistribute) { if (isStopRequested()) throw new CompactionInterruptedException(getCompactionInfo()); @@ -115,14 +132,14 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder logger.trace("Total reads/sec across all sstables in index summary resize process: {}", totalReadsPerSec); // copy and sort by read rates (ascending) - List sstablesByHotness = new ArrayList<>(redistribute); + List sstablesByHotness = new ArrayList<>(redistribute); Collections.sort(sstablesByHotness, new ReadRateComparator(readRates)); long remainingBytes = memoryPoolBytes - nonRedistributingOffHeapSize; logger.trace("Index summaries for compacting SSTables are using {} MiB of space", (memoryPoolBytes - remainingBytes) / 1024.0 / 1024.0); - List newSSTables; + List newSSTables; try (Refs refs = Refs.ref(sstablesByHotness)) { newSSTables = adjustSamplingLevels(sstablesByHotness, transactions, totalReadsPerSec, remainingBytes); @@ -131,8 +148,8 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder txn.finish(); } total = nonRedistributingOffHeapSize; - for (SSTableReader sstable : newSSTables) - total += sstable.getIndexSummaryOffHeapSize(); + for (T sstable : newSSTables) + total += sstable.getIndexSummary().getOffHeapSize(); logger.info("Completed resizing of index summaries; current approximate memory used: {} MiB, time spent: {}ms", total / 1024.0 / 1024.0, TimeUnit.NANOSECONDS.toMillis(nanoTime() - start)); @@ -140,20 +157,21 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder return newSSTables; } - private List adjustSamplingLevels(List sstables, - Map transactions, - double totalReadsPerSec, long memoryPoolCapacity) throws IOException + private > List adjustSamplingLevels(List sstables, + Map transactions, + double totalReadsPerSec, + long memoryPoolCapacity) throws IOException { - List toDownsample = new ArrayList<>(sstables.size() / 4); - List toUpsample = new ArrayList<>(sstables.size() / 4); - List forceResample = new ArrayList<>(); - List forceUpsample = new ArrayList<>(); - List newSSTables = new ArrayList<>(sstables.size()); + List> toDownsample = new ArrayList<>(sstables.size() / 4); + List> toUpsample = new ArrayList<>(sstables.size() / 4); + List> forceResample = new ArrayList<>(); + List> forceUpsample = new ArrayList<>(); + List newSSTables = new ArrayList<>(sstables.size()); // Going from the coldest to the hottest sstables, try to give each sstable an amount of space proportional // to the number of total reads/sec it handles. remainingSpace = memoryPoolCapacity; - for (SSTableReader sstable : sstables) + for (T sstable : sstables) { if (isStopRequested()) throw new CompactionInterruptedException(getCompactionInfo()); @@ -165,26 +183,26 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder long idealSpace = Math.round(remainingSpace * (readsPerSec / totalReadsPerSec)); // figure out how many entries our idealSpace would buy us, and pick a new sampling level based on that - int currentNumEntries = sstable.getIndexSummarySize(); - double avgEntrySize = sstable.getIndexSummaryOffHeapSize() / (double) currentNumEntries; + int currentNumEntries = sstable.getIndexSummary().size(); + double avgEntrySize = sstable.getIndexSummary().getOffHeapSize() / (double) currentNumEntries; long targetNumEntries = Math.max(1, Math.round(idealSpace / avgEntrySize)); - int currentSamplingLevel = sstable.getIndexSummarySamplingLevel(); - int maxSummarySize = sstable.getMaxIndexSummarySize(); + int currentSamplingLevel = sstable.getIndexSummary().getSamplingLevel(); + int maxSummarySize = sstable.getIndexSummary().getMaxNumberOfEntries(); // if the min_index_interval changed, calculate what our current sampling level would be under the new min - if (sstable.getMinIndexInterval() != minIndexInterval) + if (sstable.getIndexSummary().getMinIndexInterval() != minIndexInterval) { - int effectiveSamplingLevel = (int) Math.round(currentSamplingLevel * (minIndexInterval / (double) sstable.getMinIndexInterval())); - maxSummarySize = (int) Math.round(maxSummarySize * (sstable.getMinIndexInterval() / (double) minIndexInterval)); + int effectiveSamplingLevel = (int) Math.round(currentSamplingLevel * (minIndexInterval / (double) sstable.getIndexSummary().getMinIndexInterval())); + maxSummarySize = (int) Math.round(maxSummarySize * (sstable.getIndexSummary().getMinIndexInterval() / (double) minIndexInterval)); logger.trace("min_index_interval changed from {} to {}, so the current sampling level for {} is effectively now {} (was {})", - sstable.getMinIndexInterval(), minIndexInterval, sstable, effectiveSamplingLevel, currentSamplingLevel); + sstable.getIndexSummary().getMinIndexInterval(), minIndexInterval, sstable, effectiveSamplingLevel, currentSamplingLevel); currentSamplingLevel = effectiveSamplingLevel; } int newSamplingLevel = IndexSummaryBuilder.calculateSamplingLevel(currentSamplingLevel, currentNumEntries, targetNumEntries, minIndexInterval, maxIndexInterval); int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, maxSummarySize); - double effectiveIndexInterval = sstable.getEffectiveIndexInterval(); + double effectiveIndexInterval = sstable.getIndexSummary().getEffectiveIndexInterval(); if (logger.isTraceEnabled()) logger.trace("{} has {} reads/sec; ideal space for index summary: {} ({} entries); considering moving " + @@ -200,7 +218,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder logger.trace("Forcing resample of {} because the current index interval ({}) is below min_index_interval ({})", sstable, effectiveIndexInterval, minIndexInterval); long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel); - forceResample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); + forceResample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); remainingSpace -= spaceUsed; } else if (effectiveIndexInterval > maxIndexInterval) @@ -209,28 +227,28 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder logger.trace("Forcing upsample of {} because the current index interval ({}) is above max_index_interval ({})", sstable, effectiveIndexInterval, maxIndexInterval); newSamplingLevel = Math.max(1, (BASE_SAMPLING_LEVEL * minIndexInterval) / maxIndexInterval); - numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getMaxIndexSummarySize()); + numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getIndexSummary().getMaxNumberOfEntries()); long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel); - forceUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); + forceUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel; } else if (targetNumEntries >= currentNumEntries * UPSAMPLE_THRESHOLD && newSamplingLevel > currentSamplingLevel) { long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel); - toUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); + toUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel; } else if (targetNumEntries < currentNumEntries * DOWNSAMPLE_THESHOLD && newSamplingLevel < currentSamplingLevel) { long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel); - toDownsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); + toDownsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel)); remainingSpace -= spaceUsed; } else { // keep the same sampling level logger.trace("SSTable {} is within thresholds of ideal sampling", sstable); - remainingSpace -= sstable.getIndexSummaryOffHeapSize(); + remainingSpace -= sstable.getIndexSummary().getOffHeapSize(); newSSTables.add(sstable); transactions.get(sstable.metadata().id).cancel(sstable); } @@ -239,10 +257,10 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder if (remainingSpace > 0) { - Pair, List> result = distributeRemainingSpace(toDownsample, remainingSpace); + Pair, List>> result = distributeRemainingSpace(toDownsample, remainingSpace); toDownsample = result.right; newSSTables.addAll(result.left); - for (SSTableReader sstable : result.left) + for (T sstable : result.left) transactions.get(sstable.metadata().id).cancel(sstable); } @@ -251,20 +269,20 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder toDownsample.addAll(forceResample); toDownsample.addAll(toUpsample); toDownsample.addAll(forceUpsample); - for (ResampleEntry entry : toDownsample) + for (ResampleEntry entry : toDownsample) { if (isStopRequested()) throw new CompactionInterruptedException(getCompactionInfo()); - SSTableReader sstable = entry.sstable; + T sstable = entry.sstable; logger.trace("Re-sampling index summary for {} from {}/{} to {}/{} of the original number of entries", - sstable, sstable.getIndexSummarySamplingLevel(), Downsampling.BASE_SAMPLING_LEVEL, + sstable, sstable.getIndexSummary().getSamplingLevel(), Downsampling.BASE_SAMPLING_LEVEL, entry.newSamplingLevel, Downsampling.BASE_SAMPLING_LEVEL); ColumnFamilyStore cfs = Keyspace.open(sstable.metadata().keyspace).getColumnFamilyStore(sstable.metadata().id); long oldSize = sstable.bytesOnDisk(); long oldSizeUncompressed = sstable.logicalBytesOnDisk(); - SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(cfs, entry.newSamplingLevel); + T replacement = sstable.cloneWithNewSummarySamplingLevel(cfs, entry.newSamplingLevel); long newSize = replacement.bytesOnDisk(); long newSizeUncompressed = replacement.logicalBytesOnDisk(); @@ -307,31 +325,24 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder } @VisibleForTesting - static Pair, List> distributeRemainingSpace(List toDownsample, long remainingSpace) + static > Pair, List>> distributeRemainingSpace(List> toDownsample, long remainingSpace) { // sort by the amount of space regained by doing the downsample operation; we want to try to avoid operations // that will make little difference. - Collections.sort(toDownsample, new Comparator() - { - public int compare(ResampleEntry o1, ResampleEntry o2) - { - return Double.compare(o1.sstable.getIndexSummaryOffHeapSize() - o1.newSpaceUsed, - o2.sstable.getIndexSummaryOffHeapSize() - o2.newSpaceUsed); - } - }); + Collections.sort(toDownsample, Comparator.comparingDouble(o -> o.sstable.getIndexSummary().getOffHeapSize() - o.newSpaceUsed)); int noDownsampleCutoff = 0; - List willNotDownsample = new ArrayList<>(); + List willNotDownsample = new ArrayList<>(); while (remainingSpace > 0 && noDownsampleCutoff < toDownsample.size()) { - ResampleEntry entry = toDownsample.get(noDownsampleCutoff); + ResampleEntry entry = toDownsample.get(noDownsampleCutoff); - long extraSpaceRequired = entry.sstable.getIndexSummaryOffHeapSize() - entry.newSpaceUsed; + long extraSpaceRequired = entry.sstable.getIndexSummary().getOffHeapSize() - entry.newSpaceUsed; // see if we have enough leftover space to keep the current sampling level if (extraSpaceRequired <= remainingSpace) { logger.trace("Using leftover space to keep {} at the current sampling level ({})", - entry.sstable, entry.sstable.getIndexSummarySamplingLevel()); + entry.sstable, entry.sstable.getIndexSummary().getSamplingLevel()); willNotDownsample.add(entry.sstable); remainingSpace -= extraSpaceRequired; } @@ -358,9 +369,9 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder /** Utility class for sorting sstables by their read rates. */ private static class ReadRateComparator implements Comparator { - private final Map readRates; + private final Map readRates; - ReadRateComparator(Map readRates) + ReadRateComparator(Map readRates) { this.readRates = readRates; } @@ -381,13 +392,13 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder } } - private static class ResampleEntry + private static class ResampleEntry> { - public final SSTableReader sstable; + public final T sstable; public final long newSpaceUsed; public final int newSamplingLevel; - ResampleEntry(SSTableReader sstable, long newSpaceUsed, int newSamplingLevel) + ResampleEntry(T sstable, long newSpaceUsed, int newSamplingLevel) { this.sstable = sstable; this.newSpaceUsed = newSpaceUsed; diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java new file mode 100644 index 0000000000..b4cba50cb5 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.indexsummary; + +import java.io.IOException; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +public interface IndexSummarySupport> +{ + IndexSummary getIndexSummary(); + + T cloneWithNewSummarySamplingLevel(ColumnFamilyStore cfs, int newSamplingLevel) throws IOException; + + static boolean isSupportedBy(SSTableFormat.Type formatType) + { + return IndexSummarySupport.class.isAssignableFrom(formatType.info.getReaderFactory().getReaderClass()); + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java new file mode 100644 index 0000000000..7ed1955b3f --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCache.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.keycache; + +import java.util.concurrent.atomic.LongAdder; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cache.InstrumentingCache; +import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; + +/** + * A simple wrapper of possibly global cache with local metrics. + */ +public class KeyCache +{ + public static final KeyCache NO_CACHE = new KeyCache(null); + + private final static Logger logger = LoggerFactory.getLogger(KeyCache.class); + + private final InstrumentingCache cache; + private final LongAdder hits = new LongAdder(); + private final LongAdder requests = new LongAdder(); + + public KeyCache(@Nullable InstrumentingCache cache) + { + this.cache = cache; + } + + public long getHits() + { + return cache != null ? hits.sum() : 0; + } + + public long getRequests() + { + return cache != null ? requests.sum() : 0; + } + + public void put(@Nonnull KeyCacheKey cacheKey, @Nonnull AbstractRowIndexEntry info) + { + if (cache == null) + return; + + logger.trace("Adding cache entry for {} -> {}", cacheKey, info); + cache.put(cacheKey, info); + } + + public @Nullable AbstractRowIndexEntry get(KeyCacheKey key, boolean updateStats) + { + if (cache == null) + return null; + + if (updateStats) + { + requests.increment(); + AbstractRowIndexEntry r = cache.get(key); + if (r != null) + hits.increment(); + return r; + } + else + { + return cache.getInternal(key); + } + } + + public boolean isEnabled() + { + return cache != null; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java new file mode 100644 index 0000000000..7ba1a68049 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheMetrics.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.keycache; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cassandra.io.sstable.AbstractMetricsProviders; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +public class KeyCacheMetrics> extends AbstractMetricsProviders +{ + public final static KeyCacheMetrics instance = new KeyCacheMetrics<>(); + + @Override + protected R map(SSTableReader r) + { + if (r instanceof KeyCacheSupport) + return (R) r; + return null; + } + + /** Key cache hit rate for this CF */ + private final GaugeProvider keyCacheHitRate = newGaugeProvider("KeyCacheHitRate", readers -> { + long hits = 0L; + long requests = 0L; + for (R sstable : readers) + { + hits += sstable.getKeyCache().getHits(); + requests += sstable.getKeyCache().getRequests(); + } + + return (double) hits / (double) Math.max(1, requests); + }); + + private final List> gaugeProviders = Arrays.asList(keyCacheHitRate); + + public List> getGaugeProviders() + { + return gaugeProviders; + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java new file mode 100644 index 0000000000..f9c34d5d09 --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable.keycache; + +import java.io.IOException; +import java.nio.ByteBuffer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.DataInputPlus; + +public interface KeyCacheSupport> +{ + @Nonnull + KeyCache getKeyCache(); + + /** + * Should quickly get a lower bound prefix from cache only if everything is already availabe in memory and does not + * need to be loaded from disk. + */ + @Nullable + ClusteringBound getLowerBoundPrefixFromCache(DecoratedKey partitionKey, boolean isReversed); + + default @Nonnull KeyCacheKey getCacheKey(ByteBuffer key) + { + T reader = (T) this; + return new KeyCacheKey(reader.metadata(), reader.descriptor, key); + } + + default @Nonnull KeyCacheKey getCacheKey(DecoratedKey key) + { + return getCacheKey(key.getKey()); + } + + /** + * Will return null if key is not found or cache is not available. + */ + default @Nullable AbstractRowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats) + { + return getCachedPosition(getCacheKey(key), updateStats); + } + + /** + * Will return null if key is not found or cache is not available. + */ + default @Nullable AbstractRowIndexEntry getCachedPosition(KeyCacheKey key, boolean updateStats) + { + KeyCache keyCache = getKeyCache(); + AbstractRowIndexEntry cachedEntry = keyCache.get(key, updateStats); + assert cachedEntry == null || cachedEntry.getSSTableFormat() == ((T) this).descriptor.formatType.info; + + return cachedEntry; + } + + /** + * Caches a key only if cache is available. + */ + default void cacheKey(@Nonnull DecoratedKey key, @Nonnull AbstractRowIndexEntry info) + { + T reader = (T) this; + assert info.getSSTableFormat() == reader.descriptor.formatType.info; + + KeyCacheKey cacheKey = getCacheKey(key); + getKeyCache().put(cacheKey, info); + } + + @Nonnull + AbstractRowIndexEntry deserializeKeyCacheValue(@Nonnull DataInputPlus input) throws IOException; + + static boolean isSupportedBy(SSTableFormat.Type formatType) + { + return KeyCacheSupport.class.isAssignableFrom(formatType.info.getReaderFactory().getReaderClass()); + } + +} diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index 6ab1551898..d14ebb2a16 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@ -55,6 +55,8 @@ public class MetadataCollector implements PartitionStatisticsCollector { public static final double NO_COMPRESSION_RATIO = -1.0; + private long currentPartitionCells = 0; + static EstimatedHistogram defaultCellPerPartitionCountHistogram() { // EH of 118 can track a max value of 4139110981, i.e., > 4B cells @@ -194,6 +196,13 @@ public class MetadataCollector implements PartitionStatisticsCollector return this; } + public MetadataCollector addCellPerPartitionCount() + { + estimatedCellPerPartitionCount.add(currentPartitionCells); + currentPartitionCells = 0; + return this; + } + /** * Ratio is compressed/uncompressed and it is * if you have 1.x then compression isn't helping @@ -218,6 +227,7 @@ public class MetadataCollector implements PartitionStatisticsCollector public void update(Cell cell) { + ++currentPartitionCells; updateTimestamp(cell.timestamp()); updateTTL(cell.ttl()); updateLocalDeletionTime(cell.localDeletionTime()); @@ -231,6 +241,7 @@ public class MetadataCollector implements PartitionStatisticsCollector hasPartitionLevelDeletions = true; update(dt); } + public void update(DeletionTime dt) { if (!dt.isLive()) @@ -473,4 +484,4 @@ public class MetadataCollector implements PartitionStatisticsCollector return isSet ? max : defaultMax; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index f13daa48ac..5ecb582b04 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -17,25 +17,33 @@ */ package org.apache.cassandra.io.sstable.metadata; -import org.apache.cassandra.io.util.*; - import java.io.FileNotFoundException; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; import java.util.function.UnaryOperator; import java.util.zip.CRC32; import com.google.common.base.Throwables; import com.google.common.collect.Lists; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; @@ -113,7 +121,7 @@ public class MetadataSerializer implements IMetadataSerializer { Map components; logger.trace("Load metadata for {}", descriptor); - File statsFile = new File(descriptor.filenameFor(Component.STATS)); + File statsFile = descriptor.fileFor(Components.STATS); if (!statsFile.exists()) { logger.trace("No sstable stats for {}", descriptor); @@ -213,8 +221,8 @@ public class MetadataSerializer implements IMetadataSerializer if (actualChecksum != expectedChecksum) { - String filename = descriptor.filenameFor(Component.STATS); - throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename); + File file = descriptor.fileFor(Components.STATS); + throw new CorruptSSTableException(new IOException("Checksums do not match for " + file), file); } } @@ -222,7 +230,7 @@ public class MetadataSerializer implements IMetadataSerializer public void mutate(Descriptor descriptor, String description, UnaryOperator transform) throws IOException { if (logger.isTraceEnabled() ) - logger.trace("Mutating {} to {}", descriptor.filenameFor(Component.STATS), description); + logger.trace("Mutating {} to {}", descriptor.fileFor(Components.STATS), description); mutate(descriptor, transform); } @@ -231,7 +239,7 @@ public class MetadataSerializer implements IMetadataSerializer public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException { if (logger.isTraceEnabled()) - logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel); + logger.trace("Mutating {} to level {}", descriptor.fileFor(Components.STATS), newLevel); mutate(descriptor, stats -> stats.mutateLevel(newLevel)); } @@ -241,7 +249,7 @@ public class MetadataSerializer implements IMetadataSerializer { if (logger.isTraceEnabled()) logger.trace("Mutating {} to repairedAt time {} and pendingRepair {}", - descriptor.filenameFor(Component.STATS), newRepairedAt, newPendingRepair); + descriptor.fileFor(Components.STATS), newRepairedAt, newPendingRepair); mutate(descriptor, stats -> stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair, isTransient)); } @@ -257,8 +265,8 @@ public class MetadataSerializer implements IMetadataSerializer public void rewriteSSTableMetadata(Descriptor descriptor, Map currentComponents) throws IOException { - String filePath = descriptor.tmpFilenameFor(Component.STATS); - try (DataOutputStreamPlus out = new FileOutputStreamPlus(filePath)) + File file = descriptor.tmpFileFor(Components.STATS); + try (DataOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE)) { serialize(currentComponents, out, descriptor.version); out.flush(); @@ -266,8 +274,8 @@ public class MetadataSerializer implements IMetadataSerializer catch (IOException e) { Throwables.throwIfInstanceOf(e, FileNotFoundException.class); - throw new FSWriteError(e, filePath); + throw new FSWriteError(e, file); } - FileUtils.renameWithConfirm(filePath, descriptor.filenameFor(Component.STATS)); + file.move(descriptor.fileFor(Components.STATS)); } } diff --git a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java index dba89b07f2..96257511aa 100644 --- a/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java +++ b/src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java @@ -85,6 +85,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void write(byte[] b, int off, int len) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (b == null) throw new NullPointerException(); @@ -121,6 +122,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void write(ByteBuffer src) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; int srcPos = src.position(); int srcCount; int trgAvailable; @@ -138,6 +140,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void write(int b) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (!buffer.hasRemaining()) doFlush(1); buffer.put((byte) (b & 0xFF)); @@ -146,6 +149,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeBoolean(boolean v) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (!buffer.hasRemaining()) doFlush(1); buffer.put(v ? (byte)1 : (byte)0); @@ -160,6 +164,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeBytes(long register, int bytes) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (buffer.remaining() < Long.BYTES) { super.writeBytes(register, bytes); @@ -181,6 +186,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeChar(int v) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (buffer.remaining() < 2) writeSlow(v, 2); else @@ -190,6 +196,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeInt(int v) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (buffer.remaining() < 4) writeSlow(v, 4); else @@ -199,6 +206,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeLong(long v) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; if (buffer.remaining() < 8) writeSlow(v, 8); else @@ -220,6 +228,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @DontInline private void writeSlow(long bytes, int count) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; int origCount = count; if (ByteOrder.BIG_ENDIAN == buffer.order()) while (count > 0) writeByte((int) (bytes >>> (8 * --count))); @@ -244,6 +253,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @Override public void writeUTF(String s) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; UnbufferedDataOutputStreamPlus.writeUTF(s, this); } @@ -253,6 +263,7 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus @DontInline protected void doFlush(int count) throws IOException { + assert buffer != null : "Attempt to use a closed data output"; buffer.flip(); while (buffer.hasRemaining()) @@ -281,7 +292,8 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus public BufferedDataOutputStreamPlus order(ByteOrder order) { + assert buffer != null : "Attempt to use a closed data output"; this.buffer.order(order); return this; } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/util/ChannelProxy.java b/src/java/org/apache/cassandra/io/util/ChannelProxy.java index 717def75d6..81665beecd 100644 --- a/src/java/org/apache/cassandra/io/util/ChannelProxy.java +++ b/src/java/org/apache/cassandra/io/util/ChannelProxy.java @@ -40,6 +40,7 @@ import org.apache.cassandra.utils.concurrent.SharedCloseableImpl; */ public final class ChannelProxy extends SharedCloseableImpl { + private final File file; private final String filePath; private final FileChannel channel; @@ -62,14 +63,15 @@ public final class ChannelProxy extends SharedCloseableImpl public ChannelProxy(File file) { - this(file.path(), openChannel(file)); + this(file, openChannel(file)); } - public ChannelProxy(String filePath, FileChannel channel) + public ChannelProxy(File file, FileChannel channel) { - super(new Cleanup(filePath, channel)); + super(new Cleanup(file.path(), channel)); - this.filePath = filePath; + this.file = file; + this.filePath = file.path(); this.channel = channel; } @@ -77,6 +79,7 @@ public final class ChannelProxy extends SharedCloseableImpl { super(copy); + this.file = copy.file; this.filePath = copy.filePath; this.channel = copy.channel; } @@ -130,6 +133,11 @@ public final class ChannelProxy extends SharedCloseableImpl return filePath; } + public File file() + { + return file; + } + public int read(ByteBuffer buffer, long position) { try diff --git a/src/java/org/apache/cassandra/io/util/DataInputPlus.java b/src/java/org/apache/cassandra/io/util/DataInputPlus.java index 50d0b4790b..d117c7fe89 100644 --- a/src/java/org/apache/cassandra/io/util/DataInputPlus.java +++ b/src/java/org/apache/cassandra/io/util/DataInputPlus.java @@ -17,7 +17,10 @@ */ package org.apache.cassandra.io.util; -import java.io.*; +import java.io.DataInput; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; import org.apache.cassandra.utils.Shared; import org.apache.cassandra.utils.vint.VIntCoding; @@ -97,14 +100,8 @@ public interface DataInputPlus extends DataInput /** * Wrapper around an InputStream that provides no buffering but can decode varints - * - * TODO: probably shouldn't use DataInputStream as a parent */ - public class DataInputStreamPlus extends DataInputStream implements DataInputPlus + abstract class DataInputStreamPlus extends InputStream implements DataInputPlus { - public DataInputStreamPlus(InputStream is) - { - super(is); - } } } diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java index 65d4e58f0f..aef3614da6 100644 --- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java +++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java @@ -23,51 +23,30 @@ import java.nio.ByteBuffer; import java.util.zip.CheckedInputStream; import java.util.zip.Checksum; -import com.google.common.annotations.VisibleForTesting; - -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.utils.ChecksumType; -import org.apache.cassandra.utils.Throwables; public class DataIntegrityMetadata { - public static ChecksumValidator checksumValidator(Descriptor desc) throws IOException - { - return new ChecksumValidator(desc); - } - public static class ChecksumValidator implements Closeable { private final ChecksumType checksumType; private final RandomAccessReader reader; public final int chunkSize; - private final String dataFilename; - public ChecksumValidator(Descriptor descriptor) throws IOException + public ChecksumValidator(File dataFile, File crcFile) throws IOException { this(ChecksumType.CRC32, - RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))), - descriptor.filenameFor(Component.DATA)); + RandomAccessReader.open(crcFile), + dataFile.absolutePath()); } public ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, String dataFilename) throws IOException { this.checksumType = checksumType; this.reader = reader; - this.dataFilename = dataFilename; chunkSize = reader.readInt(); } - @VisibleForTesting - protected ChecksumValidator(ChecksumType checksumType, RandomAccessReader reader, int chunkSize) - { - this.checksumType = checksumType; - this.reader = reader; - this.dataFilename = null; - this.chunkSize = chunkSize; - } - public void seek(long offset) { long start = chunkStart(offset); @@ -82,10 +61,10 @@ public class DataIntegrityMetadata public void validate(byte[] bytes, int start, int end) throws IOException { - int current = (int) checksumType.of(bytes, start, end); - int actual = reader.readInt(); - if (current != actual) - throw new IOException("Corrupted File : " + dataFilename); + int calculatedValue = (int) checksumType.of(bytes, start, end); + int storedValue = reader.readInt(); + if (calculatedValue != storedValue) + throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue)); } /** @@ -96,10 +75,10 @@ public class DataIntegrityMetadata */ public void validate(ByteBuffer buffer) throws IOException { - int current = (int) checksumType.of(buffer); - int actual = reader.readInt(); - if (current != actual) - throw new IOException("Corrupted File : " + dataFilename); + int calculatedValue = (int) checksumType.of(buffer); + int storedValue = reader.readInt(); + if (calculatedValue != storedValue) + throw new IOException(String.format("Corrupted file: integrity check (%s) failed for %s: %d != %d", checksumType.name(), reader.getPath(), storedValue, calculatedValue)); } public void close() @@ -108,55 +87,33 @@ public class DataIntegrityMetadata } } - public static FileDigestValidator fileDigestValidator(Descriptor desc) throws IOException - { - return new FileDigestValidator(desc); - } - - public static class FileDigestValidator implements Closeable + public static class FileDigestValidator { private final Checksum checksum; - private final RandomAccessReader digestReader; - private final RandomAccessReader dataReader; - private final Descriptor descriptor; - private long storedDigestValue; + private final File dataFile; + private final File digestFile; - public FileDigestValidator(Descriptor descriptor) throws IOException + public FileDigestValidator(File dataFile, File digestFile) throws IOException { - this.descriptor = descriptor; - checksum = ChecksumType.CRC32.newInstance(); - digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DIGEST))); - dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA))); - try - { - storedDigestValue = Long.parseLong(digestReader.readLine()); - } - catch (Exception e) - { - close(); - // Attempting to create a FileDigestValidator without a DIGEST file will fail - throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA)); - } + this.dataFile = dataFile; + this.digestFile = digestFile; + this.checksum = ChecksumType.CRC32.newInstance(); } // Validate the entire file public void validate() throws IOException { - CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum); - byte[] chunk = new byte[64 * 1024]; - - while( checkedInputStream.read(chunk) > 0 ) { } - long calculatedDigestValue = checkedInputStream.getChecksum().getValue(); - if (storedDigestValue != calculatedDigestValue) + try (RandomAccessReader digestReader = RandomAccessReader.open(digestFile); + RandomAccessReader dataReader = RandomAccessReader.open(dataFile); + CheckedInputStream checkedInputStream = new CheckedInputStream(dataReader, checksum);) { - throw new IOException("Corrupted SSTable : " + descriptor.filenameFor(Component.DATA)); + long storedDigestValue = Long.parseLong(digestReader.readLine()); + byte[] chunk = new byte[64 * 1024]; + while (checkedInputStream.read(chunk) > 0) ; + long calculatedDigestValue = checkedInputStream.getChecksum().getValue(); + if (storedDigestValue != calculatedDigestValue) + throw new IOException(String.format("Corrupted file: integrity check (digest) failed for %s: %d != %d", dataFile, storedDigestValue, calculatedDigestValue)); } } - - public void close() - { - Throwables.perform(digestReader::close, - dataReader::close); - } } } diff --git a/src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java similarity index 58% rename from src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java rename to src/java/org/apache/cassandra/io/util/EmptyRebufferer.java index b046e841b6..aa8e7e046f 100644 --- a/src/java/org/apache/cassandra/utils/AlwaysPresentFilter.java +++ b/src/java/org/apache/cassandra/io/util/EmptyRebufferer.java @@ -15,42 +15,57 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.utils; -import org.apache.cassandra.utils.concurrent.Ref; +package org.apache.cassandra.io.util; -public class AlwaysPresentFilter implements IFilter +public class EmptyRebufferer implements Rebufferer, RebuffererFactory { - public boolean isPresent(FilterKey key) + private final ChannelProxy channel; + + public EmptyRebufferer(ChannelProxy channel) { - return true; + this.channel = channel; } - public void add(FilterKey key) { } - - public void clear() { } - - public void close() { } - - public IFilter sharedCopy() - { - return this; - } - - public Throwable close(Throwable accumulate) - { - return accumulate; - } - - public void addTo(Ref.IdentityCollection identities) - { - } - - public long serializedSize() { return 0; } - @Override - public long offHeapSize() + public void close() + { + + } + + @Override + public ChannelProxy channel() + { + return channel; + } + + @Override + public long fileLength() { return 0; } + + @Override + public double getCrcCheckChance() + { + return 0; + } + + @Override + public BufferHolder rebuffer(long position) + { + return EMPTY; + } + + @Override + public void closeReader() + { + + } + + @Override + public Rebufferer instantiateRebufferer() + { + return this; + } } diff --git a/src/java/org/apache/cassandra/io/util/File.java b/src/java/org/apache/cassandra/io/util/File.java index 71f8fc589e..3b6bbb68a4 100644 --- a/src/java/org/apache/cassandra/io/util/File.java +++ b/src/java/org/apache/cassandra/io/util/File.java @@ -23,7 +23,12 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; import java.nio.channels.FileChannel; -import java.nio.file.*; // checkstyle: permit this import +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Objects; import java.util.function.BiPredicate; import java.util.function.Consumer; @@ -31,7 +36,6 @@ import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.stream.Stream; - import javax.annotation.Nullable; import com.google.common.util.concurrent.RateLimiter; @@ -748,6 +752,13 @@ public class File implements Comparable return new FileInputStreamPlus(this); } + public File withSuffix(String suffix) + { + if (path == null) + throw new IllegalStateException("Cannot suffix an empty path"); + return new File(path.getParent().resolve(path.getFileName().toString() + suffix)); + } + private Path toPathForWrite() { if (path == null) diff --git a/src/java/org/apache/cassandra/io/util/FileHandle.java b/src/java/org/apache/cassandra/io/util/FileHandle.java index 6bab46079a..5c4bd727a8 100644 --- a/src/java/org/apache/cassandra/io/util/FileHandle.java +++ b/src/java/org/apache/cassandra/io/util/FileHandle.java @@ -17,31 +17,29 @@ */ package org.apache.cassandra.io.util; -import java.util.Objects; import java.util.Optional; +import java.util.function.Function; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.RateLimiter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.config.Config; import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.io.compress.CompressionMetadata; import org.apache.cassandra.utils.NativeLibrary; +import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.RefCounted; import org.apache.cassandra.utils.concurrent.SharedCloseableImpl; -import static org.apache.cassandra.utils.Throwables.maybeFail; -import org.apache.cassandra.utils.Throwables; - /** * {@link FileHandle} provides access to a file for reading, including the ones written by various {@link SequentialWriter} * instances, and it is typically used by {@link org.apache.cassandra.io.sstable.format.SSTableReader}. - * + *

* Use {@link FileHandle.Builder} to create an instance, and call {@link #createReader()} (and its variants) to * access the readers for the underlying file. - * + *

* You can use {@link Builder#complete()} several times during its lifecycle with different {@code overrideLength}(i.e. early opening file). * For that reason, the builder keeps a reference to the file channel and makes a copy for each {@link Builder#complete()} call. * Therefore, it is important to close the {@link Builder} when it is no longer needed, as well as any {@link FileHandle} @@ -49,8 +47,6 @@ import org.apache.cassandra.utils.Throwables; */ public class FileHandle extends SharedCloseableImpl { - private static final Logger logger = LoggerFactory.getLogger(FileHandle.class); - public final ChannelProxy channel; public final long onDiskLength; @@ -88,8 +84,13 @@ public class FileHandle extends SharedCloseableImpl } /** - * @return Path to the file this factory is referencing + * @return file this factory is referencing */ + public File file() + { + return new File(channel.filePath()); + } + public String path() { return channel.filePath(); @@ -114,7 +115,6 @@ public class FileHandle extends SharedCloseableImpl public void addTo(Ref.IdentityCollection identities) { super.addTo(identities); - compressionMetadata.ifPresent(metadata -> metadata.addTo(identities)); } @Override @@ -155,7 +155,14 @@ public class FileHandle extends SharedCloseableImpl } catch (Throwable t) { - try { reader.close(); } catch (Throwable t2) { t.addSuppressed(t2); } + try + { + reader.close(); + } + catch (Throwable t2) + { + t.addSuppressed(t2); + } throw t; } } @@ -173,10 +180,10 @@ public class FileHandle extends SharedCloseableImpl else return metadata.chunkFor(before).offset; }).orElse(before); - NativeLibrary.trySkipCache(channel.getFileDescriptor(), 0, position, path()); + NativeLibrary.trySkipCache(channel.getFileDescriptor(), 0, position, file().absolutePath()); } - private Rebufferer instantiateRebufferer(RateLimiter limiter) + public Rebufferer instantiateRebufferer(RateLimiter limiter) { Rebufferer rebufferer = rebuffererFactory.instantiateRebufferer(); @@ -238,35 +245,23 @@ public class FileHandle extends SharedCloseableImpl /** * Configures how the file will be read (compressed, mmapped, use cache etc.) */ - public static class Builder implements AutoCloseable + public static class Builder { - private final String path; + public static final long NO_LENGTH_OVERRIDE = -1; + + public final File file; - private ChannelProxy channel; private CompressionMetadata compressionMetadata; - private MmappedRegions regions; private ChunkCache chunkCache; private int bufferSize = RandomAccessReader.DEFAULT_BUFFER_SIZE; private BufferType bufferType = BufferType.OFF_HEAP; - private boolean mmapped = false; - private boolean compressed = false; + private long lengthOverride = -1; + private MmappedRegionsCache mmappedRegionsCache; - public Builder(String path) + public Builder(File file) { - this.path = path; - } - - public Builder(ChannelProxy channel) - { - this.channel = channel; - this.path = channel.filePath(); - } - - public Builder compressed(boolean compressed) - { - this.compressed = compressed; - return this; + this.file = file; } /** @@ -283,13 +278,15 @@ public class FileHandle extends SharedCloseableImpl /** * Provide {@link CompressionMetadata} to use when reading compressed file. + * Upon completion, builder will create a shared copy of this object and that copy will be used in the created + * instance of {@link FileHandle}. Therefore, the caller is responsible for closing the instance of + * {@link CompressionMetadata} passed to this method after builder completion. * - * @param metadata CompressionMetadata to use + * @param metadata CompressionMetadata to use, can be {@code null} if no compression is used * @return this object */ public Builder withCompressionMetadata(CompressionMetadata metadata) { - this.compressed = Objects.nonNull(metadata); this.compressionMetadata = metadata; return this; } @@ -306,6 +303,18 @@ public class FileHandle extends SharedCloseableImpl return this; } + public Builder mmapped(Config.DiskAccessMode diskAccessMode) + { + this.mmapped = diskAccessMode == Config.DiskAccessMode.mmap; + return this; + } + + public Builder withMmappedRegionsCache(MmappedRegionsCache mmappedRegionsCache) + { + this.mmappedRegionsCache = mmappedRegionsCache; + return this; + } + /** * Set the buffer size to use (if appropriate). * @@ -331,128 +340,97 @@ public class FileHandle extends SharedCloseableImpl } /** - * Complete building {@link FileHandle} without overriding file length. + * Override the file length. * - * @see #complete(long) - */ - public FileHandle complete() - { - return complete(-1L); - } - - /** - * Complete building {@link FileHandle} with the given length, which overrides the file length. - * - * @param overrideLength Override file length (in bytes) so that read cannot go further than this value. + * @param lengthOverride Override file length (in bytes) so that read cannot go further than this value. * If the value is less than or equal to 0, then the value is ignored. * @return Built file */ - @SuppressWarnings("resource") - public FileHandle complete(long overrideLength) + public Builder withLengthOverride(long lengthOverride) { - boolean channelOpened = false; - if (channel == null) - { - channel = new ChannelProxy(path); - channelOpened = true; - } + this.lengthOverride = lengthOverride; + return this; + } - ChannelProxy channelCopy = channel.sharedCopy(); + /** + * Complete building {@link FileHandle}. + */ + public FileHandle complete() + { + return complete(ChannelProxy::new); + } + + @VisibleForTesting + @SuppressWarnings("resource") + public FileHandle complete(Function channelProxyFactory) + { + ChannelProxy channel = null; + MmappedRegions regions = null; + CompressionMetadata compressionMetadata = null; try { - if (compressed && compressionMetadata == null) - compressionMetadata = CompressionMetadata.create(channelCopy.filePath()); + compressionMetadata = this.compressionMetadata != null ? this.compressionMetadata.sharedCopy() : null; + channel = channelProxyFactory.apply(file); - long length = overrideLength > 0 ? overrideLength : compressed ? compressionMetadata.compressedFileLength : channelCopy.size(); + long fileLength = (compressionMetadata != null) ? compressionMetadata.compressedFileLength : channel.size(); + long length = lengthOverride > 0 ? lengthOverride : fileLength; RebuffererFactory rebuffererFactory; - if (mmapped) + if (length == 0) { - if (compressed) + rebuffererFactory = new EmptyRebufferer(channel); + } + else if (mmapped) + { + if (compressionMetadata != null) { - regions = MmappedRegions.map(channelCopy, compressionMetadata); - rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channelCopy, compressionMetadata, - regions)); + regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, compressionMetadata) + : MmappedRegions.map(channel, compressionMetadata); + rebuffererFactory = maybeCached(new CompressedChunkReader.Mmap(channel, compressionMetadata, regions)); } else { - updateRegions(channelCopy, length); - rebuffererFactory = new MmapRebufferer(channelCopy, length, regions.sharedCopy()); + regions = mmappedRegionsCache != null ? mmappedRegionsCache.getOrCreate(channel, length) + : MmappedRegions.map(channel, length); + rebuffererFactory = new MmapRebufferer(channel, length, regions); } } else { - regions = null; - if (compressed) + if (compressionMetadata != null) { - rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channelCopy, compressionMetadata)); + rebuffererFactory = maybeCached(new CompressedChunkReader.Standard(channel, compressionMetadata)); } else { int chunkSize = DiskOptimizationStrategy.roundForCaching(bufferSize, ChunkCache.roundUp); - rebuffererFactory = maybeCached(new SimpleChunkReader(channelCopy, length, bufferType, chunkSize)); + rebuffererFactory = maybeCached(new SimpleChunkReader(channel, length, bufferType, chunkSize)); } } - Cleanup cleanup = new Cleanup(channelCopy, rebuffererFactory, compressionMetadata, chunkCache); - return new FileHandle(cleanup, channelCopy, rebuffererFactory, compressionMetadata, length); + Cleanup cleanup = new Cleanup(channel, rebuffererFactory, compressionMetadata, chunkCache); + + FileHandle fileHandle = new FileHandle(cleanup, channel, rebuffererFactory, compressionMetadata, length); + return fileHandle; } catch (Throwable t) { - channelCopy.close(); - if (channelOpened) - { - ChannelProxy c = channel; - channel = null; - throw Throwables.cleaned(c.close(t)); - } + Throwables.closeNonNullAndAddSuppressed(t, regions, channel, compressionMetadata); throw t; } } - public Throwable close(Throwable accumulate) - { - if (!compressed && regions != null) - accumulate = regions.close(accumulate); - if (channel != null) - return channel.close(accumulate); - - return accumulate; - } - - public void close() - { - maybeFail(close(null)); - } - private RebuffererFactory maybeCached(ChunkReader reader) { if (chunkCache != null && chunkCache.capacity() > 0) - return chunkCache.maybeWrap(reader); + return chunkCache.wrap(reader); return reader; } - - private void updateRegions(ChannelProxy channel, long length) - { - if (regions != null && !regions.isValid(channel)) - { - Throwable err = regions.close(null); - if (err != null) - logger.error("Failed to close mapped regions", err); - - regions = null; - } - - if (regions == null) - regions = MmappedRegions.map(channel, length); - else - regions.extend(length); - } } @Override public String toString() { - return getClass().getSimpleName() + "(path='" + path() + '\'' + + return getClass().getSimpleName() + "(path='" + file() + '\'' + ", length=" + rebuffererFactory.fileLength() + ')'; } diff --git a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java index 79e8438108..2bd57a99d2 100644 --- a/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java +++ b/src/java/org/apache/cassandra/io/util/FileInputStreamPlus.java @@ -27,37 +27,29 @@ import java.nio.file.Path; public class FileInputStreamPlus extends RebufferingInputStream { final FileChannel channel; + public final File file; public FileInputStreamPlus(String file) throws NoSuchFileException { this(new File(file)); } - public FileInputStreamPlus(File file) throws NoSuchFileException - { - this(file.newReadChannel()); - } - public FileInputStreamPlus(Path path) throws NoSuchFileException { - this(PathUtils.newReadChannel(path)); + this(new File(path)); } - public FileInputStreamPlus(Path path, int bufferSize) throws NoSuchFileException + public FileInputStreamPlus(File file) throws NoSuchFileException { - this(PathUtils.newReadChannel(path), bufferSize); + this(file, 1 << 14); } - private FileInputStreamPlus(FileChannel channel) - { - this(channel, 1 << 14); - } - - private FileInputStreamPlus(FileChannel channel, int bufferSize) + public FileInputStreamPlus(File file, int bufferSize) throws NoSuchFileException { super(ByteBuffer.allocateDirect(bufferSize)); - this.channel = channel; + this.channel = file.newReadChannel(); this.buffer.limit(0); + this.file = file; } @Override diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index b02814f39c..9d8efc79cf 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -35,7 +35,6 @@ import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; @@ -52,9 +51,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.util.concurrent.RateLimiter; import com.google.common.base.Preconditions; - +import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,10 +207,15 @@ public final class FileUtils } public static void createHardLinkWithoutConfirm(String from, String to) + { + createHardLinkWithoutConfirm(new File(from), new File(to)); + } + + public static void createHardLinkWithoutConfirm(File from, File to) { try { - createHardLink(new File(from), new File(to)); + createHardLink(from, to); } catch (FSWriteError fse) { @@ -222,10 +225,15 @@ public final class FileUtils } public static void copyWithOutConfirm(String from, String to) + { + copyWithOutConfirm(new File(from), new File(to)); + } + + public static void copyWithOutConfirm(File from, File to) { try { - Files.copy(Paths.get(from), Paths.get(to)); + Files.copy(from.toPath(), to.toPath()); } catch (IOException e) { @@ -790,4 +798,4 @@ public final class FileUtils } } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java index b5c7f3498d..bcbf2ef58a 100644 --- a/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java +++ b/src/java/org/apache/cassandra/io/util/LimitingRebufferer.java @@ -20,30 +20,28 @@ */ package org.apache.cassandra.io.util; -import java.nio.ByteBuffer; +import javax.annotation.concurrent.NotThreadSafe; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.RateLimiter; /** * Rebufferer wrapper that applies rate limiting. - * + *

* Instantiated once per RandomAccessReader, thread-unsafe. * The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call. + * Only one BufferHolder can be active at a time. Calling {@link #rebuffer(long)} before the previously obtained + * buffer holder is released will throw {@link AssertionError}. */ -public class LimitingRebufferer implements Rebufferer, Rebufferer.BufferHolder +@NotThreadSafe +public class LimitingRebufferer extends WrappingRebufferer { - final private Rebufferer wrapped; final private RateLimiter limiter; final private int limitQuant; - private BufferHolder bufferHolder; - private ByteBuffer buffer; - private long offset; - public LimitingRebufferer(Rebufferer wrapped, RateLimiter limiter, int limitQuant) { - this.wrapped = wrapped; + super(wrapped); this.limiter = limiter; this.limitQuant = limitQuant; } @@ -51,9 +49,7 @@ public class LimitingRebufferer implements Rebufferer, Rebufferer.BufferHolder @Override public BufferHolder rebuffer(long position) { - bufferHolder = wrapped.rebuffer(position); - buffer = bufferHolder.buffer(); - offset = bufferHolder.offset(); + super.rebuffer(position); int posInBuffer = Ints.checkedCast(position - offset); int remaining = buffer.limit() - posInBuffer; if (remaining == 0) @@ -68,59 +64,9 @@ public class LimitingRebufferer implements Rebufferer, Rebufferer.BufferHolder return this; } - @Override - public ChannelProxy channel() - { - return wrapped.channel(); - } - - @Override - public long fileLength() - { - return wrapped.fileLength(); - } - - @Override - public double getCrcCheckChance() - { - return wrapped.getCrcCheckChance(); - } - - @Override - public void close() - { - wrapped.close(); - } - - @Override - public void closeReader() - { - wrapped.closeReader(); - } - @Override public String toString() { return "LimitingRebufferer[" + limiter + "]:" + wrapped; } - - // BufferHolder methods - - @Override - public ByteBuffer buffer() - { - return buffer; - } - - @Override - public long offset() - { - return offset; - } - - @Override - public void release() - { - bufferHolder.release(); - } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java index 0b7dd39353..be64654b27 100644 --- a/src/java/org/apache/cassandra/io/util/MmappedRegions.java +++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java @@ -36,18 +36,25 @@ import static org.apache.cassandra.utils.Throwables.perform; public class MmappedRegions extends SharedCloseableImpl { - /** In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size */ + /** + * In a perfect world, MAX_SEGMENT_SIZE would be final, but we need to test with a smaller size + */ public static int MAX_SEGMENT_SIZE = Integer.MAX_VALUE; - /** When we need to grow the arrays, we add this number of region slots */ + /** + * When we need to grow the arrays, we add this number of region slots + */ static final int REGION_ALLOC_SIZE = 15; - /** The original state, which is shared with the tidier and + /** + * The original state, which is shared with the tidier and * contains all the regions mapped so far. It also - * does the actual mapping. */ + * does the actual mapping. + */ private final State state; - /** A copy of the latest state. We update this each time the original state is + /** + * A copy of the latest state. We update this each time the original state is * updated and we share this with copies. If we are a copy, then this * is null. Copies can only access existing regions, they cannot create * new ones. This is for thread safety and because MmappedRegions is @@ -92,7 +99,7 @@ public class MmappedRegions extends SharedCloseableImpl } /** - * @param channel file to map. the MmappedRegions instance will hold shared copy of given channel. + * @param channel file to map. the MmappedRegions instance will hold shared copy of given channel. * @param metadata * @return new instance */ @@ -126,7 +133,12 @@ public class MmappedRegions extends SharedCloseableImpl return copy == null; } - public void extend(long length) + /** + * Extends this collection of mmapped regions up to the provided total length. + * + * @return {@code true} if new regions have been created + */ + public boolean extend(long length) { if (length < 0) throw new IllegalArgumentException("Length must not be negative"); @@ -134,12 +146,41 @@ public class MmappedRegions extends SharedCloseableImpl assert !isCopy() : "Copies cannot be extended"; if (length <= state.length) - return; + return false; + int initialRegions = state.last; updateState(length); copy = new State(state); + return state.last > initialRegions; } + /** + * Extends this collection of mmapped regions up to the length of the compressed file described by the provided + * metadata. + * + * @return {@code true} if new regions have been created + */ + public boolean extend(CompressionMetadata compressionMetadata) + { + assert !isCopy() : "Copies cannot be extended"; + + if (compressionMetadata.compressedFileLength <= state.length) + return false; + + int initialRegions = state.last; + if (compressionMetadata.compressedFileLength - state.length <= MAX_SEGMENT_SIZE) + updateState(compressionMetadata.compressedFileLength); + else + updateState(compressionMetadata); + + copy = new State(state); + return state.last > initialRegions; + } + + /** + * Updates state by adding the remaining segments. It starts with the current state last segment end position and + * subsequently add new segments until all data up to the provided length are mapped. + */ private void updateState(long length) { state.length = length; @@ -154,8 +195,8 @@ public class MmappedRegions extends SharedCloseableImpl private void updateState(CompressionMetadata metadata) { - long offset = 0; - long lastSegmentOffset = 0; + long lastSegmentOffset = state.getPosition(); + long offset = metadata.getDataOffsetForChunkOffset(lastSegmentOffset); long segmentSize = 0; while (offset < metadata.dataLength) @@ -198,7 +239,7 @@ public class MmappedRegions extends SharedCloseableImpl assert !isCleanedUp() : "Attempted to use closed region"; return state.floor(position); } - + public void closeQuietly() { Throwable err = close(null); @@ -245,19 +286,29 @@ public class MmappedRegions extends SharedCloseableImpl private static final class State { - /** The file channel */ + /** + * The file channel + */ private final ChannelProxy channel; - /** An array of region buffers, synchronized with offsets */ + /** + * An array of region buffers, synchronized with offsets + */ private ByteBuffer[] buffers; - /** An array of region offsets, synchronized with buffers */ + /** + * An array of region offsets, synchronized with buffers + */ private long[] offsets; - /** The maximum file length we have mapped */ + /** + * The maximum file length we have mapped + */ private long length; - /** The index to the last region added */ + /** + * The index to the last region added + */ private int last; private State(ChannelProxy channel) @@ -292,7 +343,7 @@ public class MmappedRegions extends SharedCloseableImpl { assert 0 <= position && position <= length : String.format("%d > %d", position, length); - int idx = Arrays.binarySearch(offsets, 0, last +1, position); + int idx = Arrays.binarySearch(offsets, 0, last + 1, position); assert idx != -1 : String.format("Bad position %d for regions %s, last %d in %s", position, Arrays.toString(offsets), last, channel); if (idx < 0) idx = -(idx + 2); // round down to entry at insertion point @@ -362,5 +413,4 @@ public class MmappedRegions extends SharedCloseableImpl } } } - } diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java new file mode 100644 index 0000000000..9e687ac4a5 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/MmappedRegionsCache.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.io.compress.CompressionMetadata; + +/** + * It is a utility class for caching {@link MmappedRegions} primarily used by a {@link FileHandle.Builder} when a handle + * to the same file is created multiple times (as when an sstable is opened early). + */ +@NotThreadSafe +public class MmappedRegionsCache implements AutoCloseable +{ + private final Map cache = new HashMap<>(); + private boolean closed = false; + + /** + * Looks for mmapped regions in cache. If found, a shared copy is created and extended to the provided length. + * If mmapped regions do not exist yet for the provided key, they are created and a shared copy is returned. + * + * @param channel channel for which the mmapped regions are requested + * @param length length of the file + * @return a shared copy of the cached mmapped regions + */ + @SuppressWarnings("resource") + public MmappedRegions getOrCreate(ChannelProxy channel, long length) + { + Preconditions.checkState(!closed); + MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, length)); + Preconditions.checkArgument(regions.isValid(channel)); + regions.extend(length); + return regions.sharedCopy(); + } + + /** + * Looks for mmapped regions in cache. If found, a shared copy is created and extended according to the provided metadata. + * If mmapped regions do not exist yet for the provided key, they are created and a shared copy is returned. + * + * @param channel channel for which the mmapped regions are requested + * @param metadata compression metadata of the file + * @return a shared copy of the cached mmapped regions + */ + @SuppressWarnings("resource") + public MmappedRegions getOrCreate(ChannelProxy channel, CompressionMetadata metadata) + { + Preconditions.checkState(!closed); + MmappedRegions regions = cache.computeIfAbsent(channel.file(), ignored -> MmappedRegions.map(channel, metadata)); + Preconditions.checkArgument(regions.isValid(channel)); + regions.extend(metadata); + return regions.sharedCopy(); + } + + @Override + @SuppressWarnings("resource") + public void close() + { + closed = true; + Iterator it = cache.values().iterator(); + while (it.hasNext()) + { + MmappedRegions region = it.next(); + region.closeQuietly(); + it.remove(); + } + } +} diff --git a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java index 9ca020871b..b7ae205e96 100644 --- a/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java +++ b/src/java/org/apache/cassandra/io/util/RebufferingInputStream.java @@ -17,13 +17,17 @@ */ package org.apache.cassandra.io.util; -import java.io.*; +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import com.google.common.base.Preconditions; import net.nicoulaj.compilecommand.annotations.DontInline; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import org.apache.cassandra.utils.FastByteOperations; import org.apache.cassandra.utils.vint.VIntCoding; @@ -32,10 +36,10 @@ import static java.lang.Math.min; /** * Rough equivalent of BufferedInputStream and DataInputStream wrapping a ByteBuffer that can be refilled * via rebuffer. Implementations provide this buffer from various channels (socket, file, memory, etc). - * + *

* RebufferingInputStream is not thread safe. */ -public abstract class RebufferingInputStream extends InputStream implements DataInputPlus, Closeable +public abstract class RebufferingInputStream extends DataInputStreamPlus implements DataInputPlus, Closeable { protected ByteBuffer buffer; diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java index 431ece3976..ddd0c83a11 100644 --- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java +++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java @@ -37,6 +37,7 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr { // absolute path to the given file private final String filePath; + private final File file; // Offset for start of buffer relative to underlying file protected long bufferOffset; @@ -162,6 +163,7 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr this.strictFlushing = strictFlushing; this.fchannel = (FileChannel)channel; + this.file = file; this.filePath = file.absolutePath(); this.option = option; @@ -296,6 +298,11 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr return filePath; } + public File getFile() + { + return file; + } + protected void resetBuffer() { bufferOffset = current(); diff --git a/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java new file mode 100644 index 0000000000..5fbe5eaa00 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/WrappingRebufferer.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.util; + +import java.nio.ByteBuffer; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * Instantiated once per RandomAccessReader, thread-unsafe. + * The instances reuse themselves as the BufferHolder to avoid having to return a new object for each rebuffer call. + * Only one buffer holder can be active at a time. Calling {@link #rebuffer(long)} before the previously obtained + * buffer holder is released will throw {@link AssertionError}. We will get that exception also in case we try to close + * the rebufferer without closing the recently obtained buffer holder. + *

+ * Calling methods of {@link BufferHolder} will also produce {@link AssertionError} if buffer holder is not acquired. + *

+ * The overriding classes must conform to the aforementioned rules. + */ +@NotThreadSafe +public abstract class WrappingRebufferer implements Rebufferer, Rebufferer.BufferHolder +{ + protected final Rebufferer wrapped; + + protected BufferHolder bufferHolder; + protected ByteBuffer buffer; + protected long offset; + + public WrappingRebufferer(Rebufferer wrapped) + { + this.wrapped = wrapped; + } + + @Override + public BufferHolder rebuffer(long position) + { + assert buffer == null; + bufferHolder = wrapped.rebuffer(position); + buffer = bufferHolder.buffer(); + offset = bufferHolder.offset(); + + return this; + } + + @Override + public ChannelProxy channel() + { + return wrapped.channel(); + } + + @Override + public long fileLength() + { + return wrapped.fileLength(); + } + + @Override + public double getCrcCheckChance() + { + return wrapped.getCrcCheckChance(); + } + + @Override + public void close() + { + assert buffer == null : "Rebufferer is attempted to be closed but the buffer holder has not been released"; + wrapped.close(); + } + + @Override + public void closeReader() + { + wrapped.closeReader(); + } + + @Override + public String toString() + { + return String.format("%s[]:%s", getClass().getSimpleName(), wrapped.toString()); + } + + @Override + public ByteBuffer buffer() + { + assert buffer != null : "Buffer holder has not been acquired"; + return buffer; + } + + @Override + public long offset() + { + assert buffer != null : "Buffer holder has not been acquired"; + return offset; + } + + @Override + public void release() + { + assert buffer != null; + if (bufferHolder != null) + { + bufferHolder.release(); + bufferHolder = null; + } + buffer = null; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 8b077e745a..63481fcd7f 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -20,6 +20,9 @@ package org.apache.cassandra.metrics; import java.util.Set; import java.util.function.ToLongFunction; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; + import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; @@ -27,11 +30,11 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.metrics.CassandraMetricsRegistry.MetricName; import org.apache.cassandra.metrics.TableMetrics.ReleasableMetric; -import com.google.common.collect.Sets; - import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; /** @@ -68,12 +71,6 @@ public class KeyspaceMetrics /** Uncompressed/logical size of SSTables belonging to tables in this keyspace, scaled down by replication factor */ public final Gauge unreplicatedUncompressedLiveDiskSpaceUsed; public final Gauge totalDiskSpaceUsed; - /** Disk space used by bloom filter */ - public final Gauge bloomFilterDiskSpaceUsed; - /** Off heap memory used by bloom filter */ - public final Gauge bloomFilterOffHeapMemoryUsed; - /** Off heap memory used by index summary */ - public final Gauge indexSummaryOffHeapMemoryUsed; /** Off heap memory used by compression meta data*/ public final Gauge compressionMetadataOffHeapMemoryUsed; /** (Local) read metrics */ @@ -175,6 +172,8 @@ public class KeyspaceMetrics public final Meter rowIndexSizeAborts; public final Histogram rowIndexSize; + public final ImmutableMap>> formatSpecificGauges; + public final MetricNameFactory factory; private final Keyspace keyspace; @@ -217,12 +216,6 @@ public class KeyspaceMetrics metric -> metric.uncompressedLiveDiskSpaceUsed.getCount() / keyspace.getReplicationStrategy().getReplicationFactor().fullReplicas); totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", metric -> metric.totalDiskSpaceUsed.getCount()); - bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed", - metric -> metric.bloomFilterDiskSpaceUsed.getValue()); - bloomFilterOffHeapMemoryUsed = createKeyspaceGauge("BloomFilterOffHeapMemoryUsed", - metric -> metric.bloomFilterOffHeapMemoryUsed.getValue()); - indexSummaryOffHeapMemoryUsed = createKeyspaceGauge("IndexSummaryOffHeapMemoryUsed", - metric -> metric.indexSummaryOffHeapMemoryUsed.getValue()); compressionMetadataOffHeapMemoryUsed = createKeyspaceGauge("CompressionMetadataOffHeapMemoryUsed", metric -> metric.compressionMetadataOffHeapMemoryUsed.getValue()); @@ -280,6 +273,8 @@ public class KeyspaceMetrics rowIndexSizeWarnings = createKeyspaceMeter("RowIndexSizeWarnings"); rowIndexSizeAborts = createKeyspaceMeter("RowIndexSizeAborts"); rowIndexSize = createKeyspaceHistogram("RowIndexSize", false); + + formatSpecificGauges = createFormatSpecificGauges(keyspace); } /** @@ -293,6 +288,24 @@ public class KeyspaceMetrics } } + private ImmutableMap>> createFormatSpecificGauges(Keyspace keyspace) + { + ImmutableMap.Builder>> builder = ImmutableMap.builder(); + for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) + { + ImmutableMap.Builder> gauges = ImmutableMap.builder(); + for (GaugeProvider gaugeProvider : formatType.info.getFormatSpecificMetricsProviders().getGaugeProviders()) + { + String finalName = gaugeProvider.name; + allMetrics.add(() -> releaseMetric(finalName)); + Gauge gauge = Metrics.register(factory.createMetricName(finalName), gaugeProvider.getKeyspaceGauge(keyspace)); + gauges.put(gaugeProvider.name, gauge); + } + builder.put(formatType, gauges.build()); + } + return builder.build(); + } + /** * Creates a gauge that will sum the current value of a metric for all column families in this keyspace * @@ -401,4 +414,4 @@ public class KeyspaceMetrics return new MetricName(groupName, "keyspace", metricName, keyspaceName, mbeanName.toString()); } } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 5dd8e13702..688b75b160 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -17,33 +17,40 @@ */ package org.apache.cassandra.metrics; -import static java.util.concurrent.TimeUnit.MICROSECONDS; -import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.codahale.metrics.Timer; - -import com.google.common.annotations.VisibleForTesting; - import org.apache.commons.lang3.ArrayUtils; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Metric; +import com.codahale.metrics.Timer; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.GaugeProvider; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.metrics.Sampler.SamplerType; @@ -52,12 +59,9 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.Pair; -import com.codahale.metrics.Counter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Meter; -import com.codahale.metrics.Metric; -import com.codahale.metrics.RatioGauge; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; /** * Metrics for {@link ColumnFamilyStore}. @@ -134,24 +138,8 @@ public class TableMetrics public final Gauge maxPartitionSize; /** Size of the smallest compacted partition */ public final Gauge meanPartitionSize; - /** Number of false positives in bloom filter */ - public final Gauge bloomFilterFalsePositives; - /** Number of false positives in bloom filter from last read */ - public final Gauge recentBloomFilterFalsePositives; - /** False positive ratio of bloom filter */ - public final Gauge bloomFilterFalseRatio; - /** False positive ratio of bloom filter from last read */ - public final Gauge recentBloomFilterFalseRatio; - /** Disk space used by bloom filter */ - public final Gauge bloomFilterDiskSpaceUsed; - /** Off heap memory used by bloom filter */ - public final Gauge bloomFilterOffHeapMemoryUsed; - /** Off heap memory used by index summary */ - public final Gauge indexSummaryOffHeapMemoryUsed; /** Off heap memory used by compression meta data*/ public final Gauge compressionMetadataOffHeapMemoryUsed; - /** Key cache hit rate for this CF */ - public final Gauge keyCacheHitRate; /** Tombstones scanned in queries on this CF */ public final TableHistogram tombstoneScannedHistogram; /** Live rows scanned in queries on this CF */ @@ -286,6 +274,8 @@ public class TableMetrics public final TableMeter rowIndexSizeAborts; public final TableHistogram rowIndexSize; + public final ImmutableMap>> formatSpecificGauges; + private static Pair totalNonSystemTablesSize(Predicate predicate) { long total = 0; @@ -726,132 +716,6 @@ public class TableMetrics return count > 0 ? sum / count : 0; } }); - bloomFilterFalsePositives = createTableGauge("BloomFilterFalsePositives", new Gauge() - { - public Long getValue() - { - long count = 0L; - for (SSTableReader sstable: cfs.getSSTables(SSTableSet.LIVE)) - count += sstable.getBloomFilterFalsePositiveCount(); - return count; - } - }); - recentBloomFilterFalsePositives = createTableGauge("RecentBloomFilterFalsePositives", new Gauge() - { - public Long getValue() - { - long count = 0L; - for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE)) - count += sstable.getRecentBloomFilterFalsePositiveCount(); - return count; - } - }); - bloomFilterFalseRatio = createTableGauge("BloomFilterFalseRatio", new Gauge() - { - public Double getValue() - { - long falsePositiveCount = 0L; - long truePositiveCount = 0L; - long trueNegativeCount = 0L; - for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE)) - { - falsePositiveCount += sstable.getBloomFilterFalsePositiveCount(); - truePositiveCount += sstable.getBloomFilterTruePositiveCount(); - trueNegativeCount += sstable.getBloomFilterTrueNegativeCount(); - } - if (falsePositiveCount == 0L && truePositiveCount == 0L) - return 0d; - return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); - } - }, new Gauge() // global gauge - { - public Double getValue() - { - long falsePositiveCount = 0L; - long truePositiveCount = 0L; - long trueNegativeCount = 0L; - for (Keyspace keyspace : Keyspace.all()) - { - for (SSTableReader sstable : keyspace.getAllSSTables(SSTableSet.LIVE)) - { - falsePositiveCount += sstable.getBloomFilterFalsePositiveCount(); - truePositiveCount += sstable.getBloomFilterTruePositiveCount(); - trueNegativeCount += sstable.getBloomFilterTrueNegativeCount(); - } - } - if (falsePositiveCount == 0L && truePositiveCount == 0L) - return 0d; - return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); - } - }); - recentBloomFilterFalseRatio = createTableGauge("RecentBloomFilterFalseRatio", new Gauge() - { - public Double getValue() - { - long falsePositiveCount = 0L; - long truePositiveCount = 0L; - long trueNegativeCount = 0L; - for (SSTableReader sstable: cfs.getSSTables(SSTableSet.LIVE)) - { - falsePositiveCount += sstable.getRecentBloomFilterFalsePositiveCount(); - truePositiveCount += sstable.getRecentBloomFilterTruePositiveCount(); - trueNegativeCount += sstable.getRecentBloomFilterTrueNegativeCount(); - } - if (falsePositiveCount == 0L && truePositiveCount == 0L) - return 0d; - return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); - } - }, new Gauge() // global gauge - { - public Double getValue() - { - long falsePositiveCount = 0L; - long truePositiveCount = 0L; - long trueNegativeCount = 0L; - for (Keyspace keyspace : Keyspace.all()) - { - for (SSTableReader sstable : keyspace.getAllSSTables(SSTableSet.LIVE)) - { - falsePositiveCount += sstable.getRecentBloomFilterFalsePositiveCount(); - truePositiveCount += sstable.getRecentBloomFilterTruePositiveCount(); - trueNegativeCount += sstable.getRecentBloomFilterTrueNegativeCount(); - } - } - if (falsePositiveCount == 0L && truePositiveCount == 0L) - return 0d; - return (double) falsePositiveCount / (truePositiveCount + falsePositiveCount + trueNegativeCount); - } - }); - bloomFilterDiskSpaceUsed = createTableGauge("BloomFilterDiskSpaceUsed", new Gauge() - { - public Long getValue() - { - long total = 0; - for (SSTableReader sst : cfs.getSSTables(SSTableSet.CANONICAL)) - total += sst.getBloomFilterSerializedSize(); - return total; - } - }); - bloomFilterOffHeapMemoryUsed = createTableGauge("BloomFilterOffHeapMemoryUsed", new Gauge() - { - public Long getValue() - { - long total = 0; - for (SSTableReader sst : cfs.getSSTables(SSTableSet.LIVE)) - total += sst.getBloomFilterOffHeapSize(); - return total; - } - }); - indexSummaryOffHeapMemoryUsed = createTableGauge("IndexSummaryOffHeapMemoryUsed", new Gauge() - { - public Long getValue() - { - long total = 0; - for (SSTableReader sst : cfs.getSSTables(SSTableSet.LIVE)) - total += sst.getIndexSummaryOffHeapSize(); - return total; - } - }); compressionMetadataOffHeapMemoryUsed = createTableGauge("CompressionMetadataOffHeapMemoryUsed", new Gauge() { public Long getValue() @@ -870,30 +734,6 @@ public class TableMetrics additionalWrites = createTableCounter("AdditionalWrites"); additionalWriteLatencyNanos = createTableGauge("AdditionalWriteLatencyNanos", () -> MICROSECONDS.toNanos(cfs.additionalWriteLatencyMicros)); - keyCacheHitRate = createTableGauge("KeyCacheHitRate", "KeyCacheHitRate", new RatioGauge() - { - @Override - public Ratio getRatio() - { - return Ratio.of(getNumerator(), getDenominator()); - } - - protected double getNumerator() - { - long hits = 0L; - for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE)) - hits += sstable.getKeyCacheHit(); - return hits; - } - - protected double getDenominator() - { - long requests = 0L; - for (SSTableReader sstable : cfs.getSSTables(SSTableSet.LIVE)) - requests += sstable.getKeyCacheRequest(); - return Math.max(requests, 1); // to avoid NaN. - } - }, null); tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram, false); liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram, false); colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram", cfs.keyspace.metric.colUpdateTimeDeltaHistogram, false); @@ -984,6 +824,8 @@ public class TableMetrics rowIndexSizeWarnings = createTableMeter("RowIndexSizeWarnings", cfs.keyspace.metric.rowIndexSizeWarnings); rowIndexSizeAborts = createTableMeter("RowIndexSizeAborts", cfs.keyspace.metric.rowIndexSizeAborts); rowIndexSize = createTableHistogram("RowIndexSize", cfs.keyspace.metric.rowIndexSize, false); + + formatSpecificGauges = createFormatSpecificGauges(cfs); } private Memtable.MemoryUsage getMemoryUsageWithIndexes(ColumnFamilyStore cfs) @@ -1016,6 +858,22 @@ public class TableMetrics } } + private ImmutableMap>> createFormatSpecificGauges(ColumnFamilyStore cfs) + { + ImmutableMap.Builder>> builder = ImmutableMap.builder(); + for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) + { + ImmutableMap.Builder> gauges = ImmutableMap.builder(); + for (GaugeProvider gaugeProvider : formatType.info.getFormatSpecificMetricsProviders().getGaugeProviders()) + { + Gauge gauge = createTableGauge(gaugeProvider.name, gaugeProvider.getTableGauge(cfs), gaugeProvider.getGlobalGauge()); + gauges.put(gaugeProvider.name, gauge); + } + builder.put(formatType, gauges.build()); + } + return builder.build(); + } + /** * Create a gauge that will be part of a merged version of all column families. The global gauge * will merge each CF gauge by adding their values @@ -1142,6 +1000,7 @@ public class TableMetrics // using SSTableSet.CANONICAL. assert sstable.openReason != SSTableReader.OpenReason.EARLY; + @SuppressWarnings("resource") CompressionMetadata compressionMetadata = sstable.getCompressionMetadata(); compressedLengthSum += compressionMetadata.compressedFileLength; dataLengthSum += compressionMetadata.dataLength; @@ -1433,4 +1292,4 @@ public class TableMetrics return total; } } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java b/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java index 56a860aa84..6ec0c4189b 100644 --- a/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java +++ b/src/java/org/apache/cassandra/metrics/TopPartitionTracker.java @@ -45,7 +45,6 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; @@ -238,7 +237,7 @@ public class TopPartitionTracker implements Closeable return; if (top.size() < maxTopPartitionCount || value > currentMinValue) - track(new TopPartition(SSTable.getMinimalKey(key), value)); + track(new TopPartition(key.retainable(), value)); } private void track(TopPartition tp) @@ -404,4 +403,4 @@ public class TopPartitionTracker implements Closeable this.lastUpdated = lastUpdated; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java index 28a6accff7..5df05c79c2 100644 --- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java +++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java @@ -149,7 +149,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler state.trace("{}/{} ({}%) {} idx:{}{}", new Object[] { FBUtilities.prettyPrintMemory(pi.currentBytes), FBUtilities.prettyPrintMemory(pi.totalBytes), - pi.currentBytes * 100 / pi.totalBytes, + pi.progressPercentage(), pi.direction == ProgressInfo.Direction.OUT ? "sent to" : "received from", pi.sessionIndex, pi.peer }); @@ -201,4 +201,4 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler plan.getCoordinator().getAllStreamSessions().forEach(StreamSession::abort); }); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/schema/Schema.java b/src/java/org/apache/cassandra/schema/Schema.java index 52c18f496d..5cbac5410b 100644 --- a/src/java/org/apache/cassandra/schema/Schema.java +++ b/src/java/org/apache/cassandra/schema/Schema.java @@ -18,7 +18,14 @@ package org.apache.cassandra.schema; import java.time.Duration; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Supplier; @@ -28,10 +35,17 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.MapDifference; import org.apache.commons.lang3.ObjectUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.cql3.functions.*; -import org.apache.cassandra.db.*; +import org.apache.cassandra.cql3.functions.Function; +import org.apache.cassandra.cql3.functions.FunctionName; +import org.apache.cassandra.cql3.functions.UserFunction; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.KeyspaceNotDefinedException; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.exceptions.ConfigurationException; @@ -49,9 +63,6 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Awaitable; import org.apache.cassandra.utils.concurrent.LoadingMap; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import static com.google.common.collect.Iterables.size; import static java.lang.String.format; import static org.apache.cassandra.config.DatabaseDescriptor.isDaemonInitialized; @@ -209,6 +220,12 @@ public class Schema implements SchemaProvider return keyspaceInstances.getIfReady(keyspaceName); } + /** + * Returns {@link ColumnFamilyStore} by the table identifier. Note that though, if called for {@link TableMetadata#id}, + * when metadata points to a secondary index table, the {@link TableMetadata#id} denotes the identifier of the main + * table, not the index table. Thus, this method will return CFS of the main table rather than, probably expected, + * CFS for the index backing table. + */ public ColumnFamilyStore getColumnFamilyStoreInstance(TableId id) { TableMetadata metadata = getTableMetadata(id); @@ -779,4 +796,4 @@ public class Schema implements SchemaProvider : Collections.emptyMap(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 5e51876670..fb44034b4f 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -18,14 +18,27 @@ package org.apache.cassandra.schema; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; - +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import javax.annotation.Nullable; import com.google.common.base.MoreObjects; -import com.google.common.collect.*; - +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +47,18 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.CqlBuilder; import org.apache.cassandra.cql3.SchemaElement; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.CompositeType; +import org.apache.cassandra.db.marshal.EmptyType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UserType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -474,7 +497,7 @@ public class TableMetadata implements SchemaElement return !columnName.bytes.hasRemaining(); } - void validateCompatibility(TableMetadata previous) + public void validateCompatibility(TableMetadata previous) { if (isIndex()) return; @@ -1607,4 +1630,4 @@ public class TableMetadata implements SchemaElement } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java index 9c23e716f7..70ce53db54 100644 --- a/src/java/org/apache/cassandra/service/CacheService.java +++ b/src/java/org/apache/cassandra/service/CacheService.java @@ -19,7 +19,11 @@ package org.apache.cassandra.service; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -27,21 +31,35 @@ import java.util.concurrent.ExecutionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.cache.*; +import org.apache.cassandra.cache.AutoSavingCache; import org.apache.cassandra.cache.AutoSavingCache.CacheSerializer; +import org.apache.cassandra.cache.CacheProvider; +import org.apache.cassandra.cache.CaffeineCache; +import org.apache.cassandra.cache.CounterCacheKey; +import org.apache.cassandra.cache.ICache; +import org.apache.cassandra.cache.IRowCacheEntry; +import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.cache.RowCacheKey; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ClockAndCount; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.context.CounterContext; -import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.partitions.CachedBTreePartition; import org.apache.cassandra.db.partitions.CachedPartition; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableIdFactory; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableMetadata; @@ -80,7 +98,7 @@ public class CacheService implements CacheServiceMBean public final static CacheService instance = new CacheService(); - public final AutoSavingCache keyCache; + public final AutoSavingCache keyCache; public final AutoSavingCache rowCache; public final AutoSavingCache counterCache; @@ -96,7 +114,7 @@ public class CacheService implements CacheServiceMBean /** * @return auto saving cache object */ - private AutoSavingCache initKeyCache() + private AutoSavingCache initKeyCache() { logger.info("Initializing key cache with capacity of {} MiBs.", DatabaseDescriptor.getKeyCacheSizeInMiB()); @@ -104,9 +122,9 @@ public class CacheService implements CacheServiceMBean // as values are constant size we can use singleton weigher // where 48 = 40 bytes (average size of the key) + 8 bytes (size of value) - ICache kc; + ICache kc; kc = CaffeineCache.create(keyCacheInMemoryCapacity); - AutoSavingCache keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer()); + AutoSavingCache keyCache = new AutoSavingCache<>(kc, CacheType.KEY_CACHE, new KeyCacheSerializer()); int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave(); @@ -412,7 +430,7 @@ public class CacheService implements CacheServiceMBean } } - public static class KeyCacheSerializer implements CacheSerializer + public static class KeyCacheSerializer implements CacheSerializer { // For column families with many SSTables the linear nature of getSSTables slowed down KeyCache loading // by orders of magnitude. So we cache the sstables once and rely on cleanupAfterDeserialize to cleanup any @@ -421,7 +439,7 @@ public class CacheService implements CacheServiceMBean public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException { - RowIndexEntry entry = CacheService.instance.keyCache.getInternal(key); + AbstractRowIndexEntry entry = CacheService.instance.keyCache.getInternal(key); if (entry == null) return; @@ -438,13 +456,15 @@ public class CacheService implements CacheServiceMBean out.writeInt(Integer.MIN_VALUE); // backwards compatibility for "int based generation only" ByteBufferUtil.writeWithShortLength(key.desc.id.asBytes(), out); } - out.writeBoolean(true); + // Format type id is stored so that in case there is no sstable for the key we can still figure out which + // serializer (of which sstable format) was used and thus, we can use the right implemnentation to skip + // the unmatched entry + out.writeByte(key.desc.formatType.ordinal); + entry.serializeForCache(out); - SerializationHeader header = new SerializationHeader(false, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS); - key.desc.getFormat().getIndexSerializer(cfs.metadata(), key.desc.version, header).serializeForCache(entry, out); } - public Future> deserialize(DataInputPlus input, ColumnFamilyStore cfs) throws IOException + public Future> deserialize(DataInputPlus input, ColumnFamilyStore cfs) throws IOException { boolean skipEntry = cfs == null || !cfs.isKeyCacheEnabled(); @@ -461,7 +481,18 @@ public class CacheService implements CacheServiceMBean SSTableId generationId = generation == Integer.MIN_VALUE ? SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(input)) : new SequenceBasedSSTableId(generation); // Backwards compatibility for "int based generation sstables" - input.readBoolean(); // backwards compatibility for "promoted indexes" boolean + int typeOrdinal = input.readByte(); + SSTableFormat.Type type; + try + { + type = SSTableFormat.Type.getByOrdinal(typeOrdinal); + } + catch (RuntimeException ex) + { + throw new IOException("Failed to deserialize key of key cache - invalid type ordinal " + typeOrdinal, ex); + } + SSTableFormat.KeyCacheValueSerializer serializer = type.info.getKeyCacheValueSerializer(); + SSTableReader reader = null; if (!skipEntry) { @@ -486,15 +517,14 @@ public class CacheService implements CacheServiceMBean // wrong is during upgrade, in which case we fail at deserialization. This is not a huge deal however since 1) this is unlikely enough that // this won't affect many users (if any) and only once, 2) this doesn't prevent the node from starting and 3) CASSANDRA-10219 shows that this // part of the code has been broken for a while without anyone noticing (it is, btw, still broken until CASSANDRA-10219 is fixed). - RowIndexEntry.Serializer.skipForCache(input); + + serializer.skip(input); return null; } - - RowIndexEntry.IndexSerializer indexSerializer = reader.descriptor.getFormat().getIndexSerializer(reader.metadata(), - reader.descriptor.version, - reader.header); - RowIndexEntry entry = indexSerializer.deserializeForCache(input); - return ImmediateFuture.success(Pair.create(new KeyCacheKey(cfs.metadata(), reader.descriptor, key), entry)); + KeyCacheSupport keyCacheSupportingReader = (KeyCacheSupport) reader; + AbstractRowIndexEntry cacheValue = keyCacheSupportingReader.deserializeKeyCacheValue(input); + KeyCacheKey cacheKey = keyCacheSupportingReader.getCacheKey(key); + return ImmediateFuture.success(Pair.create(cacheKey, cacheValue)); } public void cleanupAfterDeserialize() diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java index c099045537..5b93055a2e 100644 --- a/src/java/org/apache/cassandra/service/StartupChecks.java +++ b/src/java/org/apache/cassandra/service/StartupChecks.java @@ -555,7 +555,7 @@ public class StartupChecks try { - Descriptor desc = Descriptor.fromFilename(file); + Descriptor desc = Descriptor.fromFileWithComponent(file, false).left; if (!desc.isCompatible()) invalid.add(file.toString()); diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 74a15da07b..e6a05e04cc 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -28,8 +28,26 @@ import java.nio.ByteBuffer; import java.nio.file.Paths; import java.time.Instant; import java.time.format.DateTimeParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; @@ -44,9 +62,11 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; - import javax.annotation.Nullable; -import javax.management.*; +import javax.management.ListenerNotFoundException; +import javax.management.NotificationBroadcasterSupport; +import javax.management.NotificationFilter; +import javax.management.NotificationListener; import javax.management.openmbean.CompositeData; import javax.management.openmbean.OpenDataException; import javax.management.openmbean.TabularData; @@ -57,27 +77,18 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.collect.*; -import com.google.common.util.concurrent.*; - -import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.concurrent.*; -import org.apache.cassandra.config.DataStorageSpec; -import org.apache.cassandra.cql3.QueryHandler; -import org.apache.cassandra.dht.RangeStreamer.FetchReplica; -import org.apache.cassandra.fql.FullQueryLogger; -import org.apache.cassandra.fql.FullQueryLoggerOptions; -import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; -import org.apache.cassandra.metrics.Sampler; -import org.apache.cassandra.metrics.SamplingManager; -import org.apache.cassandra.schema.Keyspaces; -import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster; -import org.apache.cassandra.utils.concurrent.Future; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.utils.concurrent.FutureCombiner; -import org.apache.cassandra.utils.concurrent.ImmediateFuture; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Ordering; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.RateLimiter; +import com.google.common.util.concurrent.Uninterruptibles; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,62 +96,136 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.auth.AuthCacheService; -import org.apache.cassandra.config.Config.PaxosStatePurging; -import org.apache.cassandra.service.paxos.*; -import org.apache.cassandra.service.paxos.cleanup.*; -import org.apache.cassandra.utils.progress.ProgressListener; import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.auth.AuthSchemaChangeListener; import org.apache.cassandra.batchlog.BatchlogManager; +import org.apache.cassandra.concurrent.ExecutorLocals; +import org.apache.cassandra.concurrent.FutureTask; +import org.apache.cassandra.concurrent.FutureTaskWithResources; +import org.apache.cassandra.concurrent.NamedThreadFactory; +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.Config.PaxosStatePurging; +import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.cql3.QueryHandler; import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SizeEstimatesRecorder; +import org.apache.cassandra.db.SnapshotDetailsTabularData; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; -import org.apache.cassandra.dht.*; +import org.apache.cassandra.dht.BootStrapper; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.RangeStreamer; +import org.apache.cassandra.dht.RangeStreamer.FetchReplica; +import org.apache.cassandra.dht.StreamStateStore; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.dht.Token.TokenFactory; -import org.apache.cassandra.exceptions.*; -import org.apache.cassandra.gms.*; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.fql.FullQueryLogger; +import org.apache.cassandra.fql.FullQueryLoggerOptions; +import org.apache.cassandra.fql.FullQueryLoggerOptionsCompositeData; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; +import org.apache.cassandra.gms.IFailureDetector; +import org.apache.cassandra.gms.TokenSerializer; +import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.hints.HintsService; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.SSTableLoader; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.PathUtils; -import org.apache.cassandra.locator.*; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.DynamicEndpointSnitch; +import org.apache.cassandra.locator.EndpointsByRange; +import org.apache.cassandra.locator.EndpointsByReplica; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.RangesByEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; +import org.apache.cassandra.locator.Replicas; +import org.apache.cassandra.locator.SystemReplicas; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.metrics.Sampler; +import org.apache.cassandra.metrics.SamplingManager; import org.apache.cassandra.metrics.StorageMetrics; -import org.apache.cassandra.net.*; -import org.apache.cassandra.repair.*; +import org.apache.cassandra.net.AsyncOneResponse; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.repair.RepairRunnable; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaTransformations; import org.apache.cassandra.schema.SystemDistributedKeyspace; +import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.ViewMetadata; +import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.service.paxos.PaxosCommit; +import org.apache.cassandra.service.paxos.PaxosRepair; +import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupLocalCoordinator; +import org.apache.cassandra.service.paxos.cleanup.PaxosTableRepairs; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.TableSnapshot; -import org.apache.cassandra.net.AsyncOneResponse; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.streaming.*; +import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; +import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.tracing.TraceKeyspace; import org.apache.cassandra.transport.ClientResourceLimits; import org.apache.cassandra.transport.ProtocolVersion; -import org.apache.cassandra.utils.*; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; +import org.apache.cassandra.utils.MBeanWrapper; +import org.apache.cassandra.utils.MD5Digest; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.WrappedRunnable; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.FutureCombiner; +import org.apache.cassandra.utils.concurrent.ImmediateFuture; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import org.apache.cassandra.utils.logging.LoggingSupportFactory; import org.apache.cassandra.utils.progress.ProgressEvent; import org.apache.cassandra.utils.progress.ProgressEventType; +import org.apache.cassandra.utils.progress.ProgressListener; import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor; import org.apache.cassandra.utils.progress.jmx.JMXProgressSupport; @@ -149,8 +234,8 @@ import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Iterables.tryFind; import static java.util.Arrays.asList; import static java.util.Arrays.stream; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; @@ -163,11 +248,12 @@ import static org.apache.cassandra.index.SecondaryIndexManager.getIndexName; import static org.apache.cassandra.index.SecondaryIndexManager.isIndexColumnFamily; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.net.Verb.REPLICATION_DONE_REQ; -import static org.apache.cassandra.service.ActiveRepairService.*; +import static org.apache.cassandra.service.ActiveRepairService.ParentRepairStatus; +import static org.apache.cassandra.service.ActiveRepairService.repairCommandExecutor; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.apache.cassandra.utils.FBUtilities.now; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; +import static org.apache.cassandra.utils.FBUtilities.now; /** * This abstraction contains the token/identifier of this node @@ -805,7 +891,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // Ensure StorageProxy is initialized on start-up; see CASSANDRA-3797. Class.forName("org.apache.cassandra.service.StorageProxy"); // also IndexSummaryManager, which is otherwise unreferenced - Class.forName("org.apache.cassandra.io.sstable.IndexSummaryManager"); + Class.forName("org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager"); } catch (ClassNotFoundException e) { @@ -3882,27 +3968,22 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return status.statusCode; } - public int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException - { - return scrub(disableSnapshot, skipCorrupted, true, 0, keyspaceName, tables); - } - - public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException - { - return scrub(disableSnapshot, skipCorrupted, checkData, 0, keyspaceName, tables); - } - - public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException - { - return scrub(disableSnapshot, skipCorrupted, checkData, false, jobs, keyspaceName, tables); - } - public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException + { + IScrubber.Options options = IScrubber.options() + .skipCorrupted(skipCorrupted) + .checkData(checkData) + .reinsertOverflowedTTLRows(reinsertOverflowedTTL) + .build(); + return scrub(disableSnapshot, options, jobs, keyspaceName, tables); + } + + public int scrub(boolean disableSnapshot, IScrubber.Options options, int jobs, String keyspaceName, String... tables) throws IOException, ExecutionException, InterruptedException { CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL; for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, false, keyspaceName, tables)) { - CompactionManager.AllSSTableOpStatus oneStatus = cfStore.scrub(disableSnapshot, skipCorrupted, reinsertOverflowedTTL, checkData, jobs); + CompactionManager.AllSSTableOpStatus oneStatus = cfStore.scrub(disableSnapshot, options, jobs); if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL) status = oneStatus; } @@ -3918,12 +3999,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public int verify(boolean extendedVerify, boolean checkVersion, boolean diskFailurePolicy, boolean mutateRepairStatus, boolean checkOwnsTokens, boolean quick, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException { CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL; - Verifier.Options options = Verifier.options().invokeDiskFailurePolicy(diskFailurePolicy) - .extendedVerification(extendedVerify) - .checkVersion(checkVersion) - .mutateRepairStatus(mutateRepairStatus) - .checkOwnsTokens(checkOwnsTokens) - .quick(quick).build(); + IVerifier.Options options = IVerifier.options().invokeDiskFailurePolicy(diskFailurePolicy) + .extendedVerification(extendedVerify) + .checkVersion(checkVersion) + .mutateRepairStatus(mutateRepairStatus) + .checkOwnsTokens(checkOwnsTokens) + .quick(quick).build(); logger.info("Verifying {}.{} with options = {}", keyspaceName, Arrays.toString(tableNames), options); for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, tableNames)) { diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 43208b1308..e6aa9cd6a2 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -385,11 +385,22 @@ public interface StorageServiceMBean extends NotificationEmitter * Scrubbed CFs will be snapshotted first, if disableSnapshot is false */ @Deprecated - public int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException; + default int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException + { + return scrub(disableSnapshot, skipCorrupted, true, keyspaceName, tableNames); + } + @Deprecated - public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException; + default int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, String keyspaceName, String... tableNames) throws IOException, ExecutionException, InterruptedException + { + return scrub(disableSnapshot, skipCorrupted, checkData, 0, keyspaceName, tableNames); + } + @Deprecated - public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException; + default int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException + { + return scrub(disableSnapshot, skipCorrupted, checkData, false, jobs, keyspaceName, columnFamilies); + } public int scrub(boolean disableSnapshot, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException; @@ -1174,4 +1185,4 @@ public interface StorageServiceMBean extends NotificationEmitter public void setSkipStreamDiskSpaceCheck(boolean value); public boolean getSkipStreamDiskSpaceCheck(); -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java index 5e3b5404e8..b03701d10b 100644 --- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java @@ -18,7 +18,12 @@ package org.apache.cassandra.service.paxos.uncommitted; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -29,13 +34,24 @@ import com.google.common.util.concurrent.Callables; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RangeTombstone; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.db.WriteContext; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.memtable.Memtable; -import org.apache.cassandra.db.partitions.*; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -43,14 +59,14 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.IndexRegistry; import org.apache.cassandra.index.transactions.IndexTransaction; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Indexes; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.CloseableIterator; -import static java.util.Collections.*; +import static java.util.Collections.singletonList; import static org.apache.cassandra.schema.SchemaConstants.SYSTEM_KEYSPACE_NAME; import static org.apache.cassandra.service.paxos.PaxosState.ballotTracker; import static org.apache.cassandra.service.paxos.PaxosState.uncommittedTracker; diff --git a/src/java/org/apache/cassandra/streaming/ProgressInfo.java b/src/java/org/apache/cassandra/streaming/ProgressInfo.java index 2ed78aca27..159775c324 100644 --- a/src/java/org/apache/cassandra/streaming/ProgressInfo.java +++ b/src/java/org/apache/cassandra/streaming/ProgressInfo.java @@ -60,7 +60,6 @@ public class ProgressInfo implements Serializable public ProgressInfo(InetAddressAndPort peer, int sessionIndex, String fileName, Direction direction, long currentBytes, long deltaBytes, long totalBytes) { - assert totalBytes > 0; this.peer = peer; this.sessionIndex = sessionIndex; @@ -79,6 +78,11 @@ public class ProgressInfo implements Serializable return currentBytes >= totalBytes; } + public int progressPercentage() + { + return totalBytes == 0 ? 100 : (int) ((100 * currentBytes) / totalBytes); + } + /** * ProgressInfo is considered to be equal only when all attributes except currentBytes are equal. */ @@ -114,10 +118,10 @@ public class ProgressInfo implements Serializable StringBuilder sb = new StringBuilder(fileName); sb.append(" ").append(currentBytes); sb.append("/").append(totalBytes).append(" bytes "); - sb.append("(").append(currentBytes*100/totalBytes).append("%) "); + sb.append("(").append(progressPercentage()).append("%) "); sb.append(direction == Direction.OUT ? "sent to " : "received from "); sb.append("idx:").append(sessionIndex); sb.append(peer.toString(withPorts)); return sb.toString(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tools/GenerateTokens.java b/src/java/org/apache/cassandra/tools/GenerateTokens.java index a6888d76e7..560a53f8d9 100644 --- a/src/java/org/apache/cassandra/tools/GenerateTokens.java +++ b/src/java/org/apache/cassandra/tools/GenerateTokens.java @@ -26,7 +26,6 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; - import org.slf4j.LoggerFactory; import ch.qos.logback.classic.Level; @@ -107,7 +106,7 @@ public class GenerateTokens } catch (Throwable t) { - logger.warn("Error running tool.", t); + logger.warn(t, "Error running tool."); System.exit(1); } } @@ -152,5 +151,4 @@ public class GenerateTokens "Options are:"; new HelpFormatter().printHelp(usage, header, options, ""); } -} - +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java index f5d24ed7ea..04d77a9d4a 100644 --- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java +++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java @@ -26,14 +26,15 @@ import java.util.Set; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; @@ -74,12 +75,12 @@ public class SSTableExpiredBlockers { try { - SSTableReader reader = SSTableReader.open(sstable.getKey()); + SSTableReader reader = SSTableReader.open(cfs, sstable.getKey()); sstables.add(reader); } catch (Throwable t) { - out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA)+" ("+t.getMessage()+")"); + out.println("Couldn't open sstable: " + sstable.getKey().fileFor(Components.DATA) + " (" + t.getMessage() + ")"); } } } diff --git a/src/java/org/apache/cassandra/tools/SSTableExport.java b/src/java/org/apache/cassandra/tools/SSTableExport.java index 771380b6a1..43fda169ef 100644 --- a/src/java/org/apache/cassandra/tools/SSTableExport.java +++ b/src/java/org/apache/cassandra/tools/SSTableExport.java @@ -25,8 +25,14 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.TableMetadata; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; @@ -39,13 +45,8 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.KeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.FBUtilities; @@ -135,20 +136,21 @@ public class SSTableExport printUsage(); System.exit(1); } - String ssTableFileName = new File(cmd.getArgs()[0]).absolutePath(); + File ssTableFile = new File(cmd.getArgs()[0]); - if (!new File(ssTableFileName).exists()) + if (!ssTableFile.exists()) { - System.err.println("Cannot find file " + ssTableFileName); + System.err.println("Cannot find file " + ssTableFile.absolutePath()); System.exit(1); } - Descriptor desc = Descriptor.fromFilename(ssTableFileName); + Descriptor desc = Descriptor.fromFileWithComponent(ssTableFile, false).left; try { TableMetadata metadata = Util.metadataFromSSTable(desc); + SSTableReader sstable = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata)); if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) { - try (KeyIterator iter = new KeyIterator(desc, metadata)) + try (KeyIterator iter = sstable.keyIterator()) { JsonTransformer.keysToJson(null, Util.iterToStream(iter), cmd.hasOption(RAW_TIMESTAMPS), @@ -158,7 +160,6 @@ public class SSTableExport } else { - SSTableReader sstable = SSTableReader.openNoValidation(desc, TableMetadataRef.forOfflineTools(metadata)); IPartitioner partitioner = sstable.getPartitioner(); final ISSTableScanner currentScanner; if ((keys != null) && (keys.length > 0)) diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java index 9618d3621f..0aa9e10814 100644 --- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java +++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java @@ -21,15 +21,16 @@ import java.io.PrintStream; import java.util.Map; import java.util.Set; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.JVMStabilityInspector; /** @@ -90,19 +91,19 @@ public class SSTableLevelResetter boolean foundSSTable = false; for (Map.Entry> sstable : lister.list().entrySet()) { - if (sstable.getValue().contains(Component.STATS)) + if (sstable.getValue().contains(Components.STATS)) { foundSSTable = true; Descriptor descriptor = sstable.getKey(); - StatsMetadata metadata = (StatsMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.STATS); + StatsMetadata metadata = StatsComponent.load(descriptor).statsMetadata(); if (metadata.sstableLevel > 0) { - out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.filenameFor(Component.DATA)); + out.println("Changing level from " + metadata.sstableLevel + " to 0 on " + descriptor.fileFor(Components.DATA)); descriptor.getMetadataSerializer().mutateLevel(descriptor, 0); } else { - out.println("Skipped " + descriptor.filenameFor(Component.DATA) + " since it is already on level 0"); + out.println("Skipped " + descriptor.fileFor(Components.DATA) + " since it is already on level 0"); } } } diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java index 48f63e3897..42def40ddf 100644 --- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java +++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java @@ -17,14 +17,11 @@ */ package org.apache.cassandra.tools; -import java.io.DataInputStream; import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.util.Comparator; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -51,14 +48,13 @@ import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.compress.CompressionMetadata; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.compress.ICompressor; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.io.sstable.IndexSummary; +import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; -import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; import org.apache.cassandra.io.util.File; @@ -175,7 +171,7 @@ public class SSTableMetadataViewer private void printScannedOverview(Descriptor descriptor, StatsMetadata stats) throws IOException { TableMetadata cfm = Util.metadataFromSSTable(descriptor); - SSTableReader reader = SSTableReader.openNoValidation(descriptor, TableMetadataRef.forOfflineTools(cfm)); + SSTableReader reader = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(cfm)); try (ISSTableScanner scanner = reader.getScanner()) { long bytes = scanner.getLengthInBytes(); @@ -316,20 +312,19 @@ public class SSTableMetadataViewer } } - private void printSStableMetadata(String fname, boolean scan) throws IOException + private void printSStableMetadata(File file, boolean scan) throws IOException { - Descriptor descriptor = Descriptor.fromFilename(fname); - Map metadata = descriptor.getMetadataSerializer() - .deserialize(descriptor, EnumSet.allOf(MetadataType.class)); - ValidationMetadata validation = (ValidationMetadata) metadata.get(MetadataType.VALIDATION); - StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS); - CompactionMetadata compaction = (CompactionMetadata) metadata.get(MetadataType.COMPACTION); - CompressionMetadata compression = null; - File compressionFile = new File(descriptor.filenameFor(Component.COMPRESSION_INFO)); - if (compressionFile.exists()) - compression = CompressionMetadata.create(fname); - SerializationHeader.Component header = (SerializationHeader.Component) metadata - .get(MetadataType.HEADER); + Descriptor descriptor = Descriptor.fromFileWithComponent(file, false).left; + StatsComponent statsComponent = StatsComponent.load(descriptor); + ValidationMetadata validation = statsComponent.validationMetadata(); + StatsMetadata stats = statsComponent.statsMetadata(); + CompactionMetadata compaction = statsComponent.compactionMetadata(); + SerializationHeader.Component header = statsComponent.serializationHeader(); + Class compressorClass = null; + try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor)) + { + compressorClass = compression != null ? compression.compressor().getClass() : null; + } field("SSTable", descriptor); if (scan && descriptor.version.getVersion().compareTo("ma") >= 0) @@ -347,8 +342,8 @@ public class SSTableMetadataViewer field("Maximum timestamp", stats.maxTimestamp, toDateString(stats.maxTimestamp, tsUnit)); field("SSTable min local deletion time", stats.minLocalDeletionTime, deletion(stats.minLocalDeletionTime)); field("SSTable max local deletion time", stats.maxLocalDeletionTime, deletion(stats.maxLocalDeletionTime)); - field("Compressor", compression != null ? compression.compressor().getClass().getName() : "-"); - if (compression != null) + field("Compressor", compressorClass != null ? compressorClass.getName() : "-"); + if (compressorClass != null) field("Compression ratio", stats.compressionRatio); field("TTL min", stats.minTTL, toDurationString(stats.minTTL, TimeUnit.SECONDS)); field("TTL max", stats.maxTTL, toDurationString(stats.maxTTL, TimeUnit.SECONDS)); @@ -465,7 +460,7 @@ public class SSTableMetadataViewer } private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType keyType, StatsMetadata statsMetadata) - throws IOException + throws IOException { if (descriptor.version.hasKeyRange()) { @@ -477,17 +472,12 @@ public class SSTableMetadataViewer } else { - File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY)); - if (!summariesFile.exists()) + Pair firstLast = descriptor.getFormat().getReaderFactory().readKeyRange(descriptor, partitioner); + if (firstLast == null) return; - try (DataInputStream iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath()))) - { - Pair firstLast = new IndexSummary.IndexSummarySerializer() - .deserializeFirstLastKey(iStream, partitioner); - field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey())); - field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey())); - } + field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey())); + field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey())); } } @@ -545,7 +535,7 @@ public class SSTableMetadataViewer File sstable = new File(fname); if (sstable.exists()) { - metawriter.printSStableMetadata(sstable.absolutePath(), fullScan); + metawriter.printSStableMetadata(sstable, fullScan); } else { diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java index b6948b4f17..16faa9283b 100644 --- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java +++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java @@ -33,16 +33,17 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.SetMultimap; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.Schema; /** * Create a decent leveling for the given keyspace/column family @@ -116,12 +117,12 @@ public class SSTableOfflineRelevel { try { - SSTableReader reader = SSTableReader.open(sstable.getKey()); + SSTableReader reader = SSTableReader.open(cfs, sstable.getKey()); sstableMultimap.put(reader.descriptor.directory, reader); } catch (Throwable t) { - out.println("Couldn't open sstable: "+sstable.getKey().filenameFor(Component.DATA)); + out.println("Couldn't open sstable: "+sstable.getKey().fileFor(Components.DATA)); Throwables.throwIfUnchecked(t); throw new RuntimeException(t); } diff --git a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java index 62dd76ee56..63f30890a6 100644 --- a/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java +++ b/src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java @@ -27,20 +27,8 @@ import java.nio.file.attribute.FileTime; import java.util.Arrays; import java.util.List; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; - -/** - * Set repairedAt status on a given set of sstables. - * - * If you pass --is-repaired, it will set the repairedAt time to the last modified time. - * - * If you know you ran repair 2 weeks ago, you can do something like - * - * {@code - * sstablerepairset --is-repaired -f <(find /var/lib/cassandra/data/.../ -iname "*Data.db*" -mtime +14) - * } - */ +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.util.File; public class SSTableRepairedAtSetter @@ -82,7 +70,7 @@ public class SSTableRepairedAtSetter for (String fname: fileNames) { - Descriptor descriptor = Descriptor.fromFilename(fname); + Descriptor descriptor = Descriptor.fromFileWithComponent(new File(fname), false).left; if (!descriptor.version.isCompatible()) { System.err.println("SSTable " + fname + " is in a old and unsupported format"); @@ -91,7 +79,7 @@ public class SSTableRepairedAtSetter if (setIsRepaired) { - FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath()); + FileTime f = Files.getLastModifiedTime(descriptor.fileFor(Components.DATA).toPath()); descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null, false); } else diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index d6e99b9c2f..7a458683bf 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -18,7 +18,6 @@ */ package org.apache.cassandra.tools; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -26,7 +25,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.io.util.File; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -43,12 +41,15 @@ import org.apache.cassandra.db.compaction.CompactionStrategyManager; import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.compaction.Scrubber; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IScrubber; import org.apache.cassandra.io.sstable.SSTableHeaderFix; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.tools.BulkLoader.CmdLineOptions; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -121,7 +122,7 @@ public class StandaloneScrubber { Descriptor descriptor = entry.getKey(); Set components = entry.getValue(); - if (!components.contains(Component.DATA)) + if (!components.contains(Components.DATA)) continue; listResult.add(Pair.create(descriptor, components)); @@ -144,7 +145,7 @@ public class StandaloneScrubber headerFixBuilder = headerFixBuilder.dryRun(); for (Pair> p : listResult) - headerFixBuilder.withPath(Paths.get(p.left.filenameFor(Component.DATA))); + headerFixBuilder.withPath(p.left.fileFor(Components.DATA).toPath()); SSTableHeaderFix headerFix = headerFixBuilder.build(); try @@ -198,7 +199,7 @@ public class StandaloneScrubber { Descriptor descriptor = pair.left; Set components = pair.right; - if (!components.contains(Component.DATA)) + if (!components.contains(Components.DATA)) continue; try @@ -222,7 +223,9 @@ public class StandaloneScrubber try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable)) { txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591 - try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, !options.noValidate, options.reinserOverflowedTTL)) + + SSTableFormat format = sstable.descriptor.getFormat(); + try (IScrubber scrubber = format.getScrubber(cfs, txn, handler, options.build())) { scrubber.scrub(); } @@ -276,7 +279,7 @@ public class StandaloneScrubber } } - private static class Options + private static class Options extends IScrubber.Options.Builder { public final String keyspaceName; public final String cfName; @@ -284,9 +287,6 @@ public class StandaloneScrubber public boolean debug; public boolean verbose; public boolean manifestCheckOnly; - public boolean skipCorrupted; - public boolean noValidate; - public boolean reinserOverflowedTTL; public HeaderFixMode headerFixMode = HeaderFixMode.VALIDATE; enum HeaderFixMode @@ -345,9 +345,9 @@ public class StandaloneScrubber opts.debug = cmd.hasOption(DEBUG_OPTION); opts.verbose = cmd.hasOption(VERBOSE_OPTION); opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION); - opts.skipCorrupted = cmd.hasOption(SKIP_CORRUPTED_OPTION); - opts.noValidate = cmd.hasOption(NO_VALIDATE_OPTION); - opts.reinserOverflowedTTL = cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION); + opts.skipCorrupted(cmd.hasOption(SKIP_CORRUPTED_OPTION)); + opts.checkData(!cmd.hasOption(NO_VALIDATE_OPTION)); + opts.reinsertOverflowedTTLRows(cmd.hasOption(REINSERT_OVERFLOWED_TTL_OPTION)); if (cmd.hasOption(HEADERFIX_OPTION)) { try diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java index efadb56e0c..5229c7764a 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java +++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java @@ -18,23 +18,35 @@ */ package org.apache.cassandra.tools; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.commons.cli.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.ParseException; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.SSTableSplitter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.JVMStabilityInspector; import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions; @@ -50,7 +62,7 @@ public class StandaloneSplitter private static final String NO_SNAPSHOT_OPTION = "no-snapshot"; private static final String SIZE_OPTION = "size"; - public static void main(String args[]) + public static void main(String[] args) { Options options = Options.parseArgs(args); if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST)) @@ -74,7 +86,7 @@ public class StandaloneSplitter continue; } - Descriptor desc = SSTable.tryDescriptorFromFilename(file); + Descriptor desc = SSTable.tryDescriptorFromFile(file); if (desc == null) { System.out.println("Skipping non sstable file " + file); continue; @@ -90,21 +102,7 @@ public class StandaloneSplitter else if (!cfName.equals(desc.cfname)) throw new IllegalArgumentException("All sstables must be part of the same table"); - Set components = new HashSet(Arrays.asList(new Component[]{ - Component.DATA, - Component.PRIMARY_INDEX, - Component.FILTER, - Component.COMPRESSION_INFO, - Component.STATS - })); - - Iterator iter = components.iterator(); - while (iter.hasNext()) { - Component component = iter.next(); - if (!(new File(desc.filenameFor(component)).exists())) - iter.remove(); - } - parsedFilenames.put(desc, components); + parsedFilenames.put(desc, desc.getComponents(Collections.emptySet(), desc.getFormat().batchComponents())); } if (ksName == null || cfName == null) @@ -125,8 +123,8 @@ public class StandaloneSplitter { SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs); if (!isSSTableLargerEnough(sstable, options.sizeInMB)) { - System.out.println(String.format("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)", - sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB)); + System.out.printf("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)%n", + sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB); continue; } sstables.add(sstable); @@ -140,7 +138,7 @@ public class StandaloneSplitter catch (Exception e) { JVMStabilityInspector.inspectThrowable(e); - System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage())); + System.err.printf("Error Loading %s: %s%n", fn.getKey(), e.getMessage()); if (options.debug) e.printStackTrace(System.err); } @@ -150,7 +148,7 @@ public class StandaloneSplitter System.exit(0); } if (options.snapshot) - System.out.println(String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName)); + System.out.printf("Pre-split sstables snapshotted into snapshot %s%n", snapshotName); for (SSTableReader sstable : sstables) { @@ -160,7 +158,7 @@ public class StandaloneSplitter } catch (Exception e) { - System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage())); + System.err.printf("Error splitting %s: %s%n", sstable, e.getMessage()); if (options.debug) e.printStackTrace(System.err); @@ -200,7 +198,7 @@ public class StandaloneSplitter this.filenames = filenames; } - public static Options parseArgs(String cmdArgs[]) + public static Options parseArgs(String[] cmdArgs) { CommandLineParser parser = new GnuParser(); CmdLineOptions options = getCmdLineOptions(); @@ -251,19 +249,18 @@ public class StandaloneSplitter options.addOption(null, DEBUG_OPTION, "display stack traces"); options.addOption("h", HELP_OPTION, "display this help message"); options.addOption(null, NO_SNAPSHOT_OPTION, "don't snapshot the sstables before splitting"); - options.addOption("s", SIZE_OPTION, "size", "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ")"); + options.addOption("s", SIZE_OPTION, "size", "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ')'); return options; } public static void printUsage(CmdLineOptions options) { String usage = String.format("%s [options] []*", TOOL_NAME); - StringBuilder header = new StringBuilder(); - header.append("--\n"); - header.append("Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." ); - header.append("\n--\n"); - header.append("Options are:"); - new HelpFormatter().printHelp(usage, header.toString(), options, ""); + String header = "--\n" + + "Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." + + "\n--\n" + + "Options are:"; + new HelpFormatter().printHelp(usage, header, options, ""); } } } diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java index 442a95b0c7..657ec05d65 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java +++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java @@ -17,10 +17,17 @@ */ package org.apache.cassandra.tools; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.commons.cli.*; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.ParseException; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; @@ -30,7 +37,8 @@ import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.Upgrader; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.Schema; @@ -80,7 +88,7 @@ public class StandaloneUpgrader for (Map.Entry> entry : lister.list().entrySet()) { Set components = entry.getValue(); - if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX)) + if (!components.containsAll(entry.getKey().getFormat().primaryComponents())) continue; try diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java index d7554dd5d3..b32d81bb13 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java +++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java @@ -39,16 +39,17 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Throwables; import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions; @@ -78,6 +79,8 @@ public class StandaloneVerifier System.out.println("sstableverify using the following options: " + options); + List sstables = new ArrayList<>(); + int exitCode = 0; try { // load keyspace descriptions. @@ -97,13 +100,11 @@ public class StandaloneVerifier OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug); Directories.SSTableLister lister = cfs.getDirectories().sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true); - List sstables = new ArrayList<>(); - // Verify sstables for (Map.Entry> entry : lister.list().entrySet()) { Set components = entry.getValue(); - if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX)) + if (!components.containsAll(entry.getKey().getFormat().primaryComponents())) continue; try @@ -117,40 +118,51 @@ public class StandaloneVerifier System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage())); if (options.debug) e.printStackTrace(System.err); - System.exit(1); + exitCode = 1; + return; } } - Verifier.Options verifyOptions = Verifier.options().invokeDiskFailurePolicy(false) - .extendedVerification(options.extended) - .checkVersion(options.checkVersion) - .mutateRepairStatus(options.mutateRepairStatus) - .checkOwnsTokens(!options.tokens.isEmpty()) - .tokenLookup(ignore -> options.tokens) - .build(); + IVerifier.Options verifyOptions = IVerifier.options().invokeDiskFailurePolicy(false) + .extendedVerification(options.extended) + .checkVersion(options.checkVersion) + .mutateRepairStatus(options.mutateRepairStatus) + .checkOwnsTokens(!options.tokens.isEmpty()) + .tokenLookup(ignore -> options.tokens) + .build(); handler.output("Running verifier with the following options: " + verifyOptions); for (SSTableReader sstable : sstables) { - try (Verifier verifier = new Verifier(cfs, sstable, handler, true, verifyOptions)) + try (IVerifier verifier = sstable.getVerifier(cfs, handler, true, verifyOptions)) { verifier.verify(); } catch (Exception e) { - handler.warn(String.format("Error verifying %s: %s", sstable, e.getMessage()), e); + handler.warn(e, String.format("Error verifying %s: %s", sstable, e.getMessage())); hasFailed = true; } } CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES); - System.exit( hasFailed ? 1 : 0 ); // We need that to stop non daemonized threads + for (SSTableReader reader : sstables) + Throwables.perform((Throwable) null, () -> reader.selfRef().close()); + + exitCode = hasFailed ? 1 : 0; // We need that to stop non daemonized threads } catch (Exception e) { System.err.println(e.getMessage()); if (options.debug) e.printStackTrace(System.err); - System.exit(1); + exitCode = 1; + } + finally + { + for (SSTableReader reader : sstables) + Throwables.perform((Throwable) null, () -> reader.selfRef().close()); + + System.exit(exitCode); } } diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java index 3757754f35..414d04ad4c 100644 --- a/src/java/org/apache/cassandra/tools/Util.java +++ b/src/java/org/apache/cassandra/tools/Util.java @@ -18,22 +18,22 @@ package org.apache.cassandra.tools; -import static java.lang.String.format; - import java.io.IOException; import java.io.PrintStream; -import java.util.EnumSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Spliterator; import java.util.Spliterators; import java.util.TreeMap; -import java.util.Map.Entry; import java.util.function.LongFunction; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.SerializationHeader; @@ -41,15 +41,14 @@ import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.streamhist.TombstoneHistogram; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; +import static java.lang.String.format; @SuppressWarnings("serial") public final class Util @@ -310,12 +309,11 @@ public final class Util */ public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOException { - if (desc.version.getVersion().compareTo("ma") < 0) - throw new IOException("pre-3.0 SSTable is not supported."); + if (!desc.version.isCompatible()) + throw new IOException("Unsupported SSTable version " + desc.getFormat().getType().name + "/" + desc.version); - EnumSet types = EnumSet.of(MetadataType.STATS, MetadataType.HEADER); - Map sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types); - SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); + StatsComponent statsComponent = StatsComponent.load(desc, MetadataType.STATS, MetadataType.HEADER); + SerializationHeader.Component header = statsComponent.serializationHeader(); IPartitioner partitioner = FBUtilities.newPartitioner(desc); @@ -337,4 +335,4 @@ public final class Util } return builder.build(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/BloomFilter.java b/src/java/org/apache/cassandra/utils/BloomFilter.java index bf48d43418..a95d131a39 100644 --- a/src/java/org/apache/cassandra/utils/BloomFilter.java +++ b/src/java/org/apache/cassandra/utils/BloomFilter.java @@ -17,10 +17,13 @@ */ package org.apache.cassandra.utils; +import java.io.IOException; + import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.FastThreadLocal; import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.concurrent.WrappedSharedCloseable; import org.apache.cassandra.utils.obs.IBitSet; @@ -29,6 +32,7 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter { private final static FastThreadLocal reusableIndexes = new FastThreadLocal() { + @Override protected long[] initialValue() { return new long[21]; @@ -52,9 +56,15 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter this.bitset = copy.bitset; } - public long serializedSize() + public long serializedSize(boolean old) { - return BloomFilterSerializer.serializedSize(this); + return BloomFilterSerializer.forVersion(old).serializedSize(this); + } + + @Override + public void serialize(DataOutputStreamPlus out, boolean old) throws IOException + { + BloomFilterSerializer.forVersion(old).serialize(this, out); } // Murmur is faster than an SHA-based approach and provides as-good collision @@ -101,6 +111,7 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter } } + @Override public void add(FilterKey key) { long[] indexes = indexes(key); @@ -110,6 +121,7 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter } } + @Override public final boolean isPresent(FilterKey key) { long[] indexes = indexes(key); @@ -123,12 +135,14 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter return true; } + @Override public void clear() { bitset.clear(); } - public IFilter sharedCopy() + @Override + public BloomFilter sharedCopy() { return new BloomFilter(this); } @@ -139,11 +153,19 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter return bitset.offHeapSize(); } + @Override + public boolean isInformative() + { + return bitset.offHeapSize() > 0; + } + + @Override public String toString() { return "BloomFilter[hashCount=" + hashCount + ";capacity=" + bitset.capacity() + ']'; } + @Override public void addTo(Ref.IdentityCollection identities) { super.addTo(identities); diff --git a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java index 3df43147ce..e4b34a49d7 100644 --- a/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java +++ b/src/java/org/apache/cassandra/utils/BloomFilterSerializer.java @@ -17,47 +17,65 @@ */ package org.apache.cassandra.utils; -import java.io.DataInput; import java.io.IOException; -import java.io.InputStream; import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.IGenericSerializer; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; +import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.utils.obs.IBitSet; import org.apache.cassandra.utils.obs.OffHeapBitSet; -public final class BloomFilterSerializer +public final class BloomFilterSerializer implements IGenericSerializer { - private BloomFilterSerializer() + public final static BloomFilterSerializer newFormatInstance = new BloomFilterSerializer(false); + public final static BloomFilterSerializer oldFormatInstance = new BloomFilterSerializer(true); + + private final boolean oldFormat; + + private BloomFilterSerializer(boolean oldFormat) { + this.oldFormat = oldFormat; } - public static void serialize(BloomFilter bf, DataOutputPlus out) throws IOException + public static BloomFilterSerializer forVersion(boolean oldSerializationFormat) { + if (oldSerializationFormat) + return oldFormatInstance; + + return newFormatInstance; + } + + @Override + public void serialize(BloomFilter bf, DataOutputStreamPlus out) throws IOException + { + assert !oldFormat : "Filter should not be serialized in old format"; out.writeInt(bf.hashCount); bf.bitset.serialize(out); } - @SuppressWarnings("resource") - public static BloomFilter deserialize(I in, boolean oldBfFormat) throws IOException - { - int hashes = in.readInt(); - IBitSet bs = OffHeapBitSet.deserialize(in, oldBfFormat); - - return new BloomFilter(hashes, bs); - } - /** * Calculates a serialized size of the given Bloom Filter - * @param bf Bloom filter to calculate serialized size - * @see org.apache.cassandra.io.ISerializer#serialize(Object, org.apache.cassandra.io.util.DataOutputPlus) * + * @param bf Bloom filter to calculate serialized size * @return serialized size of the given bloom filter + * @see org.apache.cassandra.io.ISerializer#serialize(Object, org.apache.cassandra.io.util.DataOutputPlus) */ - public static long serializedSize(BloomFilter bf) + @Override + public long serializedSize(BloomFilter bf) { int size = TypeSizes.sizeof(bf.hashCount); // hash count size += bf.bitset.serializedSize(); return size; } + + @Override + @SuppressWarnings("resource") + public BloomFilter deserialize(DataInputStreamPlus in) throws IOException + { + int hashes = in.readInt(); + IBitSet bs = OffHeapBitSet.deserialize(in, oldFormat); + + return new BloomFilter(hashes, bs); + } } diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index c279e8b54c..1eb1de192a 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -27,37 +27,46 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.lang.reflect.Field; import java.math.BigInteger; -import java.net.*; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.URL; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Optional; +import java.util.Properties; +import java.util.TreeSet; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.zip.CRC32; import java.util.zip.Checksum; - import javax.annotation.Nonnull; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; - -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileInputStreamPlus; -import org.apache.cassandra.io.util.FileOutputStreamPlus; -import org.apache.cassandra.utils.concurrent.*; +import com.google.common.collect.ImmutableList; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.cassandra.audit.IAuditLogger; import org.apache.cassandra.auth.AllowAllNetworkAuthorizer; import org.apache.cassandra.auth.IAuthenticator; @@ -66,7 +75,6 @@ import org.apache.cassandra.auth.INetworkAuthorizer; import org.apache.cassandra.auth.IRoleManager; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.LocalPartitioner; @@ -75,14 +83,17 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; -import org.apache.cassandra.io.sstable.metadata.ValidationMetadata; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputBufferFixed; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.security.ISslContextFactory; +import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import org.objectweb.asm.Opcodes; @@ -608,11 +619,8 @@ public class FBUtilities */ public static IPartitioner newPartitioner(Descriptor desc) throws IOException { - EnumSet types = EnumSet.of(MetadataType.VALIDATION, MetadataType.HEADER); - Map sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types); - ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION); - SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER); - return newPartitioner(validationMetadata.partitioner, Optional.of(header.getKeyType())); + StatsComponent statsComponent = StatsComponent.load(desc, MetadataType.VALIDATION, MetadataType.HEADER); + return newPartitioner(statsComponent.validationMetadata().partitioner, Optional.of(statsComponent.serializationHeader().getKeyType())); } public static IPartitioner newPartitioner(String partitionerClassName) throws ConfigurationException @@ -1161,4 +1169,16 @@ public class FBUtilities } return sb.toString(); } -} + + @SafeVarargs + public static ImmutableList immutableListWithFilteredNulls(T... values) + { + ImmutableList.Builder builder = ImmutableList.builderWithExpectedSize(values.length); + for (int i = 0; i < values.length; i++) + { + if (values[i] != null) + builder.add(values[i]); + } + return builder.build(); + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/FilterFactory.java b/src/java/org/apache/cassandra/utils/FilterFactory.java index 4cf0cbf74d..6276a3a338 100644 --- a/src/java/org/apache/cassandra/utils/FilterFactory.java +++ b/src/java/org/apache/cassandra/utils/FilterFactory.java @@ -17,15 +17,19 @@ */ package org.apache.cassandra.utils; +import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.utils.concurrent.Ref; import org.apache.cassandra.utils.obs.IBitSet; import org.apache.cassandra.utils.obs.OffHeapBitSet; public class FilterFactory { - public static final IFilter AlwaysPresent = new AlwaysPresentFilter(); + public static final IFilter AlwaysPresent = AlwaysPresentFilter.instance; private static final Logger logger = LoggerFactory.getLogger(FilterFactory.class); private static final long BITSET_EXCESS = 20; @@ -57,7 +61,7 @@ public class FilterFactory { assert maxFalsePosProbability <= 1.0 : "Invalid probability"; if (maxFalsePosProbability == 1.0) - return new AlwaysPresentFilter(); + return FilterFactory.AlwaysPresent; int bucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements); BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement, maxFalsePosProbability); return createFilter(spec.K, numElements, spec.bucketsPerElement); @@ -70,4 +74,56 @@ public class FilterFactory IBitSet bitset = new OffHeapBitSet(numBits); return new BloomFilter(hash, bitset); } + + private static class AlwaysPresentFilter implements IFilter + { + public static final AlwaysPresentFilter instance = new AlwaysPresentFilter(); + + private AlwaysPresentFilter() { } + + public boolean isPresent(FilterKey key) + { + return true; + } + + public void add(FilterKey key) { } + + public void clear() { } + + public void close() { } + + public IFilter sharedCopy() + { + return this; + } + + public Throwable close(Throwable accumulate) + { + return accumulate; + } + + public void addTo(Ref.IdentityCollection identities) + { + } + + public long serializedSize(boolean oldSerializationFormat) { return 0; } + + @Override + public void serialize(DataOutputStreamPlus out, boolean oldSerializationFormat) throws IOException + { + // no-op + } + + @Override + public long offHeapSize() + { + return 0; + } + + @Override + public boolean isInformative() + { + return false; + } + } } diff --git a/src/java/org/apache/cassandra/utils/IFilter.java b/src/java/org/apache/cassandra/utils/IFilter.java index b5eb2c4160..1c21c0a9bc 100644 --- a/src/java/org/apache/cassandra/utils/IFilter.java +++ b/src/java/org/apache/cassandra/utils/IFilter.java @@ -17,13 +17,18 @@ */ package org.apache.cassandra.utils; +import java.io.IOException; + +import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.utils.concurrent.SharedCloseable; public interface IFilter extends SharedCloseable { interface FilterKey { - /** Places the murmur3 hash of the key in the given long array of size at least two. */ + /** + * Places the murmur3 hash of the key in the given long array of size at least two. + */ void filterHash(long[] dest); } @@ -33,7 +38,9 @@ public interface IFilter extends SharedCloseable void clear(); - long serializedSize(); + long serializedSize(boolean oldSerializationFormat); + + void serialize(DataOutputStreamPlus out, boolean oldSerializationFormat) throws IOException; void close(); @@ -41,7 +48,10 @@ public interface IFilter extends SharedCloseable /** * Returns the amount of memory in bytes used off heap. + * * @return the amount of memory in bytes used off heap */ long offHeapSize(); + + boolean isInformative(); } diff --git a/src/java/org/apache/cassandra/utils/OutputHandler.java b/src/java/org/apache/cassandra/utils/OutputHandler.java index 820160d323..76eb34558f 100644 --- a/src/java/org/apache/cassandra/utils/OutputHandler.java +++ b/src/java/org/apache/cassandra/utils/OutputHandler.java @@ -26,20 +26,36 @@ import org.slf4j.LoggerFactory; public interface OutputHandler { // called when an important info need to be displayed - public void output(String msg); - - // called when a less important info need to be displayed - public void debug(String msg); - - // called when the user needs to be warn - public void warn(String msg); - public void warn(String msg, Throwable th); - public default void warn(Throwable th) + void output(String msg); + default void output(String msg, Object ... args) { - warn(th.getMessage(), th); + output(String.format(msg, args)); } - public static class LogOutput implements OutputHandler + // called when a less important info need to be displayed + void debug(String msg); + default void debug(String msg, Object ... args) + { + debug(String.format(msg, args)); + } + + // called when the user needs to be warn + void warn(String msg); + void warn(Throwable th, String msg); + default void warn(Throwable th) + { + warn(th, th.getMessage()); + } + default void warn(Throwable th, String msg, Object... args) + { + warn(th, String.format(msg, args)); + } + default void warn(String msg, Object ... args) + { + warn(String.format(msg, args)); + } + + class LogOutput implements OutputHandler { private static Logger logger = LoggerFactory.getLogger(LogOutput.class); @@ -58,13 +74,13 @@ public interface OutputHandler logger.warn(msg); } - public void warn(String msg, Throwable th) + public void warn(Throwable th, String msg) { logger.warn(msg, th); } } - public static class SystemOutput implements OutputHandler + class SystemOutput implements OutputHandler { public final boolean debug; public final boolean printStack; @@ -95,14 +111,14 @@ public interface OutputHandler public void warn(String msg) { - warn(msg, null); + warn((Throwable) null, msg); } - public void warn(String msg, Throwable th) + public void warn(Throwable th, String msg) { warnOut.println("WARNING: " + msg); if (printStack && th != null) th.printStackTrace(warnOut); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/StatusLogger.java b/src/java/org/apache/cassandra/utils/StatusLogger.java index dcb1135bfc..0850224f2f 100644 --- a/src/java/org/apache/cassandra/utils/StatusLogger.java +++ b/src/java/org/apache/cassandra/utils/StatusLogger.java @@ -19,17 +19,19 @@ package org.apache.cassandra.utils; import java.util.concurrent.locks.ReentrantLock; -import org.apache.cassandra.cache.*; -import org.apache.cassandra.metrics.CassandraMetricsRegistry; -import org.apache.cassandra.metrics.ThreadPoolMetrics; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.cache.AutoSavingCache; +import org.apache.cassandra.cache.IRowCacheEntry; +import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.cache.RowCacheKey; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.metrics.CassandraMetricsRegistry; +import org.apache.cassandra.metrics.ThreadPoolMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.CacheService; @@ -92,7 +94,7 @@ public class StatusLogger "MessagingService", "n/a", pendingLargeMessages + "/" + pendingSmallMessages)); // Global key/row cache information - AutoSavingCache keyCache = CacheService.instance.keyCache; + AutoSavingCache keyCache = CacheService.instance.keyCache; AutoSavingCache rowCache = CacheService.instance.rowCache; int keyCacheKeysToSave = DatabaseDescriptor.getKeyCacheKeysToSave(); diff --git a/src/java/org/apache/cassandra/utils/Throwables.java b/src/java/org/apache/cassandra/utils/Throwables.java index 8337a56b5c..d636711c55 100644 --- a/src/java/org/apache/cassandra/utils/Throwables.java +++ b/src/java/org/apache/cassandra/utils/Throwables.java @@ -18,19 +18,25 @@ */ package org.apache.cassandra.utils; -import org.apache.cassandra.io.util.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.Iterator; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.function.Predicate; import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; public final class Throwables @@ -184,6 +190,59 @@ public final class Throwables })); } + /** + * @see {@link #closeAndAddSuppressed(Throwable, Iterable)} + */ + public static void closeAndAddSuppressed(@Nonnull Throwable t, AutoCloseable... closeables) + { + closeAndAddSuppressed(t, Arrays.asList(closeables)); + } + + /** + * Do what {@link #closeAndAddSuppressed(Throwable, Iterable)} does, additionally filtering out all null closables. + */ + public static void closeNonNullAndAddSuppressed(@Nonnull Throwable t, AutoCloseable... closeables) + { + closeAndAddSuppressed(t, Iterables.filter(Arrays.asList(closeables), Objects::nonNull)); + } + + /** + * Closes all closables in the provided collections and accumulates the possible exceptions thrown when closing. + * + * @param accumulate non-null exception to accumulate errors thrown when closing the provided resources + * @param closeables closeables to be closed + */ + public static void closeAndAddSuppressed(@Nonnull Throwable accumulate, Iterable closeables) + { + Preconditions.checkNotNull(accumulate); + for (AutoCloseable closeable : closeables) + { + try + { + closeable.close(); + } + catch (Throwable ex) + { + accumulate.addSuppressed(ex); + } + } + } + + /** + * See {@link #close(Throwable, Iterable)} + */ + public static Throwable close(Throwable accumulate, AutoCloseable ... closeables) + { + return close(accumulate, Arrays.asList(closeables)); + } + + /** + * Closes all the resources in the provided collections and accumulates the possible exceptions thrown when closing. + * + * @param accumulate the initial value for the exception accumulator, can be {@code null} + * @param closeables closeables to be closed + * @return {@code null}, {@param accumulate} or the first exception thrown when closing the provided resources + */ public static Throwable close(Throwable accumulate, Iterable closeables) { for (AutoCloseable closeable : closeables) @@ -267,4 +326,11 @@ public final class Throwables { return unchecked(unwrapped(t)); } + + @VisibleForTesting + public static void assertAnyCause(Throwable err, Class cause) + { + if (!anyCauseMatches(err, cause::isInstance)) + throw new AssertionError("The exception is not caused by " + cause.getName(), err); + } } diff --git a/src/java/org/apache/cassandra/utils/concurrent/Refs.java b/src/java/org/apache/cassandra/utils/concurrent/Refs.java index e5d9c37a4c..fb6067e21d 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/Refs.java +++ b/src/java/org/apache/cassandra/utils/concurrent/Refs.java @@ -20,8 +20,13 @@ */ package org.apache.cassandra.utils.concurrent; -import java.util.*; - +import java.util.AbstractCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import javax.annotation.Nullable; import com.google.common.base.Function; @@ -204,7 +209,7 @@ public final class Refs> extends AbstractCollection i /** * Acquire a reference to all of the provided objects, or none */ - public static > Refs tryRef(Iterable reference) + public static > Refs tryRef(Iterable reference) { HashMap> refs = new HashMap<>(); for (T rc : reference) @@ -220,7 +225,7 @@ public final class Refs> extends AbstractCollection i return new Refs(refs); } - public static > Refs ref(Iterable reference) + public static > Refs ref(Iterable reference) { Refs refs = tryRef(reference); if (refs != null) diff --git a/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java b/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java index d643d1d88f..243e94a441 100644 --- a/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java +++ b/src/java/org/apache/cassandra/utils/concurrent/SharedCloseable.java @@ -34,4 +34,12 @@ public interface SharedCloseable extends AutoCloseable public Throwable close(Throwable accumulate); public void addTo(Ref.IdentityCollection identities); -} + + static T sharedCopyOrNull(T sharedCloseable) + { + if (sharedCloseable != null) + return (T) sharedCloseable.sharedCopy(); + + return null; + } +} \ No newline at end of file diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index cf0031e3b8..3822b02f79 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -44,7 +44,6 @@ import javax.management.Notification; import javax.management.NotificationListener; import com.google.common.annotations.VisibleForTesting; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,8 +95,8 @@ import org.apache.cassandra.hints.DTestSerializer; import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.io.IVersionedAsymmetricSerializer; -import org.apache.cassandra.io.sstable.IndexSummaryManager; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.DataOutputPlus; @@ -125,10 +124,10 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.StorageServiceMBean; import org.apache.cassandra.service.paxos.PaxosRepair; import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData; import org.apache.cassandra.service.reads.thresholds.CoordinatorWarnings; import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.StreamManager; -import org.apache.cassandra.service.paxos.uncommitted.UncommittedTableData; import org.apache.cassandra.streaming.StreamReceiveTask; import org.apache.cassandra.streaming.StreamTransferTask; import org.apache.cassandra.streaming.async.NettyStreamingChannel; diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java index eb1cdffd0e..405279aae6 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java @@ -64,9 +64,9 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableRunnab import org.apache.cassandra.distributed.impl.InstanceKiller; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.RepairParallelism; diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java index 1bf2d22d1e..4ca4f707d7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/JVMStabilityInspectorCorruptSSTableExceptionTest.java @@ -30,7 +30,6 @@ import org.apache.cassandra.config.Config.DiskFailurePolicy; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.rows.UnfilteredRowIterator; @@ -42,10 +41,9 @@ import org.apache.cassandra.distributed.shared.AbstractBuilder; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.ForwardingSSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; -import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.service.CassandraDaemon; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.Throwables; @@ -195,15 +193,9 @@ public class JVMStabilityInspectorCorruptSSTableExceptionTest extends TestBaseIm throw throwCorrupted(); } - @Override - public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) - { - throw throwCorrupted(); - } - private CorruptSSTableException throwCorrupted() { - throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFilename()); + throw new CorruptSSTableException(new IOException("failed to get position"), descriptor.baseFile()); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java b/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java index 0826954884..304136ba1f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/MultipleDataDirectoryTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.TOCComponent; import org.apache.cassandra.io.util.File; public class MultipleDataDirectoryTest extends TestBaseImpl @@ -151,14 +152,14 @@ public class MultipleDataDirectoryTest extends TestBaseImpl // getting a new file index in order to move SSTable between directories. second = cfs.newSSTableDescriptor(second.directory); // now we just move all sstables from first to second - for (Component component : SSTableReader.componentsFor(first)) + for (Component component : TOCComponent.loadOrCreate(first)) { - File file = new File(first.filenameFor(component)); + File file = first.fileFor(component); if (file.exists()) { try { - Files.copy(file.toPath(), new File(second.filenameFor(component)).toPath()); + Files.copy(file.toPath(), second.fileFor(component).toPath()); } catch (IOException e) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java index f6da888cfa..b702855f68 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java @@ -20,10 +20,8 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; import java.util.Collections; -import java.util.EnumSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.TimeUnit; @@ -31,14 +29,6 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import com.google.common.util.concurrent.Uninterruptibles; -import org.apache.cassandra.concurrent.SEPExecutor; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.AbstractReplicationStrategy; -import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.ReplicaLayout; -import org.apache.cassandra.locator.ReplicaUtils; -import org.apache.cassandra.utils.Throwables; import org.junit.Assert; import org.junit.Test; @@ -46,21 +36,32 @@ import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import org.apache.cassandra.db.*; +import org.apache.cassandra.concurrent.SEPExecutor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.metadata.MetadataComponent; -import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaLayout; +import org.apache.cassandra.locator.ReplicaUtils; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageProxy.LocalReadRunnable; import org.apache.cassandra.utils.DiagnosticSnapshotService; +import org.apache.cassandra.utils.Throwables; import static net.bytebuddy.matcher.ElementMatchers.named; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; @@ -516,10 +517,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl { SSTableReader sstable = sstables.next(); Descriptor descriptor = sstable.descriptor; - Map metadata = descriptor.getMetadataSerializer() - .deserialize(descriptor, EnumSet.of(MetadataType.STATS)); - - StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS); + StatsMetadata stats = StatsComponent.load(descriptor).statsMetadata(); Assert.assertEquals("repaired at is set for sstable: " + descriptor, stats.repairedAt, ActiveRepairService.UNREPAIRED_SSTABLE); @@ -568,10 +566,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl { SSTableReader sstable = sstables.next(); Descriptor descriptor = sstable.descriptor; - Map metadata = descriptor.getMetadataSerializer() - .deserialize(descriptor, EnumSet.of(MetadataType.STATS)); - - StatsMetadata stats = (StatsMetadata) metadata.get(MetadataType.STATS); + StatsMetadata stats = StatsComponent.load(descriptor).statsMetadata(); Assert.assertTrue("repaired at is not set for sstable: " + descriptor, stats.repairedAt > 0); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java index 8d05f3dc71..a752b7326d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java @@ -432,8 +432,16 @@ public class SSTableIdGenerationTest extends TestBaseImpl private static void assertSSTablesCount(Set descs, String tableName, int expectedSeqGenIds, int expectedUUIDGenIds) { - List seqSSTables = descs.stream().filter(desc -> desc.id instanceof SequenceBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList()); - List uuidSSTables = descs.stream().filter(desc -> desc.id instanceof UUIDBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList()); + List seqSSTables = descs.stream() + .filter(desc -> desc.id instanceof SequenceBasedSSTableId) + .map(descriptor -> descriptor.baseFile().toString()) + .sorted() + .collect(Collectors.toList()); + List uuidSSTables = descs.stream() + .filter(desc -> desc.id instanceof UUIDBasedSSTableId) + .map(descriptor -> descriptor.baseFile().toString()) + .sorted() + .collect(Collectors.toList()); assertThat(seqSSTables).describedAs("SSTables of %s with sequence based id", tableName).hasSize(expectedSeqGenIds); assertThat(uuidSSTables).describedAs("SSTables of %s with UUID based id", tableName).hasSize(expectedUUIDGenIds); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java b/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java index a096dbdff6..8e2f86622e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/TableLevelIncrementalBackupsTest.java @@ -18,19 +18,20 @@ package org.apache.cassandra.distributed.test; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; -import org.junit.Test; - import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.junit.Test; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; + import static org.apache.cassandra.Util.getBackups; import static org.apache.cassandra.distributed.Cluster.build; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; @@ -154,7 +155,11 @@ public class TableLevelIncrementalBackupsTest extends TestBaseImpl private static void assertSSTablesCount(Set descs, String tableName, int expectedSeqGenIds) { - List seqSSTables = descs.stream().filter(desc -> desc.id instanceof SequenceBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList()); + List seqSSTables = descs.stream() + .filter(desc -> desc.id instanceof SequenceBasedSSTableId) + .map(descriptor -> descriptor.baseFile().toString()) + .sorted() + .collect(Collectors.toList()); assertThat(seqSSTables).describedAs("SSTables of %s with sequence based id", tableName).hasSize(expectedSeqGenIds); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java index 445e349885..2bdcaf16c6 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/UpgradeSSTablesTest.java @@ -41,7 +41,6 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICluster; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.LogAction; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.CountDownLatch; @@ -262,7 +261,7 @@ public class UpgradeSSTablesTest extends TestBaseImpl cfs.disableAutoCompaction(); for (SSTableReader tbl : cfs.getLiveSSTables()) { - maxTs = Math.max(maxTs, tbl.getCreationTimeFor(Component.DATA)); + maxTs = Math.max(maxTs, tbl.getDataCreationTime()); } return maxTs; }).apply(KEYSPACE); @@ -284,7 +283,7 @@ public class UpgradeSSTablesTest extends TestBaseImpl assert liveSSTables.size() == 2 : String.format("Expected 2 sstables, but got " + liveSSTables.size()); for (SSTableReader tbl : liveSSTables) { - if (tbl.getCreationTimeFor(Component.DATA) <= maxTs) + if (tbl.getDataCreationTime() <= maxTs) count++; else skipped++; diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java index fc52ab6de1..f2772bc17f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/StreamCloseInMiddleTest.java @@ -37,8 +37,8 @@ import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter; -import org.apache.cassandra.io.sstable.format.big.BigTableZeroCopyWriter; +import org.apache.cassandra.io.sstable.RangeAwareSSTableWriter; +import org.apache.cassandra.io.sstable.SSTableZeroCopyWriter; import org.apache.cassandra.io.util.SequentialWriter; import org.assertj.core.api.Assertions; @@ -150,7 +150,7 @@ public class StreamCloseInMiddleTest extends TestBaseImpl @SuppressWarnings("unused") public static int writeDirectlyToChannel(ByteBuffer buf, @SuperCall Callable zuper) throws Exception { - if (isCaller(BigTableZeroCopyWriter.class.getName(), "write")) + if (isCaller(SSTableZeroCopyWriter.class.getName(), "write")) throw new java.nio.channels.ClosedChannelException(); // different context; pass through return zuper.call(); @@ -193,4 +193,4 @@ public class StreamCloseInMiddleTest extends TestBaseImpl .load(classLoader, ClassLoadingStrategy.Default.INJECTION); } } -} +} \ No newline at end of file diff --git a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java index 33e6cd6f38..1f3b6f8787 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/thresholds/RowIndexSizeWarningTest.java @@ -26,6 +26,7 @@ import org.junit.BeforeClass; import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import static org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES; import static org.assertj.core.api.Assertions.assertThat; @@ -37,6 +38,9 @@ public class RowIndexSizeWarningTest extends AbstractClientSizeWarning { AbstractClientSizeWarning.setupClass(); + //noinspection Convert2MethodRef + Assume.assumeTrue(CLUSTER.get(1).callOnInstance(() -> BigFormat.isDefault())); + CLUSTER.stream().forEach(i -> i.runOnInstance(() -> { DatabaseDescriptor.setRowIndexReadSizeWarnThreshold(new DataStorageSpec.LongBytesBound(1, KIBIBYTES)); DatabaseDescriptor.setRowIndexReadSizeFailThreshold(new DataStorageSpec.LongBytesBound(2, KIBIBYTES)); diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java index 9012dcca27..302610c47d 100644 --- a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java +++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java @@ -23,16 +23,14 @@ import java.nio.ByteBuffer; import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Set; import com.google.common.util.concurrent.RateLimiter; -import org.apache.cassandra.cache.InstrumentingCache; -import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; @@ -43,19 +41,24 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.ISSTableScanner; -import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.IVerifier; +import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.KeyReader; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.ChannelProxy; +import org.apache.cassandra.io.util.CheckedFunction; +import org.apache.cassandra.io.util.DataIntegrityMetadata; import org.apache.cassandra.io.util.FileDataInput; -import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.EstimatedHistogram; -import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Ref; @@ -63,14 +66,25 @@ public abstract class ForwardingSSTableReader extends SSTableReader { private final SSTableReader delegate; + private static class Builder extends SSTableReader.Builder + { + public Builder(SSTableReader delegate) + { + super(delegate.descriptor); + delegate.unbuildTo(this, false); + } + + @Override + public ForwardingSSTableReader buildInternal(Owner owner) + { + throw new UnsupportedOperationException(); + } + } + public ForwardingSSTableReader(SSTableReader delegate) { - super(delegate.descriptor, SSTable.componentsFor(delegate.descriptor), - TableMetadataRef.forOfflineTools(delegate.metadata()), delegate.maxDataAge, delegate.getSSTableMetadata(), - delegate.openReason, delegate.header, delegate.indexSummary, delegate.dfile, delegate.ifile, delegate.bf); + super(new Builder(delegate), delegate.owner().orElse(null)); this.delegate = delegate; - this.first = delegate.first; - this.last = delegate.last; } @Override @@ -91,12 +105,24 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.getFilename(); } + @Override + public boolean mayContainAssumingKeyIsInRange(DecoratedKey key) + { + return delegate.mayContainAssumingKeyIsInRange(key); + } + @Override public void setupOnline() { delegate.setupOnline(); } + @Override + public R runWithLock(CheckedFunction task) throws E + { + return delegate.runWithLock(task); + } + @Override public void setReplaced() { @@ -122,15 +148,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public SSTableReader cloneWithNewStart(DecoratedKey newStart, Runnable runOnClose) + public SSTableReader cloneWithNewStart(DecoratedKey newStart) { - return delegate.cloneWithNewStart(newStart, runOnClose); - } - - @Override - public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException - { - return delegate.cloneWithNewSummarySamplingLevel(parent, samplingLevel); + return delegate.cloneWithNewStart(newStart); } @Override @@ -140,39 +160,21 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public int getIndexSummarySamplingLevel() + public void releaseInMemoryComponents() { - return delegate.getIndexSummarySamplingLevel(); + delegate.releaseInMemoryComponents(); } @Override - public long getIndexSummaryOffHeapSize() + public void validate() { - return delegate.getIndexSummaryOffHeapSize(); + delegate.validate(); } @Override - public int getMinIndexInterval() + protected void closeInternalComponent(AutoCloseable closeable) { - return delegate.getMinIndexInterval(); - } - - @Override - public double getEffectiveIndexInterval() - { - return delegate.getEffectiveIndexInterval(); - } - - @Override - public void releaseSummary() - { - delegate.releaseSummary(); - } - - @Override - public long getIndexScanPosition(PartitionPosition key) - { - return delegate.getIndexScanPosition(key); + delegate.closeInternalComponent(closeable); } @Override @@ -187,24 +189,6 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.getCompressionMetadataOffHeapSize(); } - @Override - public IFilter getBloomFilter() - { - return delegate.getBloomFilter(); - } - - @Override - public long getBloomFilterSerializedSize() - { - return delegate.getBloomFilterSerializedSize(); - } - - @Override - public long getBloomFilterOffHeapSize() - { - return delegate.getBloomFilterOffHeapSize(); - } - @Override public long estimatedKeys() { @@ -217,24 +201,6 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.estimatedKeysForRanges(ranges); } - @Override - public int getIndexSummarySize() - { - return delegate.getIndexSummarySize(); - } - - @Override - public int getMaxIndexSummarySize() - { - return delegate.getMaxIndexSummarySize(); - } - - @Override - public byte[] getIndexSummaryKey(int index) - { - return delegate.getIndexSummaryKey(index); - } - @Override public Iterable getKeySamples(Range range) { @@ -248,39 +214,15 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public KeyCacheKey getCacheKey(DecoratedKey key) + protected long getPosition(PartitionPosition key, Operator op, boolean updateStats, boolean permitMatchPastLast, SSTableReadsListener listener) { - return delegate.getCacheKey(key); + return delegate.getPosition(key, op, updateStats, permitMatchPastLast, listener); } @Override - public void cacheKey(DecoratedKey key, RowIndexEntry info) + protected AbstractRowIndexEntry getRowIndexEntry(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast, SSTableReadsListener listener) { - delegate.cacheKey(key, info); - } - - @Override - public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats) - { - return delegate.getCachedPosition(key, updateStats); - } - - @Override - protected RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats) - { - return delegate.getCachedPosition(unifiedKey, updateStats); - } - - @Override - public boolean isKeyCacheEnabled() - { - return delegate.isKeyCacheEnabled(); - } - - @Override - protected RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast, SSTableReadsListener listener) - { - return delegate.getPosition(key, op, updateCacheAndStats, permitMatchPastLast, listener); + return delegate.getRowIndexEntry(key, op, updateCacheAndStats, permitMatchPastLast, listener); } @Override @@ -290,15 +232,21 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public UnfilteredRowIterator rowIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed) + public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, long dataPosition, boolean tombstoneOnly) { - return delegate.rowIterator(file, key, indexEntry, slices, selectedColumns, reversed); + return delegate.simpleIterator(file, key, dataPosition, tombstoneOnly); } @Override - public UnfilteredRowIterator simpleIterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, boolean tombstoneOnly) + public KeyReader keyReader() throws IOException { - return delegate.simpleIterator(file, key, indexEntry, tombstoneOnly); + return delegate.keyReader(); + } + + @Override + public KeyIterator keyIterator() throws IOException + { + return delegate.keyIterator(); } @Override @@ -398,9 +346,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public boolean newSince(long age) + public boolean newSince(long timestampMillis) { - return delegate.newSince(age); + return delegate.newSince(timestampMillis); } @Override @@ -409,6 +357,12 @@ public abstract class ForwardingSSTableReader extends SSTableReader delegate.createLinks(snapshotDirectoryPath); } + @Override + public void createLinks(String snapshotDirectoryPath, RateLimiter rateLimiter) + { + delegate.createLinks(snapshotDirectoryPath, rateLimiter); + } + @Override public boolean isRepaired() { @@ -416,9 +370,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public DecoratedKey keyAt(long indexPosition) throws IOException + public DecoratedKey keyAtPositionFromSecondaryIndex(long keyPositionFromSecondaryIndex) throws IOException { - return delegate.keyAt(indexPosition); + return delegate.keyAtPositionFromSecondaryIndex(keyPositionFromSecondaryIndex); } @Override @@ -451,36 +405,6 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.intersects(ranges); } - @Override - public long getBloomFilterFalsePositiveCount() - { - return delegate.getBloomFilterFalsePositiveCount(); - } - - @Override - public long getRecentBloomFilterFalsePositiveCount() - { - return delegate.getRecentBloomFilterFalsePositiveCount(); - } - - @Override - public long getBloomFilterTruePositiveCount() - { - return delegate.getBloomFilterTruePositiveCount(); - } - - @Override - public long getRecentBloomFilterTruePositiveCount() - { - return delegate.getRecentBloomFilterTruePositiveCount(); - } - - @Override - public InstrumentingCache getKeyCache() - { - return delegate.getKeyCache(); - } - @Override public EstimatedHistogram getEstimatedPartitionSize() { @@ -577,6 +501,18 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.getSSTableLevel(); } + @Override + public void mutateLevelAndReload(int newLevel) throws IOException + { + delegate.mutateLevelAndReload(newLevel); + } + + @Override + public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException + { + delegate.mutateRepairedAndReload(newRepairedAt, newPendingRepair, isTransient); + } + @Override public void reloadSSTableMetadata() throws IOException { @@ -602,9 +538,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public RandomAccessReader openIndexReader() + public void trySkipFileCacheBefore(DecoratedKey key) { - return delegate.openIndexReader(); + delegate.trySkipFileCacheBefore(key); } @Override @@ -614,33 +550,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - public ChannelProxy getIndexChannel() + public long getDataCreationTime() { - return delegate.getIndexChannel(); - } - - @Override - public FileHandle getIndexFile() - { - return delegate.getIndexFile(); - } - - @Override - public long getCreationTimeFor(Component component) - { - return delegate.getCreationTimeFor(component); - } - - @Override - public long getKeyCacheHit() - { - return delegate.getKeyCacheHit(); - } - - @Override - public long getKeyCacheRequest() - { - return delegate.getKeyCacheRequest(); + return delegate.getDataCreationTime(); } @Override @@ -674,7 +586,13 @@ public abstract class ForwardingSSTableReader extends SSTableReader } @Override - void setup(boolean trackHotness) + protected List setupInstance(boolean trackHotness) + { + return delegate.setupInstance(trackHotness); + } + + @Override + public void setup(boolean trackHotness) { delegate.setup(trackHotness); } @@ -709,12 +627,6 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.decorateKey(key); } - @Override - public String getIndexFilename() - { - return delegate.getIndexFilename(); - } - @Override public String getColumnFamilyName() { @@ -739,6 +651,30 @@ public abstract class ForwardingSSTableReader extends SSTableReader return delegate.bytesOnDisk(); } + @Override + public long logicalBytesOnDisk() + { + return delegate.logicalBytesOnDisk(); + } + + @Override + public Set getComponents() + { + return delegate.getComponents(); + } + + @Override + public UnfilteredRowIterator rowIterator(DecoratedKey key) + { + return delegate.rowIterator(key); + } + + @Override + public void maybePersistSSTableReadMeter() + { + delegate.maybePersistSSTableReadMeter(); + } + @Override public String toString() { @@ -756,4 +692,40 @@ public abstract class ForwardingSSTableReader extends SSTableReader { return delegate.getBounds(); } + + @Override + public IVerifier getVerifier(ColumnFamilyStore cfs, OutputHandler outputHandler, boolean isOffline, IVerifier.Options options) + { + return delegate.getVerifier(cfs, outputHandler, isOffline, options); + } + + @Override + protected void notifySelected(SSTableReadsListener.SelectionReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats, AbstractRowIndexEntry entry) + { + delegate.notifySelected(reason, localListener, op, updateStats, entry); + } + + @Override + protected void notifySkipped(SSTableReadsListener.SkippingReason reason, SSTableReadsListener localListener, Operator op, boolean updateStats) + { + delegate.notifySkipped(reason, localListener, op, updateStats); + } + + @Override + public boolean isEstimationInformative() + { + return delegate.isEstimationInformative(); + } + + @Override + public DataIntegrityMetadata.ChecksumValidator maybeGetChecksumValidator() throws IOException + { + return delegate.maybeGetChecksumValidator(); + } + + @Override + public DataIntegrityMetadata.FileDigestValidator maybeGetDigestValidator() throws IOException + { + return delegate.maybeGetDigestValidator(); + } } diff --git a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java index 32e048dca7..b03372e4a3 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/BloomFilterSerializerBench.java @@ -18,7 +18,6 @@ package org.apache.cassandra.test.microbench; -import org.apache.cassandra.io.util.DataOutputStreamPlus; import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.TimeUnit; @@ -53,7 +52,6 @@ import org.openjdk.jmh.annotations.Warmup; @State(Scope.Benchmark) public class BloomFilterSerializerBench { - @Param({"1", "10", "100", "1024"}) private long numElemsInK; @@ -67,6 +65,8 @@ public class BloomFilterSerializerBench private ByteBuffer testVal = ByteBuffer.wrap(new byte[] { 0, 1}); + private static final BloomFilterSerializer serializer = BloomFilterSerializer.forVersion(false); + @Benchmark public void serializationTest() throws IOException { @@ -75,16 +75,16 @@ public class BloomFilterSerializerBench { BloomFilter filter = (BloomFilter) FilterFactory.getFilter(numElemsInK * 1024, 0.01d); filter.add(wrap(testVal)); - DataOutputStreamPlus out = new FileOutputStreamPlus(file); + FileOutputStreamPlus out = new FileOutputStreamPlus(file); if (oldBfFormat) SerializationsTest.serializeOldBfFormat(filter, out); else - BloomFilterSerializer.serialize(filter, out); + serializer.serialize(filter, out); out.close(); filter.close(); FileInputStreamPlus in = new FileInputStreamPlus(file); - BloomFilter filter2 = BloomFilterSerializer.deserialize(in, oldBfFormat); + BloomFilter filter2 = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(in); FileUtils.closeQuietly(in); filter2.close(); } diff --git a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java index de788b7aad..d67e68314b 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java @@ -31,9 +31,9 @@ import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.service.CacheService; @@ -107,7 +107,7 @@ public class CacheLoaderBench extends CQLTester sstable.getPosition(Util.dk("key"), SSTableReader.Operator.EQ); } - AutoSavingCache keyCache = CacheService.instance.keyCache; + AutoSavingCache keyCache = CacheService.instance.keyCache; // serialize to file keyCache.submitWrite(keyCache.size()).get(); @@ -116,7 +116,7 @@ public class CacheLoaderBench extends CQLTester @Setup(Level.Invocation) public void setupKeyCache() { - AutoSavingCache keyCache = CacheService.instance.keyCache; + AutoSavingCache keyCache = CacheService.instance.keyCache; keyCache.clear(); } @@ -130,7 +130,7 @@ public class CacheLoaderBench extends CQLTester @Benchmark public void keyCacheLoadTest() throws Throwable { - AutoSavingCache keyCache = CacheService.instance.keyCache; + AutoSavingCache keyCache = CacheService.instance.keyCache; keyCache.loadSavedAsync().get(); } diff --git a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java index c589ca5d7e..b1f8870c96 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CachingBenchTest.java @@ -29,15 +29,14 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Predicate; import com.google.common.collect.Iterables; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assert; - +import org.apache.cassandra.cache.ChunkCache; import org.apache.cassandra.config.Config.CommitLogSync; import org.apache.cassandra.config.Config.DiskAccessMode; -import org.apache.cassandra.cache.ChunkCache; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; @@ -201,7 +200,7 @@ public class CachingBenchTest extends CQLTester { id.set(0); compactionTimeNanos = 0; - ChunkCache.instance.enable(cacheEnabled); + DatabaseDescriptor.setFileCacheEnabled(cacheEnabled); DatabaseDescriptor.setDiskAccessMode(mode); alterTable("ALTER TABLE %s WITH compaction = { 'class' : '" + compactionClass + "' };"); alterTable("ALTER TABLE %s WITH compression = { 'sstable_compression' : '" + compressorClass + "' };"); @@ -385,4 +384,4 @@ public class CachingBenchTest extends CQLTester } return instances; } -} +} \ No newline at end of file diff --git a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java index 8650ff65c0..8d0ed46e5a 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java +++ b/test/simulator/main/org/apache/cassandra/simulator/debug/Reconcile.java @@ -35,7 +35,9 @@ import java.util.zip.GZIPInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.Util; import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import org.apache.cassandra.io.util.File; import org.apache.cassandra.simulator.ClusterSimulation; import org.apache.cassandra.simulator.RandomSource; @@ -433,8 +435,8 @@ public class Reconcile File timeFile = new File(new File(loadFromDir), Long.toHexString(seed) + ".time.gz"); try (BufferedReader eventIn = new BufferedReader(new InputStreamReader(new GZIPInputStream(eventFile.newInputStream()))); - DataInputPlus.DataInputStreamPlus rngIn = new DataInputPlus.DataInputStreamPlus(rngFile.exists() && withRng != NONE ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0])); - DataInputPlus.DataInputStreamPlus timeIn = new DataInputPlus.DataInputStreamPlus(timeFile.exists() && withTime != NONE ? new GZIPInputStream(timeFile.newInputStream()) : new ByteArrayInputStream(new byte[0]))) + DataInputStreamPlus rngIn = Util.DataInputStreamPlusImpl.wrap(rngFile.exists() && withRng != NONE ? new GZIPInputStream(rngFile.newInputStream()) : new ByteArrayInputStream(new byte[0])); + DataInputStreamPlus timeIn = Util.DataInputStreamPlusImpl.wrap(timeFile.exists() && withTime != NONE ? new GZIPInputStream(timeFile.newInputStream()) : new ByteArrayInputStream(new byte[0]))) { boolean inputHasWaitSites, inputHasWakeSites, inputHasRngCallSites, inputHasTimeCallSites; { diff --git a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java index 08e1b2eab0..60ebf47781 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java +++ b/test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java @@ -42,17 +42,17 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.simulator.systems.NonInterceptible; -import org.apache.cassandra.simulator.systems.NonInterceptible.Permit; -import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.service.paxos.Commit; import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.simulator.systems.NonInterceptible; +import org.apache.cassandra.simulator.systems.NonInterceptible.Permit; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Shared; +import org.apache.cassandra.utils.TimeUUID; import static java.lang.Long.max; import static java.util.Arrays.stream; diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java index 10cb082285..fa59f0d67d 100644 --- a/test/unit/org/apache/cassandra/ServerTestUtils.java +++ b/test/unit/org/apache/cassandra/ServerTestUtils.java @@ -21,16 +21,22 @@ import java.io.IOException; import java.net.UnknownHostException; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.audit.AuditLogManager; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport; import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.AbstractEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; @@ -203,4 +209,22 @@ public final class ServerTestUtils private ServerTestUtils() { } + + public static List getLiveBigTableReaders(ColumnFamilyStore cfs) + { + return cfs.getLiveSSTables() + .stream() + .filter(BigTableReader.class::isInstance) + .map(BigTableReader.class::cast) + .collect(Collectors.toList()); + } + + public static > List getLiveIndexSummarySupportingReaders(ColumnFamilyStore cfs) + { + return cfs.getLiveSSTables() + .stream() + .filter(IndexSummarySupport.class::isInstance) + .map(r -> (R) r) + .collect(Collectors.toList()); + } } diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index b0b5964117..1285ad9291 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -20,9 +20,11 @@ package org.apache.cassandra; */ import java.io.Closeable; +import java.io.DataInputStream; import java.io.EOFException; import java.io.IOError; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.net.UnknownHostException; import java.nio.ByteBuffer; @@ -113,7 +115,10 @@ import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableLoader; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -137,6 +142,8 @@ import org.apache.cassandra.utils.FilterFactory; import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.Throwables; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.mockito.internal.stubbing.defaultanswers.ForwardsInvocations; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -465,10 +472,10 @@ public class Util assert iterator.hasNext() : "Expecting one row in one partition but got nothing"; try (RowIterator partition = iterator.next()) { - assert !iterator.hasNext() : "Expecting a single partition but got more"; assert partition.hasNext() : "Expecting one row in one partition but got an empty partition"; Row row = partition.next(); assert !partition.hasNext() : "Expecting a single row but got more"; + assert !iterator.hasNext() : "Expecting a single partition but got more"; return row; } } @@ -851,7 +858,7 @@ public class Util { if (sst.name().contains("Data")) { - Descriptor d = Descriptor.fromFilename(sst.absolutePath()); + Descriptor d = Descriptor.fromFileWithComponent(sst, false).left; assertTrue(liveIdentifiers.contains(d.id)); fileCount++; } @@ -1063,7 +1070,7 @@ public class Util { for (SSTableReader sstable : sstables) { - sstable = sstable.cloneAndReplace(FilterFactory.AlwaysPresent); + sstable = ((SSTableReaderWithFilter) sstable).cloneAndReplace(FilterFactory.AlwaysPresent); txn.update(sstable, true); txn.checkpoint(); } @@ -1071,7 +1078,7 @@ public class Util } for (SSTableReader reader : cfs.getLiveSSTables()) - assertEquals(FilterFactory.AlwaysPresent, reader.getBloomFilter()); + assertEquals(0, ((SSTableReaderWithFilter) reader).getFilterOffHeapSize()); } /** @@ -1236,4 +1243,23 @@ public class Util { view.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); } + + public static class DataInputStreamPlusImpl extends DataInputStream implements DataInputPlus + { + private DataInputStreamPlusImpl(InputStream in) + { + super(in); + } + + public static DataInputStreamPlus wrap(InputStream in) + { + DataInputStreamPlusImpl impl = new DataInputStreamPlusImpl(in); + return Mockito.mock(DataInputStreamPlus.class, new ForwardsInvocations(impl)); + } + } + + public static RuntimeException testMustBeImplementedForSSTableFormat() + { + return new UnsupportedOperationException("Test must be implemented for sstable format " + SSTableFormat.Type.current().info.getClass().getName()); + } } diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java index 040409e424..570f8c311d 100644 --- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java +++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java @@ -17,20 +17,26 @@ */ package org.apache.cassandra.cache; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.marshal.AsciiType; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.Assert; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.ByteBufferUtil; @@ -44,6 +50,8 @@ public class AutoSavingCacheTest @BeforeClass public static void defineSchema() throws ConfigurationException { + DatabaseDescriptor.daemonInitialization(); + Assume.assumeTrue(KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current())); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), @@ -85,7 +93,7 @@ public class AutoSavingCacheTest for (SSTableReader sstable : cfs.getLiveSSTables()) sstable.getPosition(Util.dk("key1"), SSTableReader.Operator.EQ); - AutoSavingCache keyCache = CacheService.instance.keyCache; + AutoSavingCache keyCache = CacheService.instance.keyCache; // serialize to file keyCache.submitWrite(keyCache.size()).get(); diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index 3ee05cf96f..38b997ab14 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -37,7 +37,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import com.google.common.base.Throwables; import org.junit.Test; import org.apache.cassandra.utils.Pair; @@ -46,7 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Verifies that {@link DatabaseDescriptor#clientInitialization()} } and a couple of apply methods + * Verifies that {@link DatabaseDescriptor#clientInitialization()} and a couple of apply methods * do not somehow lazily initialize any unwanted part of Cassandra like schema, commit log or start * unexpected threads. * @@ -56,6 +55,14 @@ import static org.junit.Assert.fail; public class DatabaseDescriptorRefTest { static final String[] validClasses = { + "org.apache.cassandra.ConsoleAppender", + "org.apache.cassandra.ConsoleAppender$1", + "org.apache.cassandra.ConsoleAppenderBeanInfo", + "org.apache.cassandra.ConsoleAppenderCustomizer", + "org.apache.cassandra.LogbackStatusListener", + "org.apache.cassandra.LogbackStatusListener$ToLoggerOutputStream", + "org.apache.cassandra.LogbackStatusListener$WrappedPrintStream", + "org.apache.cassandra.TeeingAppender", "org.apache.cassandra.audit.AuditLogOptions", "org.apache.cassandra.audit.BinAuditLogger", "org.apache.cassandra.audit.BinLogAuditLogger", @@ -63,19 +70,18 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.auth.AllowAllInternodeAuthenticator", "org.apache.cassandra.auth.AuthCache$BulkLoader", "org.apache.cassandra.auth.Cacheable", - "org.apache.cassandra.auth.IInternodeAuthenticator", "org.apache.cassandra.auth.IAuthenticator", "org.apache.cassandra.auth.IAuthorizer", - "org.apache.cassandra.auth.IRoleManager", + "org.apache.cassandra.auth.IInternodeAuthenticator", "org.apache.cassandra.auth.INetworkAuthorizer", - "org.apache.cassandra.config.DatabaseDescriptor", + "org.apache.cassandra.auth.IRoleManager", "org.apache.cassandra.config.CassandraRelevantProperties", "org.apache.cassandra.config.CassandraRelevantProperties$PropertyConverter", - "org.apache.cassandra.config.ConfigurationLoader", "org.apache.cassandra.config.Config", "org.apache.cassandra.config.Config$1", - "org.apache.cassandra.config.Config$CommitLogSync", "org.apache.cassandra.config.Config$CommitFailurePolicy", + "org.apache.cassandra.config.Config$CommitLogSync", + "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy", "org.apache.cassandra.config.Config$DiskAccessMode", "org.apache.cassandra.config.Config$DiskFailurePolicy", "org.apache.cassandra.config.Config$DiskOptimizationStrategy", @@ -87,13 +93,15 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$PaxosVariant", "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy", "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy", - "org.apache.cassandra.config.Config$CorruptedTombstoneStrategy", - "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit", + "org.apache.cassandra.config.ConfigBeanInfo", + "org.apache.cassandra.config.ConfigCustomizer", + "org.apache.cassandra.config.ConfigurationLoader", "org.apache.cassandra.config.DataRateSpec", "org.apache.cassandra.config.DataRateSpec$DataRateUnit", "org.apache.cassandra.config.DataRateSpec$DataRateUnit$1", "org.apache.cassandra.config.DataRateSpec$DataRateUnit$2", "org.apache.cassandra.config.DataRateSpec$DataRateUnit$3", + "org.apache.cassandra.config.DataRateSpec$LongBytesPerSecondBound", "org.apache.cassandra.config.DataStorageSpec", "org.apache.cassandra.config.DataStorageSpec$DataStorageUnit", "org.apache.cassandra.config.DataStorageSpec$DataStorageUnit$1", @@ -105,52 +113,59 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.DataStorageSpec$IntMebibytesBound", "org.apache.cassandra.config.DataStorageSpec$LongBytesBound", "org.apache.cassandra.config.DataStorageSpec$LongMebibytesBound", + "org.apache.cassandra.config.DatabaseDescriptor", + "org.apache.cassandra.config.DatabaseDescriptor$ByteUnit", "org.apache.cassandra.config.DurationSpec", - "org.apache.cassandra.config.DataRateSpec$LongBytesPerSecondBound", + "org.apache.cassandra.config.DurationSpec$IntMillisecondsBound", + "org.apache.cassandra.config.DurationSpec$IntMinutesBound", + "org.apache.cassandra.config.DurationSpec$IntSecondsBound", "org.apache.cassandra.config.DurationSpec$LongMillisecondsBound", "org.apache.cassandra.config.DurationSpec$LongNanosecondsBound", "org.apache.cassandra.config.DurationSpec$LongSecondsBound", - "org.apache.cassandra.config.DurationSpec$IntMillisecondsBound", - "org.apache.cassandra.config.DurationSpec$IntSecondsBound", - "org.apache.cassandra.config.DurationSpec$IntMinutesBound", "org.apache.cassandra.config.EncryptionOptions", "org.apache.cassandra.config.EncryptionOptions$ClientEncryptionOptions", "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions", "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$InternodeEncryption", "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptions$OutgoingEncryptedPortSource", + "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsBeanInfo", + "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer", + "org.apache.cassandra.config.EncryptionOptionsBeanInfo", + "org.apache.cassandra.config.EncryptionOptionsCustomizer", "org.apache.cassandra.config.GuardrailsOptions", "org.apache.cassandra.config.GuardrailsOptions$Config", "org.apache.cassandra.config.GuardrailsOptions$ConsistencyLevels", "org.apache.cassandra.config.GuardrailsOptions$TableProperties", "org.apache.cassandra.config.ParameterizedClass", "org.apache.cassandra.config.ReplicaFilteringProtectionOptions", - "org.apache.cassandra.config.YamlConfigurationLoader", - "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker", - "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1", - "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor", - "org.apache.cassandra.config.TransparentDataEncryptionOptions", "org.apache.cassandra.config.StartupChecksOptions", "org.apache.cassandra.config.SubnetGroups", "org.apache.cassandra.config.TrackWarnings", + "org.apache.cassandra.config.TransparentDataEncryptionOptions", + "org.apache.cassandra.config.YamlConfigurationLoader", + "org.apache.cassandra.config.YamlConfigurationLoader$CustomConstructor", + "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker", + "org.apache.cassandra.config.YamlConfigurationLoader$PropertiesChecker$1", + "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement", + "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement", "org.apache.cassandra.db.ConsistencyLevel", - "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory", - "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory", "org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager", - "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC", - "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard", "org.apache.cassandra.db.commitlog.CommitLog", "org.apache.cassandra.db.commitlog.CommitLogMBean", + "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC", + "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerFactory", + "org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard", + "org.apache.cassandra.db.commitlog.DefaultCommitLogSegmentMgrFactory", "org.apache.cassandra.db.guardrails.GuardrailsConfig", - "org.apache.cassandra.db.guardrails.GuardrailsConfigMBean", "org.apache.cassandra.db.guardrails.GuardrailsConfig$ConsistencyLevels", "org.apache.cassandra.db.guardrails.GuardrailsConfig$TableProperties", + "org.apache.cassandra.db.guardrails.GuardrailsConfigMBean", "org.apache.cassandra.db.guardrails.Values$Config", + "org.apache.cassandra.db.rows.UnfilteredSource", "org.apache.cassandra.dht.IPartitioner", "org.apache.cassandra.distributed.api.IInstance", - "org.apache.cassandra.distributed.api.IIsolatedExecutor", - "org.apache.cassandra.distributed.shared.InstanceClassLoader", - "org.apache.cassandra.distributed.impl.InstanceConfig", "org.apache.cassandra.distributed.api.IInvokableInstance", + "org.apache.cassandra.distributed.api.IIsolatedExecutor", + "org.apache.cassandra.distributed.impl.InstanceConfig", "org.apache.cassandra.distributed.impl.InvokableInstance$CallableNoExcept", "org.apache.cassandra.distributed.impl.InvokableInstance$InstanceFunction", "org.apache.cassandra.distributed.impl.InvokableInstance$SerializableBiConsumer", @@ -163,67 +178,93 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.distributed.impl.InvokableInstance$TriFunction", "org.apache.cassandra.distributed.impl.Message", "org.apache.cassandra.distributed.impl.NetworkTopology", - "org.apache.cassandra.exceptions.ConfigurationException", - "org.apache.cassandra.exceptions.RequestValidationException", + "org.apache.cassandra.distributed.shared.InstanceClassLoader", "org.apache.cassandra.exceptions.CassandraException", + "org.apache.cassandra.exceptions.ConfigurationException", "org.apache.cassandra.exceptions.InvalidRequestException", + "org.apache.cassandra.exceptions.RequestValidationException", "org.apache.cassandra.exceptions.TransportException", "org.apache.cassandra.fql.FullQueryLogger", "org.apache.cassandra.fql.FullQueryLoggerOptions", "org.apache.cassandra.gms.IFailureDetector", - "org.apache.cassandra.locator.IEndpointSnitch", - "org.apache.cassandra.io.FSWriteError", "org.apache.cassandra.io.FSError", + "org.apache.cassandra.io.FSWriteError", "org.apache.cassandra.io.compress.ICompressor", "org.apache.cassandra.io.compress.ICompressor$Uses", "org.apache.cassandra.io.compress.LZ4Compressor", + "org.apache.cassandra.io.sstable.Component", + "org.apache.cassandra.io.sstable.Component$Type", + "org.apache.cassandra.io.sstable.IScrubber", + "org.apache.cassandra.io.sstable.MetricsProviders", + "org.apache.cassandra.io.sstable.SSTable", + "org.apache.cassandra.io.sstable.SSTable$Builder", + "org.apache.cassandra.io.sstable.format.AbstractSSTableFormat", + "org.apache.cassandra.io.sstable.format.SSTableFormat", + "org.apache.cassandra.io.sstable.format.SSTableFormat$Components", + "org.apache.cassandra.io.sstable.format.SSTableFormat$Components$Types", + "org.apache.cassandra.io.sstable.format.SSTableFormat$KeyCacheValueSerializer", + "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableReaderFactory", + "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableWriterFactory", + "org.apache.cassandra.io.sstable.format.SSTableFormat$Type", + "org.apache.cassandra.io.sstable.format.SSTableReader", + "org.apache.cassandra.io.sstable.format.SSTableReader$Builder", + "org.apache.cassandra.io.sstable.format.SSTableReaderLoadingBuilder", + "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter", + "org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter$Builder", + "org.apache.cassandra.io.sstable.format.SSTableWriter", + "org.apache.cassandra.io.sstable.format.SSTableWriter$Builder", + "org.apache.cassandra.io.sstable.format.SortedTableWriter", + "org.apache.cassandra.io.sstable.format.SortedTableWriter$Builder", + "org.apache.cassandra.io.sstable.format.Version", + "org.apache.cassandra.io.sstable.format.big.BigFormat", + "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableReaderFactory", + "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableWriterFactory", + "org.apache.cassandra.io.sstable.format.big.BigFormat$BigVersion", + "org.apache.cassandra.io.sstable.format.big.BigFormat$Components", + "org.apache.cassandra.io.sstable.format.big.BigFormat$Components$Types", + "org.apache.cassandra.io.sstable.format.big.BigSSTableReaderLoadingBuilder", + "org.apache.cassandra.io.sstable.format.big.BigTableReader", + "org.apache.cassandra.io.sstable.format.big.BigTableReader$Builder", + "org.apache.cassandra.io.sstable.format.big.BigTableWriter", + "org.apache.cassandra.io.sstable.format.big.BigTableWriter$Builder", + "org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport", + "org.apache.cassandra.io.sstable.keycache.KeyCacheSupport", "org.apache.cassandra.io.sstable.metadata.MetadataType", "org.apache.cassandra.io.util.BufferedDataOutputStreamPlus", - "org.apache.cassandra.io.util.RebufferingInputStream", - "org.apache.cassandra.io.util.FileInputStreamPlus", - "org.apache.cassandra.io.util.FileOutputStreamPlus", - "org.apache.cassandra.io.util.File", + "org.apache.cassandra.io.util.DataInputPlus", + "org.apache.cassandra.io.util.DataInputPlus$DataInputStreamPlus", "org.apache.cassandra.io.util.DataOutputBuffer", "org.apache.cassandra.io.util.DataOutputBufferFixed", - "org.apache.cassandra.io.util.DataOutputStreamPlus", "org.apache.cassandra.io.util.DataOutputPlus", - "org.apache.cassandra.io.util.DataInputPlus", + "org.apache.cassandra.io.util.DataOutputStreamPlus", "org.apache.cassandra.io.util.DiskOptimizationStrategy", - "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy", + "org.apache.cassandra.io.util.File", + "org.apache.cassandra.io.util.FileInputStreamPlus", + "org.apache.cassandra.io.util.FileOutputStreamPlus", "org.apache.cassandra.io.util.PathUtils$IOToLongFunction", + "org.apache.cassandra.io.util.RebufferingInputStream", + "org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy", + "org.apache.cassandra.locator.IEndpointSnitch", + "org.apache.cassandra.locator.InetAddressAndPort", "org.apache.cassandra.locator.Replica", "org.apache.cassandra.locator.ReplicaCollection", - "org.apache.cassandra.locator.SimpleSeedProvider", "org.apache.cassandra.locator.SeedProvider", + "org.apache.cassandra.locator.SimpleSeedProvider", + "org.apache.cassandra.security.EncryptionContext", "org.apache.cassandra.security.ISslContextFactory", "org.apache.cassandra.security.SSLFactory", - "org.apache.cassandra.security.EncryptionContext", "org.apache.cassandra.service.CacheService$CacheType", "org.apache.cassandra.transport.ProtocolException", - "org.apache.cassandra.utils.binlog.BinLogOptions", + "org.apache.cassandra.utils.Closeable", + "org.apache.cassandra.utils.CloseableIterator", "org.apache.cassandra.utils.FBUtilities", "org.apache.cassandra.utils.FBUtilities$1", - "org.apache.cassandra.utils.CloseableIterator", "org.apache.cassandra.utils.Pair", + "org.apache.cassandra.utils.binlog.BinLogOptions", + "org.apache.cassandra.utils.concurrent.RefCounted", + "org.apache.cassandra.utils.concurrent.SelfRefCounted", + "org.apache.cassandra.utils.concurrent.Transactional", "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException", - "org.apache.cassandra.ConsoleAppender", - "org.apache.cassandra.ConsoleAppender$1", - "org.apache.cassandra.LogbackStatusListener", - "org.apache.cassandra.LogbackStatusListener$ToLoggerOutputStream", - "org.apache.cassandra.LogbackStatusListener$WrappedPrintStream", - "org.apache.cassandra.TeeingAppender", - // generated classes - "org.apache.cassandra.config.ConfigBeanInfo", - "org.apache.cassandra.config.ConfigCustomizer", - "org.apache.cassandra.config.EncryptionOptionsBeanInfo", - "org.apache.cassandra.config.EncryptionOptionsCustomizer", - "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsBeanInfo", - "org.apache.cassandra.config.EncryptionOptions$ServerEncryptionOptionsCustomizer", - "org.apache.cassandra.ConsoleAppenderBeanInfo", - "org.apache.cassandra.ConsoleAppenderCustomizer", - "org.apache.cassandra.locator.InetAddressAndPort", - "org.apache.cassandra.cql3.statements.schema.AlterKeyspaceStatement", - "org.apache.cassandra.cql3.statements.schema.CreateKeyspaceStatement" }; static final Set checkedClasses = new HashSet<>(Arrays.asList(validClasses)); @@ -370,8 +411,8 @@ public class DatabaseDescriptorRefTest StringBuilder sb = new StringBuilder(); for (Pair violation : new ArrayList<>(violations)) sb.append("\n\n") - .append("VIOLATION: ").append(violation.left).append('\n') - .append(Throwables.getStackTraceAsString(violation.right)); + .append("VIOLATION: ").append(violation.left); //.append('\n') + //.append(Throwables.getStackTraceAsString(violation.right)); String msg = sb.toString(); err.println(msg); diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java index 2b0a3c8d60..d40b001e90 100644 --- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java +++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java @@ -33,6 +33,9 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.index.Index; +import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.metrics.CacheMetrics; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.schema.CachingParams; @@ -49,6 +52,7 @@ import static org.junit.Assert.assertTrue; public class KeyCacheCqlTest extends CQLTester { + private static boolean sstableImplCachesKeys; private static final String commonColumnsDef = "part_key_a int," + @@ -98,6 +102,7 @@ public class KeyCacheCqlTest extends CQLTester { CachingParams.DEFAULT = CachingParams.CACHE_NOTHING; CQLTester.setUpClass(); + sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current()); } /** @@ -268,7 +273,7 @@ public class KeyCacheCqlTest extends CQLTester long hits = metrics.hits.getCount(); long requests = metrics.requests.getCount(); assertEquals(0, hits); - assertEquals(expectedRequests, requests); + assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests); for (int i = 0; i < 10; i++) { @@ -285,8 +290,8 @@ public class KeyCacheCqlTest extends CQLTester metrics = CacheService.instance.keyCache.getMetrics(); hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(200, hits); - assertEquals(expectedRequests, requests); + assertEquals(sstableImplCachesKeys ? 200 : 0, hits); + assertEquals(sstableImplCachesKeys ? expectedRequests : 0, requests); CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get(); @@ -362,7 +367,7 @@ public class KeyCacheCqlTest extends CQLTester long hits = metrics.hits.getCount(); long requests = metrics.requests.getCount(); assertEquals(0, hits); - assertEquals(expectedNumberOfRequests, requests); + assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests); for (int i = 0; i < 10; i++) { @@ -380,8 +385,8 @@ public class KeyCacheCqlTest extends CQLTester metrics = CacheService.instance.keyCache.getMetrics(); hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(200, hits); - assertEquals(expectedNumberOfRequests, requests); + assertEquals(sstableImplCachesKeys ? 200 : 0, hits); + assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests); dropTable("DROP TABLE %s"); @@ -428,6 +433,7 @@ public class KeyCacheCqlTest extends CQLTester long expectedNumberOfRequests = 0; + CacheMetrics metrics = CacheService.instance.keyCache.getMetrics(); for (int i = 0; i < 10; i++) { assertRows(execute("SELECT col_text FROM %s WHERE part_key_a = ? AND part_key_b = ?", i, Integer.toOctalString(i)), @@ -437,11 +443,10 @@ public class KeyCacheCqlTest extends CQLTester expectedNumberOfRequests += recentBloomFilterFalsePositives() + 1; } - CacheMetrics metrics = CacheService.instance.keyCache.getMetrics(); long hits = metrics.hits.getCount(); long requests = metrics.requests.getCount(); assertEquals(0, hits); - assertEquals(10, requests); + assertEquals(sstableImplCachesKeys ? 10 : 0, requests); for (int i = 0; i < 100; i++) { @@ -454,8 +459,8 @@ public class KeyCacheCqlTest extends CQLTester hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(10, hits); - assertEquals(expectedNumberOfRequests, requests); + assertEquals(sstableImplCachesKeys ? 10 : 0, hits); + assertEquals(sstableImplCachesKeys ? expectedNumberOfRequests : 0, requests); } @Test @@ -492,7 +497,7 @@ public class KeyCacheCqlTest extends CQLTester long hits = metrics.hits.getCount(); long requests = metrics.requests.getCount(); assertEquals(0, hits); - assertEquals(10, requests); + assertEquals(sstableImplCachesKeys ? 10 : 0, requests); // 10 queries, each 50 result rows for (int i = 0; i < 10; i++) @@ -503,8 +508,8 @@ public class KeyCacheCqlTest extends CQLTester metrics = CacheService.instance.keyCache.getMetrics(); hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(10, hits); - assertEquals(10 + 10, requests); + assertEquals(sstableImplCachesKeys ? 10 : 0, hits); + assertEquals(sstableImplCachesKeys ? 20 : 0, requests); // 100 queries - must get a hit in key-cache for (int i = 0; i < 10; i++) @@ -519,8 +524,8 @@ public class KeyCacheCqlTest extends CQLTester metrics = CacheService.instance.keyCache.getMetrics(); hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(10 + 100, hits); - assertEquals(20 + 100, requests); + assertEquals(sstableImplCachesKeys ? 10 + 100 : 0, hits); + assertEquals(sstableImplCachesKeys ? 20 + 100 : 0, requests); // 5000 queries - first 10 partitions already in key cache for (int i = 0; i < 100; i++) @@ -534,8 +539,8 @@ public class KeyCacheCqlTest extends CQLTester hits = metrics.hits.getCount(); requests = metrics.requests.getCount(); - assertEquals(110 + 4910, hits); - assertEquals(120 + 5500, requests); + assertEquals(sstableImplCachesKeys ? 110 + 4910 : 0, hits); + assertEquals(sstableImplCachesKeys ? 120 + 5500 : 0, requests); } // Inserts 100 partitions split over 10 sstables (flush after 10 partitions). @@ -614,6 +619,9 @@ public class KeyCacheCqlTest extends CQLTester private long recentBloomFilterFalsePositives() { - return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.recentBloomFilterFalsePositives.getValue(); + return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.formatSpecificGauges.get(SSTableFormat.Type.current()) + .get(BloomFilterMetrics.instance.recentBloomFilterFalsePositives.name) + .getValue() + .longValue(); } } diff --git a/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java b/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java index 01a2afd6ff..f76ec20cce 100644 --- a/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java +++ b/test/unit/org/apache/cassandra/cql3/QueryWithIndexedSSTableTest.java @@ -23,8 +23,9 @@ import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry; import org.apache.cassandra.utils.ByteBufferUtil; public class QueryWithIndexedSSTableTest extends CQLTester @@ -59,8 +60,11 @@ public class QueryWithIndexedSSTableTest extends CQLTester boolean hasIndexed = false; for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables()) { - RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ); - hasIndexed |= indexEntry != null && indexEntry.isIndexed(); + if (sstable instanceof BigTableReader) + { + RowIndexEntry indexEntry = ((BigTableReader) sstable).getRowIndexEntry(dk, SSTableReader.Operator.EQ); + hasIndexed |= indexEntry != null && indexEntry.isIndexed(); + } } assert hasIndexed; diff --git a/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java b/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java index f9ac8d16b0..a32b031075 100644 --- a/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java +++ b/test/unit/org/apache/cassandra/cql3/TombstonesWithIndexedSSTableTest.java @@ -19,17 +19,28 @@ package org.apache.cassandra.cql3; import java.util.Random; +import org.junit.Assume; +import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.Util; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry; import org.apache.cassandra.utils.ByteBufferUtil; public class TombstonesWithIndexedSSTableTest extends CQLTester { + @BeforeClass + public static void beforeClass() + { + Assume.assumeTrue("This test requires that the default SSTable format is BIG", BigFormat.isDefault()); + } + @Test public void testTombstoneBoundariesInIndexCached() throws Throwable { @@ -44,6 +55,7 @@ public class TombstonesWithIndexedSSTableTest extends CQLTester public void testTombstoneBoundariesInIndex(String cacheKeys) throws Throwable { + Assume.assumeTrue(BigFormat.isDefault()); // That test reproduces the bug from CASSANDRA-11158 where a range tombstone boundary in the column index would // cause an assertion failure. @@ -74,19 +86,16 @@ public class TombstonesWithIndexedSSTableTest extends CQLTester int indexedRow = -1; for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables()) { + BigTableReader reader = (BigTableReader) sstable; // The line below failed with key caching off (CASSANDRA-11158) - @SuppressWarnings("unchecked") - RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ); + RowIndexEntry indexEntry = reader.getRowIndexEntry(dk, SSTableReader.Operator.EQ); if (indexEntry != null && indexEntry.isIndexed()) { - try (FileDataInput reader = sstable.openIndexReader()) - { - RowIndexEntry.IndexInfoRetriever infoRetriever = indexEntry.openWithIndex(sstable.getIndexFile()); - ClusteringPrefix firstName = infoRetriever.columnsIndex(1).firstName; - if (firstName.kind().isBoundary()) - break deletionLoop; - indexedRow = Int32Type.instance.compose(firstName.bufferAt(0)); - } + RowIndexEntry.IndexInfoRetriever infoRetriever = indexEntry.openWithIndex(reader.getIndexFile()); + ClusteringPrefix firstName = infoRetriever.columnsIndex(1).firstName; + if (firstName.kind().isBoundary()) + break deletionLoop; + indexedRow = Int32Type.instance.compose(firstName.bufferAt(0)); } } assert indexedRow >= 0; diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java index 9ef2520c78..014be9015d 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/TTLTest.java @@ -21,12 +21,11 @@ package org.apache.cassandra.cql3.validation.operations; import java.io.IOException; -import org.apache.cassandra.Util; -import org.apache.cassandra.io.util.File; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Attributes; @@ -37,6 +36,8 @@ import org.apache.cassandra.db.ExpirationDateOverflowHandling; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.rows.AbstractCell; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.tools.StandaloneScrubber; @@ -379,7 +380,7 @@ public class TTLTest extends CQLTester if (runScrub) { - cfs.scrub(true, false, true, reinsertOverflowedTTL, 1); + cfs.scrub(true, IScrubber.options().checkData().reinsertOverflowedTTLRows(reinsertOverflowedTTL).build(), 1); if (reinsertOverflowedTTL) { @@ -475,4 +476,4 @@ public class TTLTest extends CQLTester else return clustering ? COMPLEX_CLUSTERING : COMPLEX_NOCLUSTERING; } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index dd024a7d90..e678413f20 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -63,8 +63,10 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.io.sstable.ScrubTest; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.ClearableHistogram; @@ -349,7 +351,7 @@ public class ColumnFamilyStoreTest liveSSTable.descriptor.id, liveSSTable.descriptor.formatType); for (Component c : liveSSTable.getComponents()) - assertTrue("Cannot find backed-up file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists()); + assertTrue("Cannot find backed-up file:" + desc.fileFor(c), desc.fileFor(c).exists()); } } @@ -578,8 +580,8 @@ public class ColumnFamilyStoreTest Set originalFiles = new HashSet<>(); Iterables.toList(cfs.concatWithIndexes()).stream() - .flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.filenameFor(Component.DATA))) - .forEach(originalFiles::add); + .flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.fileFor(Components.DATA))) + .forEach(e -> originalFiles.add(e.toString())); assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(indexTableFile))).isTrue(); assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(baseTableFile))).isTrue(); } @@ -667,9 +669,9 @@ public class ColumnFamilyStoreTest assertEquals(1, ssTables.size()); SSTableReader ssTable = ssTables.iterator().next(); - String dataFileName = ssTable.descriptor.filenameFor(Component.DATA); - String tmpDataFileName = ssTable.descriptor.tmpFilenameFor(Component.DATA); - new File(dataFileName).tryMove(new File(tmpDataFileName)); + File dataFile = ssTable.descriptor.fileFor(Components.DATA); + File tmpDataFile = ssTable.descriptor.tmpFileFor(Components.DATA); + dataFile.tryMove(tmpDataFile); ssTable.selfRef().release(); diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index ff39a1621d..4c29f6b266 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -71,6 +71,7 @@ import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; @@ -225,14 +226,14 @@ public class DirectoriesTest File snapshotDir = new File(tableDir, Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + tag); snapshotDir.tryCreateDirectories(); - Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), SSTableFormat.Type.BIG); + Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), SSTableFormat.Type.current()); createFakeSSTable(sstableDesc); SnapshotManifest manifest = null; if (createManifest) { File manifestFile = Directories.getSnapshotManifestFile(snapshotDir); - manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.filenameFor(Component.DATA)), new DurationSpec.IntSecondsBound("1m"), now(), ephemeral); + manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.fileFor(Components.DATA).absolutePath()), new DurationSpec.IntSecondsBound("1m"), now(), ephemeral); manifest.serializeToJsonFile(manifestFile); } else if (ephemeral) @@ -245,16 +246,16 @@ public class DirectoriesTest private List createFakeSSTable(File dir, String cf, int gen) { - Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), SSTableFormat.Type.BIG); + Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), SSTableFormat.Type.current()); return createFakeSSTable(desc); } private List createFakeSSTable(Descriptor desc) { List components = new ArrayList<>(3); - for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER }) + for (Component c : SSTableFormat.Type.current().info.uploadComponents()) { - File f = new File(desc.filenameFor(c)); + File f = desc.fileFor(c); f.createFileIfNotExists(); components.add(f); } @@ -287,7 +288,7 @@ public class DirectoriesTest Directories directories = new Directories(cfm, toDataDirectories(tempDataDir)); assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables()); - Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG); + Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.current()); File snapshotDir = new File(cfDir(cfm), File.pathSeparator() + Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + LEGACY_SNAPSHOT_NAME); assertEquals(snapshotDir.toCanonical(), Directories.getSnapshotDirectory(desc, LEGACY_SNAPSHOT_NAME)); @@ -363,7 +364,7 @@ public class DirectoriesTest { String tag = "test"; Directories directories = new Directories(cfm, toDataDirectories(tempDataDir)); - Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), SSTableFormat.Type.BIG); + Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), SSTableFormat.Type.current()); File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, tag); List files = new LinkedList<>(); @@ -410,8 +411,8 @@ public class DirectoriesTest { assertEquals(cfDir(INDEX_CFM), dir); } - Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG); - Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG); + Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.current()); + Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.current()); // snapshot dir should be created under its parent's File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, "test"); @@ -424,10 +425,10 @@ public class DirectoriesTest assertTrue(indexDirectories.snapshotExists("test")); // check true snapshot size - Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG); - createFile(parentSnapshot.filenameFor(Component.DATA), 30); - Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG); - createFile(indexSnapshot.filenameFor(Component.DATA), 40); + Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.current()); + createFile(parentSnapshot.fileFor(Components.DATA), 30); + Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.current()); + createFile(indexSnapshot.fileFor(Components.DATA), 40); assertEquals(30, parentDirectories.trueSnapshotsSize()); assertEquals(40, indexDirectories.trueSnapshotsSize()); @@ -444,16 +445,15 @@ public class DirectoriesTest assertEquals(parentBackupDirectory, indexBackupDirectory.parent()); } - private File createFile(String fileName, int size) + private File createFile(File file, int size) { - File newFile = new File(fileName); - try (FileOutputStreamPlus writer = new FileOutputStreamPlus(newFile);) + try (FileOutputStreamPlus writer = new FileOutputStreamPlus(file);) { writer.write(new byte[size]); writer.flush(); } catch (IOException ignore) {} - return newFile; + return file; } @Test @@ -572,7 +572,7 @@ public class DirectoriesTest final String n = Long.toString(nanoTime()); Callable directoryGetter = () -> { - Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG); + Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.current()); return Directories.getSnapshotDirectory(desc, n); }; List> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter)); @@ -737,7 +737,7 @@ public class DirectoriesTest Directories dirs = new Directories(cfm, paths); for (DataDirectory dir : paths) { - Descriptor d = Descriptor.fromFilename(new File(dir.location, getNewFilename(cfm, false)).toString()); + Descriptor d = Descriptor.fromFile(new File(dir.location, getNewFilename(cfm, false))); String p = dirs.getDataDirectoryForFile(d).location.absolutePath() + File.pathSeparator(); assertTrue(p.startsWith(dir.location.absolutePath() + File.pathSeparator())); } @@ -766,9 +766,9 @@ public class DirectoriesTest for (TableMetadata tm : CFM) { Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2)); - Descriptor desc = Descriptor.fromFilename(new File(ddir1.resolve(getNewFilename(tm, false)))); + Descriptor desc = Descriptor.fromFile(new File(ddir1.resolve(getNewFilename(tm, false)))); assertEquals(new File(ddir1), dirs.getDataDirectoryForFile(desc).location); - desc = Descriptor.fromFilename(new File(ddir2.resolve(getNewFilename(tm, false)))); + desc = Descriptor.fromFile(new File(ddir2.resolve(getNewFilename(tm, false)))); assertEquals(new File(ddir2), dirs.getDataDirectoryForFile(desc).location); } } @@ -816,9 +816,9 @@ public class DirectoriesTest for (TableMetadata tm : CFM) { Directories dirs = new Directories(tm, Sets.newHashSet(dd1, dd2)); - Descriptor desc = Descriptor.fromFilename(new File(ddir1.resolve(getNewFilename(tm, oldStyle)))); + Descriptor desc = Descriptor.fromFile(new File(ddir1.resolve(getNewFilename(tm, oldStyle)))); assertEquals(new File(ddir1), dirs.getDataDirectoryForFile(desc).location); - desc = Descriptor.fromFilename(new File(ddir2.resolve(getNewFilename(tm, oldStyle)))); + desc = Descriptor.fromFile(new File(ddir2.resolve(getNewFilename(tm, oldStyle)))); assertEquals(new File(ddir2), dirs.getDataDirectoryForFile(desc).location); } } diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java b/test/unit/org/apache/cassandra/db/ImportTest.java index ff843fac41..9dc06f01d7 100644 --- a/test/unit/org/apache/cassandra/db/ImportTest.java +++ b/test/unit/org/apache/cassandra/db/ImportTest.java @@ -40,7 +40,7 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.BootStrapper; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.InetAddressAndPort; @@ -236,7 +236,7 @@ public class ImportTest extends CQLTester sstable.selfRef().release(); for (File f : sstable.descriptor.directory.tryList()) { - if (f.toString().contains(sstable.descriptor.baseFilename())) + if (f.toString().contains(sstable.descriptor.baseFile().toString())) { System.out.println("move " + f.toPath() + " to " + backupdir); File moveFileTo = new File(backupdir, f.name()); @@ -314,8 +314,8 @@ public class ImportTest extends CQLTester getCurrentColumnFamilyStore().clearUnsafe(); - String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS); - try (FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel()) + File fileToCorrupt = sstableToCorrupt.descriptor.fileFor(Components.STATS); + try (FileChannel fileChannel = fileToCorrupt.newReadWriteChannel()) { fileChannel.position(0); fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); @@ -574,8 +574,8 @@ public class ImportTest extends CQLTester sstables.forEach(s -> s.selfRef().release()); // corrupt the sstable which is still in the data directory SSTableReader sstableToCorrupt = sstables.iterator().next(); - String filenameToCorrupt = sstableToCorrupt.descriptor.filenameFor(Component.STATS); - try (FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel()) + File fileToCorrupt = sstableToCorrupt.descriptor.fileFor(Components.STATS); + try (FileChannel fileChannel = fileToCorrupt.newReadWriteChannel()) { fileChannel.position(0); fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); @@ -615,7 +615,7 @@ public class ImportTest extends CQLTester assertEquals(20, rowCount); assertEquals(expectedFiles, getCurrentColumnFamilyStore().getLiveSSTables()); for (SSTableReader sstable : expectedFiles) - assertTrue(new File(sstable.descriptor.filenameFor(Component.DATA)).exists()); + assertTrue(sstable.descriptor.fileFor(Components.DATA).exists()); getCurrentColumnFamilyStore().truncateBlocking(); LifecycleTransaction.waitForDeletions(); for (File f : sstableToCorrupt.descriptor.directory.tryList()) // clean up the corrupt files which truncate does not handle diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java index 2445fe3aa0..a2fa9e9528 100644 --- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java @@ -19,28 +19,36 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Collection; -import org.apache.cassandra.schema.Schema; -import org.assertj.core.api.Assertions; import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; import org.apache.cassandra.metrics.ClearableHistogram; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.Assertions; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class KeyspaceTest extends CQLTester { @@ -407,8 +415,11 @@ public class KeyspaceTest extends CQLTester // verify that we do indeed have multiple index entries SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - RowIndexEntry indexEntry = sstable.getPosition(Util.dk("0"), SSTableReader.Operator.EQ); - assert indexEntry.columnsIndexCount() > 2; + if (sstable instanceof BigTableReader) + { + AbstractRowIndexEntry indexEntry = ((BigTableReader) sstable).getRowIndexEntry(Util.dk("0"), SSTableReader.Operator.EQ); + assert indexEntry.columnsIndexCount() > 2; + } validateSliceLarge(cfs); } diff --git a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java index bb272afe9f..dc2eaae930 100644 --- a/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java +++ b/test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java @@ -31,6 +31,7 @@ import org.apache.cassandra.db.rows.AbstractRow; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.ByteBufferUtil; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -261,6 +262,10 @@ public class RepairedDataTombstonesTest extends CQLTester } } } + else + { + while (rowIter.hasNext()) rowIter.next(); + } } } } @@ -312,4 +317,4 @@ public class RepairedDataTombstonesTest extends CQLTester sstable.reloadSSTableMetadata(); cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable)); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java index 8f7e24b03d..1f0ef8bc5c 100644 --- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java @@ -18,7 +18,16 @@ package org.apache.cassandra.db; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.concurrent.Callable; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; + import com.google.common.io.Files; +import org.junit.Assert; +import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; @@ -38,23 +47,12 @@ import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.format.big.BigFormat; -import org.apache.cassandra.io.sstable.format.big.BigTableWriter; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; -import org.junit.Assert; -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.concurrent.Callable; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.cassandra.io.util.File; public class SerializationHeaderTest { @@ -68,9 +66,8 @@ public class SerializationHeaderTest @Test public void testWrittenAsDifferentKind() throws Exception { + SSTableFormat format = SSTableFormat.Type.current().info; final String tableName = "testWrittenAsDifferentKind"; -// final String schemaCqlWithStatic = String.format("CREATE TABLE %s (k int, c int, v int static, PRIMARY KEY(k, c))", tableName); -// final String schemaCqlWithRegular = String.format("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))", tableName); ColumnIdentifier v = ColumnIdentifier.getInterned("v", false); TableMetadata schemaWithStatic = TableMetadata.builder(KEYSPACE, tableName) .addPartitionKeyColumn("k", Int32Type.instance) @@ -87,16 +84,26 @@ public class SerializationHeaderTest schemaWithStatic = schemaWithStatic.unbuild().recordColumnDrop(columnRegular, 0L).build(); schemaWithRegular = schemaWithRegular.unbuild().recordColumnDrop(columnStatic, 0L).build(); + SSTableReader readerWithStatic = null; + SSTableReader readerWithRegular = null; Supplier id = Util.newSeqGen(); File dir = new File(Files.createTempDir()); try { BiFunction>, Callable> writer = (schema, clusteringFunction) -> () -> { - Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.keyspace, schema.name, id.get(), SSTableFormat.Type.BIG); + Descriptor descriptor = new Descriptor(format.getLatestVersion(), dir, schema.keyspace, schema.name, id.get(), format.getType()); SerializationHeader header = SerializationHeader.makeWithoutStats(schema); try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableWriter sstableWriter = BigTableWriter.create(TableMetadataRef.forOfflineTools(schema), descriptor, 1, 0L, null, false, 0, header, Collections.emptyList(), txn)) + SSTableWriter sstableWriter = descriptor.getFormat().getWriterFactory() + .builder(descriptor) + .setTableMetadataRef(TableMetadataRef.forOfflineTools(schema)) + .setKeyCount(1) + .setSerializationHeader(header) + .setFlushObservers(Collections.emptyList()) + .setMetadataCollector(new MetadataCollector(schema.comparator)) + .addDefaultComponents() + .build(txn, null)) { ColumnMetadata cd = schema.getColumn(v); for (int i = 0 ; i < 5 ; ++i) { @@ -114,8 +121,8 @@ public class SerializationHeaderTest Descriptor sstableWithRegular = writer.apply(schemaWithRegular, BufferClustering::new).call(); Descriptor sstableWithStatic = writer.apply(schemaWithStatic, value -> Clustering.STATIC_CLUSTERING).call(); - SSTableReader readerWithStatic = SSTableReader.openNoValidation(sstableWithStatic, TableMetadataRef.forOfflineTools(schemaWithRegular)); - SSTableReader readerWithRegular = SSTableReader.openNoValidation(sstableWithRegular, TableMetadataRef.forOfflineTools(schemaWithStatic)); + readerWithStatic = SSTableReader.openNoValidation(null, sstableWithStatic, TableMetadataRef.forOfflineTools(schemaWithRegular)); + readerWithRegular = SSTableReader.openNoValidation(null, sstableWithRegular, TableMetadataRef.forOfflineTools(schemaWithStatic)); try (ISSTableScanner partitions = readerWithStatic.getScanner()) { for (int i = 0 ; i < 5 ; ++i) @@ -141,6 +148,10 @@ public class SerializationHeaderTest } finally { + if (readerWithStatic != null) + readerWithStatic.selfRef().close(); + if (readerWithRegular != null) + readerWithRegular.selfRef().close(); FileUtils.deleteRecursive(dir); } } diff --git a/test/unit/org/apache/cassandra/db/SnapshotTest.java b/test/unit/org/apache/cassandra/db/SnapshotTest.java index aa726ec3e0..fdfdd6b3aa 100644 --- a/test/unit/org/apache/cassandra/db/SnapshotTest.java +++ b/test/unit/org/apache/cassandra/db/SnapshotTest.java @@ -24,7 +24,7 @@ import java.nio.file.StandardOpenOption; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; @@ -38,7 +38,7 @@ public class SnapshotTest extends CQLTester getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); for (SSTableReader sstable : getCurrentColumnFamilyStore().getLiveSSTables()) { - File toc = new File(sstable.descriptor.filenameFor(Component.TOC)); + File toc = sstable.descriptor.fileFor(Components.TOC); Files.write(toc.toPath(), new byte[0], StandardOpenOption.TRUNCATE_EXISTING); } getCurrentColumnFamilyStore().snapshot("hello"); diff --git a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java index 9f9b7bb310..80b71ef8a7 100644 --- a/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java +++ b/test/unit/org/apache/cassandra/db/columniterator/SSTableReverseIteratorTest.java @@ -34,9 +34,10 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry; import org.apache.cassandra.schema.KeyspaceParams; public class SSTableReverseIteratorTest @@ -85,9 +86,12 @@ public class SSTableReverseIteratorTest Util.flush(tbl); SSTableReader sstable = Iterables.getOnlyElement(tbl.getLiveSSTables()); DecoratedKey dk = tbl.getPartitioner().decorateKey(Int32Type.instance.decompose(key)); - RowIndexEntry indexEntry = sstable.getPosition(dk, SSTableReader.Operator.EQ); - Assert.assertTrue(indexEntry.isIndexed()); - Assert.assertTrue(indexEntry.columnsIndexCount() > 2); + if (sstable instanceof BigTableReader) + { + RowIndexEntry indexEntry = ((BigTableReader) sstable).getRowIndexEntry(dk, SSTableReader.Operator.EQ); + Assert.assertTrue(indexEntry.isIndexed()); + Assert.assertTrue(indexEntry.columnsIndexCount() > 2); + } // drop v1 so the first 2 index blocks only contain empty unfiltereds QueryProcessor.executeInternal(String.format("ALTER TABLE %s.%s DROP v1", KEYSPACE, table)); diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java index 56d6c4082a..78497965d7 100644 --- a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java @@ -46,8 +46,10 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexBuilder; -import org.apache.cassandra.io.sstable.IndexSummaryRedistribution; +import org.apache.cassandra.io.sstable.IScrubber; +import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.FBUtilities; @@ -143,7 +145,15 @@ public class ActiveCompactionsTest extends CQLTester Map transactions = ImmutableMap.builder().put(getCurrentColumnFamilyStore().metadata().id, txn).build(); IndexSummaryRedistribution isr = new IndexSummaryRedistribution(transactions, 0, 1000); MockActiveCompactions mockActiveCompactions = new MockActiveCompactions(); - CompactionManager.instance.runIndexSummaryRedistribution(isr, mockActiveCompactions); + mockActiveCompactions.beginCompaction(isr); + try + { + isr.redistributeSummaries(); + } + finally + { + mockActiveCompactions.finishCompaction(isr); + } assertTrue(mockActiveCompactions.finished); assertNotNull(mockActiveCompactions.holder); // index redistribution operates over all keyspaces/tables, we always cancel them @@ -191,7 +201,7 @@ public class ActiveCompactionsTest extends CQLTester try (LifecycleTransaction txn = getCurrentColumnFamilyStore().getTracker().tryModify(sstable, OperationType.SCRUB)) { MockActiveCompactions mockActiveCompactions = new MockActiveCompactions(); - CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, true, false, false, mockActiveCompactions); + CompactionManager.instance.scrubOne(getCurrentColumnFamilyStore(), txn, IScrubber.options().skipCorrupted().build(), mockActiveCompactions); assertTrue(mockActiveCompactions.finished); assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable)); @@ -214,7 +224,7 @@ public class ActiveCompactionsTest extends CQLTester SSTableReader sstable = Iterables.getFirst(getCurrentColumnFamilyStore().getLiveSSTables(), null); MockActiveCompactions mockActiveCompactions = new MockActiveCompactions(); - CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, new Verifier.Options.Builder().build(), mockActiveCompactions); + CompactionManager.instance.verifyOne(getCurrentColumnFamilyStore(), sstable, IVerifier.options().build(), mockActiveCompactions); assertTrue(mockActiveCompactions.finished); assertEquals(mockActiveCompactions.holder.getCompactionInfo().getSSTables(), Sets.newHashSet(sstable)); assertFalse(mockActiveCompactions.holder.getCompactionInfo().shouldStop((s) -> false)); @@ -245,4 +255,4 @@ public class ActiveCompactionsTest extends CQLTester finished = true; } } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java index 36b587c024..8745b908a5 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ColumnFamilyStore; @@ -61,13 +62,12 @@ import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableIdFactory; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; @@ -95,7 +95,7 @@ public class CompactionsTest compactionOptions.put("tombstone_compaction_interval", "1"); // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); SchemaLoader.prepareServer(); @@ -274,7 +274,7 @@ public class CompactionsTest SSTableReader sstable = sstables.iterator().next(); SSTableId prevGeneration = sstable.descriptor.id; - String file = new File(sstable.descriptor.filenameFor(Component.DATA)).absolutePath(); + String file = sstable.descriptor.fileFor(Components.DATA).absolutePath(); // submit user defined compaction on flushed sstable CompactionManager.instance.forceUserDefinedCompaction(file); // wait until user defined compaction finishes diff --git a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java index 4df210a839..156a2a328c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java @@ -18,17 +18,21 @@ package org.apache.cassandra.db.compaction; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; -import org.junit.BeforeClass; -import org.junit.Test; - import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -39,12 +43,16 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.Pair; +import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.filterOldSSTables; import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.getBuckets; import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.newestBucket; -import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.filterOldSSTables; import static org.apache.cassandra.db.compaction.DateTieredCompactionStrategy.validateOptions; - -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class DateTieredCompactionStrategyTest extends SchemaLoader { @@ -54,7 +62,7 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader public static void defineSchema() throws ConfigurationException { // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); SchemaLoader.prepareServer(); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 07a45e7803..d6dfcd6c64 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -56,23 +57,23 @@ import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.notifications.SSTableAddedNotification; import org.apache.cassandra.notifications.SSTableRepairStatusChanged; -import org.apache.cassandra.repair.ValidationManager; -import org.apache.cassandra.repair.state.ValidationState; -import org.apache.cassandra.schema.MockSchema; -import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.ValidationManager; import org.apache.cassandra.repair.Validator; +import org.apache.cassandra.repair.state.ValidationState; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; import org.awaitility.Awaitility; import static java.util.Collections.singleton; -import static org.assertj.core.api.Assertions.assertThat; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -92,7 +93,7 @@ public class LeveledCompactionStrategyTest public static void defineSchema() throws ConfigurationException { // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); SchemaLoader.prepareServer(); diff --git a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java index bb1d1f0c5d..9e03ec3a9a 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SingleSSTableLCSTaskTest.java @@ -29,9 +29,8 @@ import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; @@ -129,8 +128,7 @@ public class SingleSSTableLCSTaskTest extends CQLTester Util.flush(cfs); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS); - try(FileChannel fc = new File(filenameToCorrupt).newReadWriteChannel()) + try(FileChannel fc = sstable.descriptor.fileFor(Components.STATS).newReadWriteChannel()) { fc.position(0); fc.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); diff --git a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java index fc98a9cc1f..0e91b701d6 100644 --- a/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategyTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; @@ -55,7 +56,7 @@ public class SizeTieredCompactionStrategyTest public static void defineSchema() throws ConfigurationException { // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); SchemaLoader.prepareServer(); diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java index 0e3fb7659f..01bb60965c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java @@ -29,18 +29,23 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.schema.SchemaTestUtil; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaTestUtil; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tools.SSTableExpiredBlockers; import org.apache.cassandra.utils.ByteBufferUtil; @@ -56,7 +61,7 @@ public class TTLExpiryTest public static void defineSchema() throws ConfigurationException { // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); SchemaLoader.prepareServer(); diff --git a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java index de60286f39..add3ec0e15 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TimeWindowCompactionStrategyTest.java @@ -28,18 +28,12 @@ import java.util.concurrent.TimeUnit; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; - import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -54,6 +48,12 @@ import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.ge import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.newestBucket; import static org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy.validateOptions; import static org.apache.cassandra.utils.FBUtilities.nowInSeconds; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class TimeWindowCompactionStrategyTest extends SchemaLoader { @@ -65,7 +65,7 @@ public class TimeWindowCompactionStrategyTest extends SchemaLoader public static void defineSchema() throws ConfigurationException { // Disable tombstone histogram rounding for tests - System.setProperty("cassandra.streaminghistogram.roundseconds", "1"); + CassandraRelevantProperties.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS.setInt(1); System.setProperty(TimeWindowCompactionStrategyOptions.UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_PROPERTY, "true"); SchemaLoader.prepareServer(); diff --git a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java index a641be1b53..c4f72ded68 100644 --- a/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/writers/CompactionAwareWriterTest.java @@ -18,15 +18,26 @@ package org.apache.cassandra.db.compaction.writers; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; import com.google.common.primitives.Longs; -import org.junit.*; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.CompactionController; import org.apache.cassandra.db.compaction.CompactionIterator; @@ -67,6 +78,13 @@ public class CompactionAwareWriterTest extends CQLTester return Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); } + @After + public void afterTest() { + Keyspace ks = Keyspace.open(KEYSPACE); + ColumnFamilyStore cfs = ks.getColumnFamilyStore(TABLE); + cfs.truncateBlocking(); + } + @Test public void testDefaultCompactionWriter() throws Throwable { @@ -260,4 +278,4 @@ public class CompactionAwareWriterTest extends CQLTester assertRows(execute(String.format("SELECT k, t FROM %s.%s WHERE k = :i", KEYSPACE, TABLE), i), expected); } } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index 334edcbbbf..eed4da8cd1 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -51,6 +51,9 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; @@ -58,7 +61,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.MockSchema; -import org.apache.cassandra.utils.AlwaysPresentFilter; +import org.apache.cassandra.utils.FilterFactory; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest; import org.apache.cassandra.utils.concurrent.Transactional; @@ -840,22 +843,17 @@ public class LogTransactionTest extends AbstractTransactionalTest assertNotNull(tmpFiles); assertEquals(numNewFiles - 1, tmpFiles.size()); - File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA)); - File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX)); + List sstableFiles = sstable2.descriptor.getFormat().primaryComponents().stream().map(sstable2.descriptor::fileFor).collect(Collectors.toList()); - assertTrue(tmpFiles.contains(ssTable2DataFile)); - assertTrue(tmpFiles.contains(ssTable2IndexFile)); + for (File f : tmpFiles) assertTrue(tmpFiles.contains(f)); List files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles(); List filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles(); assertNotNull(files); assertNotNull(filesNoTmp); - assertTrue(files.contains(ssTable2DataFile)); - assertTrue(files.contains(ssTable2IndexFile)); - - assertFalse(filesNoTmp.contains(ssTable2DataFile)); - assertFalse(filesNoTmp.contains(ssTable2IndexFile)); + for (File f : tmpFiles) assertTrue(files.contains(f)); + for (File f : tmpFiles) assertFalse(filesNoTmp.contains(f)); log.finish(); @@ -866,8 +864,8 @@ public class LogTransactionTest extends AbstractTransactionalTest filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles(); assertNotNull(filesNoTmp); - assertTrue(filesNoTmp.contains(ssTable2DataFile)); - assertTrue(filesNoTmp.contains(ssTable2IndexFile)); + + for (File f : tmpFiles) assertTrue(filesNoTmp.contains(f)); sstable1.selfRef().release(); sstable2.selfRef().release(); @@ -1262,38 +1260,45 @@ public class LogTransactionTest extends AbstractTransactionalTest private static SSTableReader sstable(File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException { - Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), SSTableFormat.Type.BIG); - Set components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC); - for (Component component : components) + if (BigFormat.isDefault()) { - File file = new File(descriptor.filenameFor(component)); - if (!file.exists()) - assertTrue(file.createFileIfNotExists()); + Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), SSTableFormat.Type.current()); + Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); + for (Component component : components) + { + File file = descriptor.fileFor(component); + if (!file.exists()) + assertTrue(file.createFileIfNotExists()); - Util.setFileLength(file, size); + Util.setFileLength(file, size); + } + + FileHandle dFile = new FileHandle.Builder(descriptor.fileFor(Components.DATA)).complete(); + FileHandle iFile = new FileHandle.Builder(descriptor.fileFor(Components.PRIMARY_INDEX)).complete(); + + DecoratedKey key = MockSchema.readerBounds(generation);SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); + StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) + .get(MetadataType.STATS); + SSTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) + .setTableMetadataRef(cfs.metadata) + .setDataFile(dFile) + .setIndexFile(iFile) + .setIndexSummary(MockSchema.indexSummary.sharedCopy()) + .setFilter(FilterFactory.AlwaysPresent) + .setMaxDataAge(1L) + .setStatsMetadata(metadata) + .setOpenReason(SSTableReader.OpenReason.NORMAL) + .setSerializationHeader(header) + .setFirst(key) + .setLast(key) + .build(cfs, false, false); + return reader; + } + else + { + throw new UnsupportedOperationException("Please implement this method for sstable format: " + SSTableFormat.Type.current().info.getClass().getName()); } - - FileHandle dFile = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).complete(); - FileHandle iFile = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).complete(); - - DecoratedKey key = MockSchema.readerBounds(generation); - SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); - StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) - .get(MetadataType.STATS); - SSTableReader reader = SSTableReader.internalOpen(descriptor, - components, - cfs.metadata, - iFile, - dFile, - MockSchema.indexSummary.sharedCopy(), - new AlwaysPresentFilter(), - 1L, - metadata, - SSTableReader.OpenReason.NORMAL, - header); - reader.first = reader.last = key; - return reader; } private static void assertFiles(String dirPath, Set expectedFiles) throws IOException diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java index 71803887a5..85c8ae092f 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java @@ -24,14 +24,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.cassandra.io.util.File; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assert; import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.schema.Schema; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; @@ -43,8 +40,11 @@ import org.apache.cassandra.io.sstable.CQLSSTableWriter; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableRewriter; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.utils.Clock.Global.nanoTime; @@ -160,16 +160,13 @@ public class RealTransactionsTest extends SchemaLoader File directory = txn.originals().iterator().next().descriptor.directory; Descriptor desc = cfs.newSSTableDescriptor(directory); TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc); - rewriter.switchWriter(SSTableWriter.create(metadata, - desc, - 0, - 0, - null, - false, - 0, - SerializationHeader.make(cfs.metadata(), txn.originals()), - cfs.indexManager.listIndexes(), - txn)); + rewriter.switchWriter(desc.getFormat().getWriterFactory().builder(desc) + .setTableMetadataRef(metadata) + .setSerializationHeader(SerializationHeader.make(cfs.metadata(), txn.originals())) + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType()) + .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) + .addDefaultComponents() + .build(txn, cfs)); while (ci.hasNext()) { ci.setTargetDirectory(rewriter.currentWriter().getFilename()); diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java index e39f71f025..66c4baaf2b 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java @@ -41,7 +41,18 @@ import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.notifications.*; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; +import org.apache.cassandra.notifications.INotification; +import org.apache.cassandra.notifications.INotificationConsumer; +import org.apache.cassandra.notifications.InitialSSTableAddedNotification; +import org.apache.cassandra.notifications.MemtableDiscardedNotification; +import org.apache.cassandra.notifications.MemtableRenewedNotification; +import org.apache.cassandra.notifications.MemtableSwitchedNotification; +import org.apache.cassandra.notifications.SSTableAddedNotification; +import org.apache.cassandra.notifications.SSTableDeletingNotification; +import org.apache.cassandra.notifications.SSTableListChangedNotification; +import org.apache.cassandra.notifications.SSTableMetadataChanged; +import org.apache.cassandra.notifications.SSTableRepairStatusChanged; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -162,7 +173,8 @@ public class TrackerTest Assert.assertTrue(listener.received.get(0) instanceof InitialSSTableAddedNotification); for (SSTableReader reader : readers) - Assert.assertTrue(reader.isKeyCacheEnabled()); + if (reader instanceof KeyCacheSupport) + Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); } @@ -184,7 +196,10 @@ public class TrackerTest Assert.assertEquals(3, tracker.view.get().sstables.size()); for (SSTableReader reader : readers) - Assert.assertTrue(reader.isKeyCacheEnabled()); + { + if (reader instanceof KeyCacheSupport) + Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); + } Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); Assert.assertEquals(1, listener.senders.size()); @@ -316,7 +331,7 @@ public class TrackerTest Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(1)).added); Assert.assertEquals(Optional.of(prev2), ((SSTableAddedNotification) listener.received.get(1)).memtable()); listener.received.clear(); - Assert.assertTrue(reader.isKeyCacheEnabled()); + Assert.assertTrue(((KeyCacheSupport) reader).getKeyCache().isEnabled()); Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount()); // test invalidated CFS diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java index efe93f6af9..86df1cbe3e 100644 --- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java +++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java @@ -45,8 +45,8 @@ import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.IntegerType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java index 93b6e71e27..6adfdedd21 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraOutgoingFileTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db.streaming; +import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -132,7 +133,7 @@ public class CassandraOutgoingFileTest int count = 0; DecoratedKey key; - try (KeyIterator iter = new KeyIterator(sstable.descriptor, sstable.metadata())) + try (KeyIterator iter = sstable.keyIterator()) { do { @@ -140,6 +141,10 @@ public class CassandraOutgoingFileTest count++; } while (iter.hasNext() && count < i); } + catch (IOException e) + { + throw new RuntimeException(e); + } return key; } diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java index 0e5187c3d0..1506c952d1 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java @@ -40,8 +40,8 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.schema.KeyspaceParams; @@ -144,9 +144,10 @@ public class CassandraStreamHeaderTest SerializationHeader.Component serializationHeader = SerializationHeader.makeWithoutStats(metadata).toComponent(); ComponentManifest componentManifest = entireSSTable ? ComponentManifest.create(sstable.descriptor) : null; DecoratedKey firstKey = entireSSTable ? sstable.first : null; + return CassandraStreamHeader.builder() - .withSSTableFormat(SSTableFormat.Type.BIG) - .withSSTableVersion(BigFormat.latestVersion) + .withSSTableFormat(sstable.descriptor.getFormat().getType()) + .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(10) .withCompressionInfo(compressionInfo) @@ -166,8 +167,8 @@ public class CassandraStreamHeaderTest TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build(); CassandraStreamHeader header = CassandraStreamHeader.builder() - .withSSTableFormat(SSTableFormat.Type.BIG) - .withSSTableVersion(BigFormat.latestVersion) + .withSSTableFormat(SSTableFormat.Type.current()) + .withSSTableVersion(SSTableFormat.Type.current().info.getLatestVersion()) .withSSTableLevel(0) .withEstimatedKeys(0) .withSections(Collections.emptyList()) @@ -184,12 +185,12 @@ public class CassandraStreamHeaderTest String ddl = "CREATE TABLE tbl (k INT PRIMARY KEY, v INT)"; TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build(); - ComponentManifest manifest = new ComponentManifest(new LinkedHashMap() {{ put(Component.DATA, 100L); }}); + ComponentManifest manifest = new ComponentManifest(new LinkedHashMap() {{ put(Components.DATA, 100L); }}); CassandraStreamHeader header = CassandraStreamHeader.builder() - .withSSTableFormat(SSTableFormat.Type.BIG) - .withSSTableVersion(BigFormat.latestVersion) + .withSSTableFormat(SSTableFormat.Type.current()) + .withSSTableVersion(SSTableFormat.Type.current().info.getLatestVersion()) .withSSTableLevel(0) .withEstimatedKeys(0) .withSections(Collections.emptyList()) diff --git a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java index 43abcc733a..9e7c9d8fb6 100644 --- a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java @@ -18,19 +18,30 @@ package org.apache.cassandra.db.streaming; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.serializers.SerializationUtils; +import java.util.LinkedHashMap; + +import org.junit.BeforeClass; import org.junit.Test; -import java.util.LinkedHashMap; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.serializers.SerializationUtils; public class ComponentManifestTest { + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.clientInitialization(); + } + @Test public void testSerialization() { - ComponentManifest expected = new ComponentManifest(new LinkedHashMap() {{ put(Component.DATA, 100L); }}); - SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializer); + ComponentManifest expected = new ComponentManifest(new LinkedHashMap() {{ put(Components.DATA, 100L); }}); + SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializers.get(BigFormat.getInstance().getType())); } // Propose removing this test which now fails on VIntOutOfRange diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java index 8b63ba53b4..5733b2fa5a 100644 --- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java @@ -56,10 +56,10 @@ import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.IndexSummaryManager; -import org.apache.cassandra.io.sstable.IndexSummaryRedistribution; import org.apache.cassandra.io.sstable.SSTableUtils; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.RangesAtEndpoint; @@ -71,7 +71,6 @@ import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.OutgoingStream; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionInfo; @@ -80,6 +79,7 @@ import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; diff --git a/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java b/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java index e5804612a6..c139760fce 100644 --- a/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java +++ b/test/unit/org/apache/cassandra/dht/tokenallocator/OfflineTokenAllocatorTestUtils.java @@ -91,13 +91,13 @@ public class OfflineTokenAllocatorTestUtils } @Override - public void warn(String msg, Throwable th) + public void warn(Throwable th, String msg) { // We can only guarantee that ownership stdev won't increase above the warn threshold for racks==1 or racks==rf if (racks == 1 || racks == rf) fail(msg); else - super.warn(msg, th); + super.warn(th, msg); } } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index 441d7b00ff..eefb1e3670 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -24,10 +24,21 @@ import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadLocalRandom; @@ -36,35 +47,58 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.index.Index; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.ColumnMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.statements.schema.IndexTarget; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Columns; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.PartitionRangeReadCommand; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.lifecycle.SSTableSet; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.ValueAccessor; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.Index; import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer; import org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer; import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer; @@ -80,11 +114,16 @@ import org.apache.cassandra.index.sasi.plan.QueryPlan; import org.apache.cassandra.index.sasi.utils.RangeIterator; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.IndexSummaryManager; import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.service.snapshot.SnapshotManifest; @@ -94,12 +133,6 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.assertj.core.api.Assertions; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.Uninterruptibles; - -import org.junit.*; - import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.cassandra.db.ColumnFamilyStoreTest.getSnapshotManifestAndSchemaFileSizes; @@ -201,8 +234,8 @@ public class SASIIndexTest for (Component c : components) { - long componentSize = Files.size(Paths.get(snapshotSSTable.filenameFor(c))); - if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX) + long componentSize = snapshotSSTable.fileFor(c).length(); + if (Component.Type.fromRepresentation(c.name, sstable.descriptor.formatType) == Components.Types.SECONDARY_INDEX) indexSize += componentSize; else tableSize += componentSize; diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java index ad0caff44f..2886db4fb7 100644 --- a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java @@ -18,25 +18,38 @@ package org.apache.cassandra.index.sasi.disk; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ThreadLocalRandom; +import com.google.common.util.concurrent.Futures; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.rows.BTreeRow; import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.index.sasi.SASIIndex; import org.apache.cassandra.index.sasi.utils.RangeIterator; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.File; @@ -49,12 +62,6 @@ import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.Tables; import org.apache.cassandra.utils.ByteBufferUtil; -import com.google.common.util.concurrent.Futures; - -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - public class PerSSTableIndexWriterTest extends SchemaLoader { private static final String KS_NAME = "sasi"; @@ -112,7 +119,9 @@ public class PerSSTableIndexWriterTest extends SchemaLoader Map.Entry key = keyIterator.next(); - indexWriter.startPartition(key.getKey(), position++); + + indexWriter.startPartition(key.getKey(), position, position); + position++; indexWriter.nextUnfilteredCluster(key.getValue()); } @@ -126,7 +135,7 @@ public class PerSSTableIndexWriterTest extends SchemaLoader for (String segment : segments) Assert.assertTrue(new File(segment).exists()); - String indexFile = indexWriter.indexes.get(column).filename(true); + File indexFile = indexWriter.indexes.get(column).file(true); // final flush indexWriter.complete(); @@ -134,7 +143,7 @@ public class PerSSTableIndexWriterTest extends SchemaLoader for (String segment : segments) Assert.assertFalse(new File(segment).exists()); - OnDiskIndex index = new OnDiskIndex(new File(indexFile), Int32Type.instance, keyPosition -> { + OnDiskIndex index = new OnDiskIndex(indexFile, Int32Type.instance, keyPosition -> { ByteBuffer key = ByteBufferUtil.bytes(String.format(keyFormat, keyPosition)); return cfs.metadata().partitioner.decorateKey(key); }); @@ -233,7 +242,7 @@ public class PerSSTableIndexWriterTest extends SchemaLoader Assert.assertFalse(new File(segment).exists()); // and combined index doesn't exist either - Assert.assertFalse(new File(index.outputFile).exists()); + Assert.assertFalse(index.outputFile.exists()); } private static void populateSegment(TableMetadata metadata, PerSSTableIndexWriter.Index index, Map> data) diff --git a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java index dcd79b91a4..332906be3c 100644 --- a/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/utils/MappedBufferTest.java @@ -17,20 +17,21 @@ */ package org.apache.cassandra.index.sasi.utils; -import java.io.*; +import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ThreadLocalRandom; +import org.junit.Assert; +import org.junit.Test; + import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; -import org.junit.Assert; -import org.junit.Test; - public class MappedBufferTest { @Test @@ -461,7 +462,7 @@ public class MappedBufferTest file.getFD().sync(); - try (MappedBuffer buffer = new MappedBuffer(new ChannelProxy(tmp.absolutePath(), file.getChannel()))) + try (MappedBuffer buffer = new MappedBuffer(new ChannelProxy(tmp, file.getChannel()))) { Assert.assertEquals(numValues * 8, buffer.limit()); Assert.assertEquals(numValues * 8, buffer.capacity()); @@ -530,7 +531,7 @@ public class MappedBufferTest try { - return new MappedBuffer(new ChannelProxy(testFile.absolutePath(), file.getChannel()), numPageBits); + return new MappedBuffer(new ChannelProxy(testFile, file.getChannel()), numPageBits); } finally { diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java index f07936593a..af56ff703b 100644 --- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java +++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; import org.apache.cassandra.Util; @@ -36,9 +37,11 @@ import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.lifecycle.SSTableSet; -import org.apache.cassandra.io.sstable.IndexSummaryManager; -import org.apache.cassandra.io.sstable.IndexSummaryRedistribution; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.FBUtilities; @@ -72,6 +75,7 @@ public class DiskSpaceMetricsTest extends CQLTester @Test public void summaryRedistribution() throws Throwable { + Assume.assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); createTable("CREATE TABLE %s (pk bigint, PRIMARY KEY (pk)) WITH min_index_interval=1"); ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java index c398ac45bc..2bf127f079 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java @@ -20,11 +20,10 @@ package org.apache.cassandra.io.compress; import java.io.EOFException; import java.io.IOException; -import java.io.RandomAccessFile; //checkstyle: permit this import +import java.io.RandomAccessFile; import java.util.Arrays; import java.util.Random; -import org.assertj.core.api.Assertions; import org.junit.BeforeClass; import org.junit.Test; @@ -34,9 +33,16 @@ import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.CorruptSSTableException; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.utils.SyncUtil; +import org.assertj.core.api.Assertions; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; @@ -98,7 +104,7 @@ public class CompressedRandomAccessReaderTest File f = FileUtils.createTempFile("compressed6791_", "3"); String filename = f.absolutePath(); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", + try(CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(filename + ".metadata"), null, SequentialWriterOption.DEFAULT, CompressionParams.snappy(32), sstableMetadataCollector)) @@ -119,9 +125,8 @@ public class CompressedRandomAccessReaderTest writer.finish(); } - try (FileHandle.Builder builder = new FileHandle.Builder(filename) - .withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true)); - FileHandle fh = builder.complete(); + try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true); + FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete(); RandomAccessReader reader = fh.createReader()) { String res = reader.readLine(); @@ -151,7 +156,7 @@ public class CompressedRandomAccessReaderTest try { writeSSTable(file, CompressionParams.snappy(chunkLength), 10); - CompressionMetadata metadata = new CompressionMetadata(filename + ".metadata", file.length(), true); + CompressionMetadata metadata = CompressionMetadata.open(new File(filename + ".metadata"), file.length(), true); long chunks = 2761628520L; long midPosition = (chunks / 2L) * chunkLength; @@ -177,9 +182,8 @@ public class CompressedRandomAccessReaderTest final String filename = f.absolutePath(); writeSSTable(f, compressed ? CompressionParams.snappy() : null, junkSize); - CompressionMetadata compressionMetadata = compressed ? new CompressionMetadata(filename + ".metadata", f.length(), true) : null; - try (FileHandle.Builder builder = new FileHandle.Builder(filename).mmapped(usemmap).withCompressionMetadata(compressionMetadata); - FileHandle fh = builder.complete(); + try (CompressionMetadata compressionMetadata = compressed ? CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true) : null; + FileHandle fh = new FileHandle.Builder(f).mmapped(usemmap).withCompressionMetadata(compressionMetadata).complete(); RandomAccessReader reader = fh.createReader()) { String expected = "The quick brown fox jumps over the lazy dog"; @@ -203,7 +207,7 @@ public class CompressedRandomAccessReaderTest final String filename = f.absolutePath(); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); try(SequentialWriter writer = params != null - ? new CompressedSequentialWriter(f, filename + ".metadata", + ? new CompressedSequentialWriter(f, new File(filename + ".metadata"), null, SequentialWriterOption.DEFAULT, params, sstableMetadataCollector) : new SequentialWriter(f)) @@ -243,7 +247,7 @@ public class CompressedRandomAccessReaderTest assertTrue(metadata.createFileIfNotExists()); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata.path(), + try (SequentialWriter writer = new CompressedSequentialWriter(file, metadata, null, SequentialWriterOption.DEFAULT, CompressionParams.snappy(), sstableMetadataCollector)) { @@ -252,16 +256,14 @@ public class CompressedRandomAccessReaderTest } // open compression metadata and get chunk information - CompressionMetadata meta = new CompressionMetadata(metadata.path(), file.length(), true); - CompressionMetadata.Chunk chunk = meta.chunkFor(0); - - try (FileHandle.Builder builder = new FileHandle.Builder(file.path()).withCompressionMetadata(meta); - FileHandle fh = builder.complete(); + try (CompressionMetadata meta = CompressionMetadata.open(metadata, file.length(), true); + FileHandle fh = new FileHandle.Builder(file).withCompressionMetadata(meta).complete(); RandomAccessReader reader = fh.createReader()) {// read and verify compressed data assertEquals(CONTENT, reader.readLine()); Random random = new Random(); - try(RandomAccessFile checksumModifier = new RandomAccessFile(file.toJavaIOFile(), "rw")) + CompressionMetadata.Chunk chunk = meta.chunkFor(0); + try (RandomAccessFile checksumModifier = new RandomAccessFile(file.toJavaIOFile(), "rw")) { byte[] checksum = new byte[4]; @@ -312,4 +314,4 @@ public class CompressedRandomAccessReaderTest file.write(checksum); SyncUtil.sync(file.getFD()); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java index 2b50633463..afa469c487 100644 --- a/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java +++ b/test/unit/org/apache/cassandra/io/compress/CompressedSequentialWriterTest.java @@ -21,29 +21,39 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; - -import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH; -import static org.apache.commons.io.FileUtils.readFileToByteArray; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; import com.google.common.io.Files; import org.junit.After; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assert; - import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.util.DataPosition; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.io.util.SequentialWriterTest; import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.schema.CompressionParams.DEFAULT_CHUNK_LENGTH; +import static org.apache.commons.io.FileUtils.readFileToByteArray; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class CompressedSequentialWriterTest extends SequentialWriterTest { private CompressionParams compressionParameters; @@ -117,7 +127,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest byte[] dataPre = new byte[bytesToTest]; byte[] rawPost = new byte[bytesToTest]; - try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, filename + ".metadata", + try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(filename + ".metadata"), null, SequentialWriterOption.DEFAULT, compressionParameters, sstableMetadataCollector)) @@ -150,8 +160,8 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest } assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true)); - FileHandle fh = builder.complete(); + try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true); + FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete(); RandomAccessReader reader = fh.createReader()) { assertEquals(dataPre.length + rawPost.length, reader.length()); @@ -217,7 +227,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest compressionParameters = new CompressionParams(MockCompressor.class.getTypeName(), MockCompressor.paramsFor(ratio, extra), DEFAULT_CHUNK_LENGTH, ratio); - try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, f.path() + ".metadata", + try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, new File(f.path() + ".metadata"), null, SequentialWriterOption.DEFAULT, compressionParameters, sstableMetadataCollector)) @@ -228,8 +238,8 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest } assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(filename).withCompressionMetadata(new CompressionMetadata(filename + ".metadata", f.length(), true)); - FileHandle fh = builder.complete(); + try (CompressionMetadata compressionMetadata = CompressionMetadata.open(new File(filename + ".metadata"), f.length(), true); + FileHandle fh = new FileHandle.Builder(f).withCompressionMetadata(compressionMetadata).complete(); RandomAccessReader reader = fh.createReader()) { assertEquals(size, reader.length()); @@ -276,7 +286,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest final int bufferSize = 48; final int writeSize = 64; byte[] toWrite = new byte[writeSize]; - try (SequentialWriter writer = new CompressedSequentialWriter(tempFile, offsetsFile.path(), + try (SequentialWriter writer = new CompressedSequentialWriter(tempFile, offsetsFile, null, SequentialWriterOption.DEFAULT, CompressionParams.lz4(bufferSize), new MetadataCollector(new ClusteringComparator(UTF8Type.instance)))) @@ -330,7 +340,7 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest private TestableCSW(File file, File offsetsFile) throws IOException { - this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile.path(), + this(file, offsetsFile, new CompressedSequentialWriter(file, offsetsFile, null, SequentialWriterOption.DEFAULT, CompressionParams.lz4(BUFFER_SIZE, MAX_COMPRESSED), new MetadataCollector(new ClusteringComparator(UTF8Type.instance)))); @@ -385,4 +395,4 @@ public class CompressedSequentialWriterTest extends SequentialWriterTest } } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java new file mode 100644 index 0000000000..321fe57356 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/compress/CompressionMetadataTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.compress; + + +import org.junit.Test; + +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.Memory; +import org.apache.cassandra.schema.CompressionParams; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class CompressionMetadataTest +{ + File chunksIndexFile = new File("/path/to/metadata"); + CompressionParams params = CompressionParams.zstd(); + long dataLength = 1000; + long compressedFileLength = 100; + + private CompressionMetadata newCompressionMetadata(Memory memory) + { + return new CompressionMetadata(chunksIndexFile, + params, + memory, + memory.size(), + dataLength, + compressedFileLength); + } + + @Test + public void testMemoryIsFreed() + { + Memory memory = Memory.allocate(10); + CompressionMetadata cm = newCompressionMetadata(memory); + + cm.close(); + assertThat(cm.isCleanedUp()).isTrue(); + assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size); + } + + @Test + public void testMemoryIsShared() + { + Memory memory = Memory.allocate(10); + CompressionMetadata cm = newCompressionMetadata(memory); + + CompressionMetadata copy = cm.sharedCopy(); + assertThat(copy).isNotSameAs(cm); + + cm.close(); + assertThat(cm.isCleanedUp()).isFalse(); + assertThat(copy.isCleanedUp()).isFalse(); + assertThat(memory.size()).isEqualTo(10); // expected that no expection is thrown since memory should not be released yet + + copy.close(); + assertThat(cm.isCleanedUp()).isTrue(); + assertThat(copy.isCleanedUp()).isTrue(); + assertThatExceptionOfType(AssertionError.class).isThrownBy(memory::size); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java new file mode 100644 index 0000000000..847f3ac753 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.HashSet; +import java.util.function.Function; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.junit.Test; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.io.sstable.Component.Type; +import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class ComponentTest +{ + public static abstract class Format1 extends AbstractSSTableFormat + { + public final static Format1 instance = Mockito.spy(Format1.class); + } + + public static abstract class Format2 extends AbstractSSTableFormat + { + public final static Format2 instance = Mockito.spy(Format2.class); + } + + static + { + DatabaseDescriptor.daemonInitialization(() -> { + Config config = DatabaseDescriptor.loadConfig(); + config.sstable_formats = Lists.newArrayList(config.sstable_formats); + config.sstable_formats.add(new ParameterizedClass(Format1.class.getName(), ImmutableMap.of(Config.SSTABLE_FORMAT_ID, "11", Config.SSTABLE_FORMAT_NAME, "first"))); + config.sstable_formats.add(new ParameterizedClass(Format2.class.getName(), ImmutableMap.of(Config.SSTABLE_FORMAT_ID, "12", Config.SSTABLE_FORMAT_NAME, "second"))); + return config; + }); + } + + @Test + public void testTypes() + { + Function componentFactory = Mockito.mock(Function.class); + + // do not allow to define a type with the same name or repr as the existing type for this or parent format + assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> Type.createSingleton(Components.Types.TOC.name, Components.Types.TOC.repr + "x", Format1.class)); + assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> Type.createSingleton(Components.Types.TOC.name + "x", Components.Types.TOC.repr, Format2.class)); + + // allow to define a format with other name and repr + Type t1 = Type.createSingleton("ONE", "One.db", Format1.class); + + // allow to define a format with the same name and repr for two different formats + Type t2f1 = Type.createSingleton("TWO", "Two.db", Format1.class); + Type t2f2 = Type.createSingleton("TWO", "Two.db", Format2.class); + assertThat(t2f1).isNotEqualTo(t2f2); + + assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> Type.createSingleton(null, "-Three.db", Format1.class)); + + assertThat(Type.fromRepresentation("should be custom", BigFormat.getInstance().getType())).isSameAs(Components.Types.CUSTOM); + assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BigFormat.getInstance().getType())).isSameAs(Components.Types.TOC); + assertThat(Type.fromRepresentation(t1.repr, Format1.instance.getType())).isSameAs(t1); + assertThat(Type.fromRepresentation(t2f1.repr, Format1.instance.getType())).isSameAs(t2f1); + assertThat(Type.fromRepresentation(t2f2.repr, Format2.instance.getType())).isSameAs(t2f2); + } + + @Test + public void testComponents() + { + Type t3f1 = Type.createSingleton("THREE", "Three.db", Format1.class); + Type t3f2 = Type.createSingleton("THREE", "Three.db", Format2.class); + Type t4f1 = Type.create("FOUR", ".*-Four.db", Format1.class); + Type t4f2 = Type.create("FOUR", ".*-Four.db", Format2.class); + + Component c1 = t3f1.getSingleton(); + Component c2 = t3f2.getSingleton(); + + assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t3f1.createComponent(t3f1.repr)); + assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t3f2.createComponent(t3f2.repr)); + assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f1.getSingleton()); + assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f2.getSingleton()); + + assertThat(Component.parse(t3f1.repr, Format1.instance.getType())).isSameAs(c1); + assertThat(Component.parse(t3f2.repr, Format2.instance.getType())).isSameAs(c2); + assertThat(c1).isNotEqualTo(c2); + assertThat(c1.type).isSameAs(t3f1); + assertThat(c2.type).isSameAs(t3f2); + + Component c3 = Component.parse("abc-Four.db", Format1.instance.getType()); + Component c4 = Component.parse("abc-Four.db", Format2.instance.getType()); + assertThat(c3.type).isSameAs(t4f1); + assertThat(c4.type).isSameAs(t4f2); + assertThat(c3.name).isEqualTo("abc-Four.db"); + assertThat(c4.name).isEqualTo("abc-Four.db"); + assertThat(c3).isNotEqualTo(c4); + assertThat(c3).isNotEqualTo(c1); + assertThat(c4).isNotEqualTo(c2); + + Component c5 = Component.parse("abc-Five.db", Format1.instance.getType()); + assertThat(c5.type).isSameAs(Components.Types.CUSTOM); + assertThat(c5.name).isEqualTo("abc-Five.db"); + + Component c6 = Component.parse("Data.db", Format2.instance.getType()); + assertThat(c6.type).isSameAs(Components.Types.DATA); + assertThat(c6).isSameAs(Components.DATA); + + HashSet s1 = Sets.newHashSet(Component.getSingletonsFor(Format1.class)); + HashSet s2 = Sets.newHashSet(Component.getSingletonsFor(Format2.class)); + assertThat(s1).contains(c1, Components.DATA, Components.STATS, Components.COMPRESSION_INFO); + assertThat(s2).contains(c2, Components.DATA, Components.STATS, Components.COMPRESSION_INFO); + assertThat(s1).doesNotContain(c2); + assertThat(s2).doesNotContain(c1); + + assertThat(Sets.newHashSet(Component.getSingletonsFor(Format1.instance))).isEqualTo(s1); + assertThat(Sets.newHashSet(Component.getSingletonsFor(Format2.instance))).isEqualTo(s2); + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java index 7ffb63f335..9d2243b3c9 100644 --- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java @@ -20,7 +20,6 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.UUID; -import org.apache.cassandra.io.util.File; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.BeforeClass; @@ -29,11 +28,14 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Directories; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class DescriptorTest { @@ -84,19 +86,19 @@ public class DescriptorTest private void testFromFilenameFor(File dir) { - checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG)); + checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), SSTableFormat.Type.current())); // secondary index String idxName = "myidx"; File idxDir = new File(dir.absolutePath() + File.pathSeparator() + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName); - checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), SSTableFormat.Type.BIG)); + checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), SSTableFormat.Type.current())); } private void checkFromFilename(Descriptor original) { - File file = new File(original.filenameFor(Component.DATA)); + File file = original.fileFor(Components.DATA); - Pair pair = Descriptor.fromFilenameWithComponent(file); + Pair pair = Descriptor.fromFileWithComponent(file); Descriptor desc = pair.left; assertEquals(original.directory, desc.directory); @@ -104,7 +106,7 @@ public class DescriptorTest assertEquals(original.cfname, desc.cfname); assertEquals(original.version, desc.version); assertEquals(original.id, desc.id); - assertEquals(Component.DATA, pair.right); + assertEquals(Components.DATA, pair.right); } @Test @@ -112,8 +114,8 @@ public class DescriptorTest { // Descriptor should be equal when parent directory points to the same directory File dir = new File("."); - Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG); - Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG); + Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.current()); + Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.current()); assertEquals(desc1, desc2); assertEquals(desc1.hashCode(), desc2.hashCode()); } @@ -121,16 +123,14 @@ public class DescriptorTest @Test public void validateNames() { - String[] names = { - "ma-1-big-Data.db", - // 2ndary index - ".idx1" + File.pathSeparator() + "ma-1-big-Data.db", - }; + String name = SSTableFormat.Type.current().name; + String[] fileNames = { "ma-1-" + name + "-Data.db", + // 2ndary index + ".idx1" + File.pathSeparator() + "ma-1-" + name + "-Data.db", + }; - for (String name : names) - { - assertNotNull(Descriptor.fromFilename(name)); - } + for (String fileName : fileNames) + assertNotNull(Descriptor.fromFileWithComponent(new File(fileName), false).left); } @Test @@ -146,7 +146,7 @@ public class DescriptorTest { try { - Descriptor d = Descriptor.fromFilename(name); + Descriptor d = Descriptor.fromFile(new File(name)); Assert.fail(name); } catch (Throwable e) { //good @@ -301,7 +301,7 @@ public class DescriptorTest { for (String filePath : filePaths) { - Descriptor descriptor = Descriptor.fromFilename(filePath); + Descriptor descriptor = Descriptor.fromFile(new File(filePath)); Assert.assertNotNull(descriptor); Assert.assertEquals(String.format("Expected keyspace not found for %s", filePath), expectedKeyspace, descriptor.ksname); Assert.assertEquals(String.format("Expected table not found for %s", filePath), expectedTable, descriptor.cfname); diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 655e552584..6c534cd603 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -19,24 +19,19 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Random; -import com.google.common.collect.Lists; +import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileInputStreamPlus; -import org.apache.cassandra.io.util.FileOutputStreamPlus; +import com.google.common.collect.Lists; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,28 +41,31 @@ import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SinglePartitionSliceCommandTest; import org.apache.cassandra.db.compaction.AbstractCompactionTask; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.repair.PendingAntiCompaction; -import org.apache.cassandra.db.streaming.CassandraOutgoingFile; -import org.apache.cassandra.db.SinglePartitionSliceCommandTest; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.rows.RangeTombstoneMarker; import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.streaming.CassandraOutgoingFile; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.OutgoingStream; -import org.apache.cassandra.streaming.StreamPlan; import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamPlan; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.OutputHandler; import org.apache.cassandra.utils.TimeUUID; import static java.util.Collections.singleton; @@ -144,8 +142,9 @@ public class LegacySSTableTest */ protected Descriptor getDescriptor(String legacyVersion, String table) throws IOException { - Path file = Files.list(getTableDir(legacyVersion, table).toPath()).findFirst().orElseThrow(() -> new RuntimeException(String.format("No files for version=%s and table=%s", legacyVersion, table))); - return Descriptor.fromFilename(new File(file)); + File[] files = getTableDir(legacyVersion, table).list(); + Preconditions.checkArgument(files.length > 0, "No files for version=%s and table=%s", legacyVersion, table); + return Descriptor.fromFileWithComponent(files[0]).left; } @Test @@ -332,7 +331,7 @@ public class LegacySSTableTest for (SSTableReader sstable : cfs.getLiveSSTables()) { - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(true).build())) { verifier.verify(); if (!sstable.descriptor.version.isLatestVersion()) @@ -344,7 +343,7 @@ public class LegacySSTableTest // make sure we don't throw any exception if not checking version: for (SSTableReader sstable : cfs.getLiveSSTables()) { - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkVersion(false).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkVersion(false).build())) { verifier.verify(); } @@ -420,7 +419,7 @@ public class LegacySSTableTest private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception { String table = String.format(tablePattern, legacyVersion); - SSTableReader sstable = SSTableReader.open(getDescriptor(legacyVersion, table)); + SSTableReader sstable = SSTableReader.open(null, getDescriptor(legacyVersion, table)); IPartitioner p = sstable.getPartitioner(); List> ranges = new ArrayList<>(); ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100")))); @@ -604,6 +603,7 @@ public class LegacySSTableTest @Test public void testGenerateSstables() throws Throwable { + SSTableFormat format = SSTableFormat.Type.current().info; Random rand = new Random(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < 128; i++) @@ -616,31 +616,31 @@ public class LegacySSTableTest { String valPk = Integer.toString(pk); QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_simple (pk, val) VALUES ('%s', '%s')", - BigFormat.latestVersion, valPk, "foo bar baz")); + format.getLatestVersion(), valPk, "foo bar baz")); QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_simple_counter SET val = val + 1 WHERE pk = '%s'", - BigFormat.latestVersion, valPk)); + format.getLatestVersion(), valPk)); for (int ck = 0; ck < 50; ck++) { String valCk = Integer.toString(ck); QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_clust (pk, ck, val) VALUES ('%s', '%s', '%s')", - BigFormat.latestVersion, valPk, valCk + longString, randomString)); + format.getLatestVersion(), valPk, valCk + longString, randomString)); QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_clust_counter SET val = val + 1 WHERE pk = '%s' AND ck='%s'", - BigFormat.latestVersion, valPk, valCk + longString)); + format.getLatestVersion(), valPk, valCk + longString)); } } StorageService.instance.forceKeyspaceFlush("legacy_tables", ColumnFamilyStore.FlushReason.UNIT_TESTS); - File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", BigFormat.latestVersion)); + File ksDir = new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables", format.getLatestVersion())); ksDir.tryCreateDirectories(); - copySstablesFromTestData(String.format("legacy_%s_simple", BigFormat.latestVersion), ksDir); - copySstablesFromTestData(String.format("legacy_%s_simple_counter", BigFormat.latestVersion), ksDir); - copySstablesFromTestData(String.format("legacy_%s_clust", BigFormat.latestVersion), ksDir); - copySstablesFromTestData(String.format("legacy_%s_clust_counter", BigFormat.latestVersion), ksDir); + copySstablesFromTestData(String.format("legacy_%s_simple", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_simple_counter", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust", format.getLatestVersion()), ksDir); + copySstablesFromTestData(String.format("legacy_%s_clust_counter", format.getLatestVersion()), ksDir); } public static void copySstablesFromTestData(String table, File ksDir) throws IOException diff --git a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java similarity index 96% rename from test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java rename to test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java index 14b48b7855..57e85927bb 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.io.sstable.format; +package org.apache.cassandra.io.sstable; import java.io.IOException; @@ -32,6 +32,7 @@ import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; @@ -78,7 +79,7 @@ public class RangeAwareSSTableWriterTest 0, null, false, - SSTableFormat.Type.BIG, + SSTableFormat.Type.current(), 0, 0, txn, @@ -88,4 +89,4 @@ public class RangeAwareSSTableWriterTest assertEquals(0L, writer.getFilePointer()); } -} \ No newline at end of file +} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java index ec01865786..21ac51ee86 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableCorruptionDetectionTest.java @@ -21,30 +21,40 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.*; -import java.util.function.*; +import java.util.Random; +import java.util.function.Consumer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.*; -import org.apache.cassandra.cache.*; -import org.apache.cassandra.config.*; -import org.apache.cassandra.db.*; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.UpdateBuilder; +import org.apache.cassandra.Util; +import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.marshal.*; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.util.*; -import org.apache.cassandra.schema.*; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.CompressionParams; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java similarity index 70% rename from test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java rename to test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java index b5aaf8edf4..69e8670a99 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable.format; +package org.apache.cassandra.io.sstable; import java.io.IOException; import java.nio.ByteBuffer; @@ -24,40 +24,42 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import org.apache.cassandra.db.commitlog.CommitLog; -import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.ColumnMetadata; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.big.BigTableWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.Pair; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; - -import org.junit.Assert; - -import org.junit.BeforeClass; -import org.junit.Test; public class SSTableFlushObserverTest { @@ -72,15 +74,15 @@ public class SSTableFlushObserverTest private static final String CF_NAME = "flush_observer"; @Test - public void testFlushObserver() + public void testFlushObserver() throws Exception { TableMetadata cfm = - TableMetadata.builder(KS_NAME, CF_NAME) - .addPartitionKeyColumn("id", UTF8Type.instance) - .addRegularColumn("first_name", UTF8Type.instance) - .addRegularColumn("age", Int32Type.instance) - .addRegularColumn("height", LongType.instance) - .build(); + TableMetadata.builder(KS_NAME, CF_NAME) + .addPartitionKeyColumn("id", UTF8Type.instance) + .addRegularColumn("first_name", UTF8Type.instance) + .addRegularColumn("age", Int32Type.instance) + .addRegularColumn("height", LongType.instance) + .build(); LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.COMPACTION); FlushObserver observer = new FlushObserver(); @@ -100,12 +102,14 @@ public class SSTableFlushObserverTest new SequenceBasedSSTableId(0), sstableFormat); - BigTableWriter writer = new BigTableWriter(descriptor, - 10L, 0L, null, false, TableMetadataRef.forOfflineTools(cfm), - new MetadataCollector(cfm.comparator).sstableLevel(0), - new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS), - Collections.singletonList(observer), - transaction); + SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor) + .setKeyCount(10) + .setTableMetadataRef(TableMetadataRef.forOfflineTools(cfm)) + .setMetadataCollector(new MetadataCollector(cfm.comparator).sstableLevel(0)) + .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS)) + .setFlushObservers(Collections.singletonList(observer)) + .addDefaultComponents() + .build(transaction, null); SSTableReader reader = null; Multimap> expected = ArrayListMultimap.create(); @@ -116,21 +120,21 @@ public class SSTableFlushObserverTest ByteBuffer key = UTF8Type.instance.fromString("key1"); expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(27)), - BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("jack")), + BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("jack")), BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(183L)))); writer.append(new RowIterator(cfm, key.duplicate(), Collections.singletonList(buildRow(expected.get(key))))); key = UTF8Type.instance.fromString("key2"); expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)), - BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("jim")), + BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("jim")), BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(180L)))); writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key))))); key = UTF8Type.instance.fromString("key3"); expected.putAll(key, Arrays.asList(BufferCell.live(getColumn(cfm, "age"), now, Int32Type.instance.decompose(30)), - BufferCell.live(getColumn(cfm, "first_name"), now,UTF8Type.instance.fromString("ken")), + BufferCell.live(getColumn(cfm, "first_name"), now, UTF8Type.instance.fromString("ken")), BufferCell.live(getColumn(cfm, "height"), now, LongType.instance.decompose(178L)))); writer.append(new RowIterator(cfm, key, Collections.singletonList(buildRow(expected.get(key))))); @@ -145,21 +149,14 @@ public class SSTableFlushObserverTest Assert.assertTrue(observer.isComplete); Assert.assertEquals(expected.size(), observer.rows.size()); - for (Pair e : observer.rows.keySet()) + for (Triple e : observer.rows.keySet()) { - ByteBuffer key = e.left; - Long indexPosition = e.right; - - try (FileDataInput index = reader.ifile.createReader(indexPosition)) - { - ByteBuffer indexKey = ByteBufferUtil.readWithShortLength(index); - Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey)); - } - catch (IOException ex) - { - throw new FSReadError(ex, reader.getIndexFilename()); - } + ByteBuffer key = e.getLeft(); + long dataPosition = e.getMiddle(); + long indexPosition = e.getRight(); + DecoratedKey indexKey = reader.keyAtPositionFromSecondaryIndex(indexPosition); + Assert.assertEquals(0, UTF8Type.instance.compare(key, indexKey.getKey())); Assert.assertEquals(expected.get(key), observer.rows.get(e)); } } @@ -190,18 +187,21 @@ public class SSTableFlushObserverTest private static class FlushObserver implements SSTableFlushObserver { - private final Multimap, Cell> rows = ArrayListMultimap.create(); - private Pair currentKey; + private final Multimap, Cell> rows = ArrayListMultimap.create(); + private final Multimap, Cell> staticRows = ArrayListMultimap.create(); + + private Triple currentKey; private boolean isComplete; @Override public void begin() - {} + { + } @Override - public void startPartition(DecoratedKey key, long indexPosition) + public void startPartition(DecoratedKey key, long dataPosition, long indexPosition) { - currentKey = Pair.create(key.getKey(), indexPosition); + currentKey = ImmutableTriple.of(key.getKey(), dataPosition, indexPosition); } @Override @@ -216,6 +216,12 @@ public class SSTableFlushObserverTest { isComplete = true; } + + @Override + public void staticRow(Row staticRow) + { + staticRow.forEach((c) -> staticRows.put(currentKey, (Cell) c)); + } } private static Row buildRow(Collection> cells) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java new file mode 100644 index 0000000000..167c02ec25 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.ParameterizedClass; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.utils.Pair; +import org.mockito.Mockito; + +import static org.apache.cassandra.config.Config.SSTABLE_FORMAT_ID; +import static org.apache.cassandra.config.Config.SSTABLE_FORMAT_NAME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class SSTableFormatTypeTest +{ + public static abstract class Format1 extends AbstractSSTableFormat + { + public static Format1 instance = null; + } + + public static abstract class Format2 extends AbstractSSTableFormat + { + public static Format2 instance = null; + } + + public static abstract class NonSSTableFormat + { + public static NonSSTableFormat instance = null; + } + + public abstract static class BrokenSSTableFormat extends AbstractSSTableFormat + { + public BrokenSSTableFormat() + { + // initialization will fail because it is abstract class + } + } + + private Map>> factories; + Pair, SSTableFormat.Type[]> formats; + + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.clientInitialization(); + } + + @Before + public void before() + { + Format1.instance = Mockito.spy(Format1.class); + Format2.instance = Mockito.spy(Format2.class); + NonSSTableFormat.instance = Mockito.spy(NonSSTableFormat.class); + } + + @Test + public void testValidConfig() + { + init(format(Format1.class, "0", "aaa"), format(Format2.class, "1", "bbb")); + verifyFormat(Format1.class, 0, "aaa"); + verifyFormat(Format2.class, 1, "bbb"); + assertThat(formats.left.get(0).name).isEqualTo("aaa"); + } + + @Test + public void testDefaultSelectedByOrder() + { + init(format(Format1.class, "1", "aaa"), format(Format2.class, "0", "bbb")); + verifyFormat(Format1.class, 1, "aaa"); + verifyFormat(Format2.class, 0, "bbb"); + assertThat(formats.left.get(0).name).isEqualTo("aaa"); + } + + @Test + public void testNonConsecutiveOrdinals() + { + init(format(Format1.class, "5", "aaa"), format(Format2.class, "10", "bbb")); + verifyFormat(Format1.class, 5, "aaa"); + verifyFormat(Format2.class, 10, "bbb"); + assertThat(formats.left.get(0).name).isEqualTo("aaa"); + } + + @Test + public void testConfigValidation() + { + // invalid name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "Aa"))) + .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "a-a"))) + .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "a1"))) + .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); + + // invalid id + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "-1", "aaa"))) + .withMessageContainingAll("'id'", "Format1", "must be within bounds [0..127] range"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "128", "aaa"))) + .withMessageContainingAll("'id'", "Format1", "must be within bounds [0..127] range"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "x", "aaa"))) + .withMessageContainingAll("'id'", "Format1", "must be an integer"); + + // duplicate name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "aaa"), format(Format2.class, "1", "aaa"))) + .withMessageContainingAll("Name 'aaa' of sstable format", "Format2", "is already defined"); + + // duplicate id + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "aaa"), format(Format2.class, "0", "bbb"))) + .withMessageContainingAll("ID '0' of sstable format", "Format2", "is already defined"); + + // missing name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class.getName(), ImmutableMap.of(SSTABLE_FORMAT_ID, "0")))) + .withMessageContainingAll("Missing 'name' parameter", "Format1"); + + // missing id + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class.getName(), ImmutableMap.of(SSTABLE_FORMAT_NAME, "aaa")))) + .withMessageContainingAll("Missing 'id' parameter", "Format1"); + + // not an sstable format class + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format(NonSSTableFormat.class, "0", "aaa"))) + .withMessageContainingAll("NonSSTableFormat", "format aaa", "does not implement org.apache.cassandra.io.sstable.format.SSTableFormat"); + + // missing class + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format("org.apache.cassandra.io.sstable.format.dummy.DummyFormat", ImmutableMap.of(SSTABLE_FORMAT_ID, "0", SSTABLE_FORMAT_NAME, "aaa")))) + .withMessageContainingAll("Unable to find sstable format class", "org.apache.cassandra.io.sstable.format.dummy.DummyFormat"); + + // failed initialization + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format(BrokenSSTableFormat.class, "0", "aaa"))) + .withMessageContainingAll(BrokenSSTableFormat.class.getName(), "sstable format"); + } + + private void verifyFormat(Class expectedClass, int expectedOrdinal, String expectedName) + { + SSTableFormat.Type formatType = formats.right[expectedOrdinal]; + assertThat(formatType).isNotNull(); + assertThat(formatType.name).isEqualTo(expectedName); + assertThat(formatType.ordinal).isEqualTo(expectedOrdinal); + assertThat(formatType.info).isInstanceOf(expectedClass); + assertThat(formats.left.stream().filter(f -> f.name.equals(expectedName)).findFirst().get()).isSameAs(formatType); + } + + private ParameterizedClass format(Class clazz, String id, String name, String... otherParams) + { + ImmutableMap.Builder b = ImmutableMap.builder(); + b.put(SSTABLE_FORMAT_ID, id); + b.put(SSTABLE_FORMAT_NAME, name); + for (int i = 0; i < otherParams.length; i += 2) + b.put(otherParams[i], otherParams[i + 1]); + return format(clazz.getName(), b.build()); + } + + private ParameterizedClass format(String className, Map params) + { + return new ParameterizedClass(className, params); + } + + private Map>> load(ParameterizedClass... formats) + { + return DatabaseDescriptor.loadSSTableFormatFactories(Arrays.asList(formats)); + } + + private void init(ParameterizedClass... formats) + { + this.factories = load(formats); + this.formats = SSTableFormat.Type.readFactories(factories); + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java index 4572e5c9e4..8e53d79d45 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java @@ -33,10 +33,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import com.google.common.collect.Sets; -import org.apache.cassandra.io.util.File; import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -60,16 +61,17 @@ import org.apache.cassandra.db.marshal.TupleType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.marshal.UserType; import org.apache.cassandra.db.rows.EncodingStats; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.SequentialWriter; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -101,6 +103,12 @@ public class SSTableHeaderFixTest return MockSchema.sstableIdGenerators(); } + @BeforeClass + public static void beforeClass() + { + Assume.assumeTrue(BigFormat.isDefault()); + } + @Before public void setup() { @@ -178,7 +186,7 @@ public class SSTableHeaderFixTest return new TupleType(Collections.singletonList(UTF8Type.instance)); } - private static final Version version = BigFormat.instance.getVersion("mc"); + private static final Version version = BigFormat.getInstance().getVersion("mc"); private TableMetadata tableMetadata; private final Set updatedColumns = new HashSet<>(); @@ -822,11 +830,11 @@ public class SSTableHeaderFixTest try { - Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation), SSTableFormat.Type.BIG); + Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation), BigFormat.getInstance().getType()); // Just create the component files - we don't really need those. for (Component component : requiredComponents) - assertTrue(new File(desc.filenameFor(component)).createFileIfNotExists()); + assertTrue(desc.fileFor(component).createFileIfNotExists()); AbstractType partitionKey = headerMetadata.partitionKeyType; List> clusteringKey = headerMetadata.clusteringColumns() @@ -842,7 +850,7 @@ public class SSTableHeaderFixTest .filter(cd -> cd.kind == ColumnMetadata.Kind.REGULAR) .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); - File statsFile = new File(desc.filenameFor(Component.STATS)); + File statsFile = desc.fileFor(Components.STATS); SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey, clusteringKey, staticColumns, @@ -855,7 +863,7 @@ public class SSTableHeaderFixTest out.finish(); } - return new File(desc.filenameFor(Component.DATA)); + return desc.fileFor(Components.DATA); } catch (Exception e) { @@ -988,9 +996,9 @@ public class SSTableHeaderFixTest private SerializationHeader.Component readHeader(File sstable) throws Exception { - Descriptor desc = Descriptor.fromFilename(sstable); + Descriptor desc = Descriptor.fromFileWithComponent(sstable, false).left; return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER); } - private static final Component[] requiredComponents = new Component[]{ Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.TOC }; + private static final Component[] requiredComponents = new Component[]{ Components.DATA, Components.FILTER, Components.PRIMARY_INDEX, Components.TOC }; } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index 9567541048..e2b46e7803 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -20,24 +20,39 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import com.google.common.collect.Sets; - +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -51,7 +66,16 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.format.big.IndexSummaryComponent; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummarySupport; +import org.apache.cassandra.io.sstable.keycache.KeyCache; +import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.MmappedRegions; @@ -60,7 +84,7 @@ import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FilterFactory; +import org.assertj.core.api.Assertions; import static java.lang.String.format; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; @@ -68,6 +92,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; public class SSTableReaderTest { @@ -180,7 +205,7 @@ public class SSTableReaderTest for (int j = 0; j < 100; j += 2) { DecoratedKey dk = Util.dk(String.valueOf(j)); - FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ).position); + FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ)); DecoratedKey keyInDisk = sstable.decorateKey(ByteBufferUtil.readWithShortLength(file)); assert keyInDisk.equals(dk) : format("%s != %s in %s", keyInDisk, dk, file.getPath()); } @@ -189,7 +214,7 @@ public class SSTableReaderTest for (int j = 1; j < 110; j += 2) { DecoratedKey dk = Util.dk(String.valueOf(j)); - assert sstable.getPosition(dk, SSTableReader.Operator.EQ) == null; + assert sstable.getPosition(dk, SSTableReader.Operator.EQ) < 0; } } finally @@ -301,10 +326,10 @@ public class SSTableReaderTest CompactionManager.instance.performMaximal(store, false); SSTableReader sstable = store.getLiveSSTables().iterator().next(); - long p2 = sstable.getPosition(k(2), SSTableReader.Operator.EQ).position; - long p3 = sstable.getPosition(k(3), SSTableReader.Operator.EQ).position; - long p6 = sstable.getPosition(k(6), SSTableReader.Operator.EQ).position; - long p7 = sstable.getPosition(k(7), SSTableReader.Operator.EQ).position; + long p2 = sstable.getPosition(k(2), SSTableReader.Operator.EQ); + long p3 = sstable.getPosition(k(3), SSTableReader.Operator.EQ); + long p6 = sstable.getPosition(k(6), SSTableReader.Operator.EQ); + long p7 = sstable.getPosition(k(7), SSTableReader.Operator.EQ); SSTableReader.PartitionPositionBounds p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0); @@ -337,6 +362,7 @@ public class SSTableReaderTest @Test public void testGetPositionsKeyCacheStats() { + Assume.assumeTrue(KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current())); ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD2); partitioner = store.getPartitioner(); CacheService.instance.keyCache.setCapacity(1000); @@ -354,18 +380,20 @@ public class SSTableReaderTest CompactionManager.instance.performMaximal(store, false); SSTableReader sstable = store.getLiveSSTables().iterator().next(); + KeyCache keyCache = ((KeyCacheSupport) sstable).getKeyCache(); + assumeTrue(keyCache.isEnabled()); // existing, non-cached key sstable.getPosition(k(2), SSTableReader.Operator.EQ); - assertEquals(1, sstable.getKeyCacheRequest()); - assertEquals(0, sstable.getKeyCacheHit()); + assertEquals(1, keyCache.getRequests()); + assertEquals(0, keyCache.getHits()); // existing, cached key sstable.getPosition(k(2), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getKeyCacheRequest()); - assertEquals(1, sstable.getKeyCacheHit()); + assertEquals(2, keyCache.getRequests()); + assertEquals(1, keyCache.getHits()); // non-existing key (it is specifically chosen to not be rejected by Bloom Filter check) sstable.getPosition(k(14), SSTableReader.Operator.EQ); - assertEquals(3, sstable.getKeyCacheRequest()); - assertEquals(1, sstable.getKeyCacheHit()); + assertEquals(3, keyCache.getRequests()); + assertEquals(1, keyCache.getHits()); } @Test @@ -388,38 +416,38 @@ public class SSTableReaderTest Util.flush(store); CompactionManager.instance.performMaximal(store, false); - SSTableReader sstable = store.getLiveSSTables().iterator().next(); + SSTableReaderWithFilter sstable = (SSTableReaderWithFilter) store.getLiveSSTables().iterator().next(); // the keys are specifically chosen to cover certain use cases // existing key is read from index sstable.getPosition(k(2), SSTableReader.Operator.EQ); - assertEquals(1, sstable.getBloomFilterTruePositiveCount()); - assertEquals(0, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(0, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(1, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(0, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount()); // existing key is read from Cache Key sstable.getPosition(k(2), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getBloomFilterTruePositiveCount()); - assertEquals(0, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(0, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(2, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(0, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount()); // non-existing key is rejected by Bloom Filter check sstable.getPosition(k(10), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getBloomFilterTruePositiveCount()); - assertEquals(1, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(0, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(2, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount()); // non-existing key is rejected by sstable keys range check sstable.getPosition(k(99), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getBloomFilterTruePositiveCount()); - assertEquals(1, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(1, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(2, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(0, sstable.getFilterTracker().getFalsePositiveCount()); // non-existing key is rejected by index interval check sstable.getPosition(k(14), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getBloomFilterTruePositiveCount()); - assertEquals(1, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(2, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(2, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(1, sstable.getFilterTracker().getFalsePositiveCount()); // non-existing key is rejected by index lookup check sstable.getPosition(k(807), SSTableReader.Operator.EQ); - assertEquals(2, sstable.getBloomFilterTruePositiveCount()); - assertEquals(1, sstable.getBloomFilterTrueNegativeCount()); - assertEquals(3, sstable.getBloomFilterFalsePositiveCount()); + assertEquals(2, sstable.getFilterTracker().getTruePositiveCount()); + assertEquals(1, sstable.getFilterTracker().getTrueNegativeCount()); + assertEquals(2, sstable.getFilterTracker().getFalsePositiveCount()); } @Test @@ -458,92 +486,130 @@ public class SSTableReaderTest Descriptor desc = sstable.descriptor; // test to see if sstable can be opened as expected - SSTableReader target = SSTableReader.open(desc); - assert target.first.equals(firstKey); - assert target.last.equals(lastKey); + SSTableReader target = SSTableReader.open(store, desc); + try + { + assert target.first.equals(firstKey); + assert target.last.equals(lastKey); + } + finally + { + target.selfRef().close(); + } + if (BigFormat.isDefault()) + checkOpenedBigTable(ks, cf, store, desc); + else + throw Util.testMustBeImplementedForSSTableFormat(); + } + + private static void checkOpenedBigTable(String ks, String cf, ColumnFamilyStore store, Descriptor desc) throws Exception + { executeInternal(format("ALTER TABLE \"%s\".\"%s\" WITH bloom_filter_fp_chance = 0.3", ks, cf)); - File summaryFile = new File(desc.filenameFor(Component.SUMMARY)); - Path bloomPath = new File(desc.filenameFor(Component.FILTER)).toPath(); - Path summaryPath = summaryFile.toPath(); + File bloomFile = desc.fileFor(Components.FILTER); + long bloomModified = bloomFile.lastModified(); - long bloomModified = Files.getLastModifiedTime(bloomPath).toMillis(); - long summaryModified = Files.getLastModifiedTime(summaryPath).toMillis(); + File summaryFile = desc.fileFor(Components.SUMMARY); + long summaryModified = summaryFile.lastModified(); TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different // Offline tests // check that bloomfilter/summary ARE NOT regenerated - target = SSTableReader.openNoValidation(desc, store.metadata); - - assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis()); - assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis()); - - target.selfRef().release(); + SSTableReader target = SSTableReader.openNoValidation(store, desc, store.metadata); + try + { + assertEquals(bloomModified, bloomFile.lastModified()); + assertEquals(summaryModified, summaryFile.lastModified()); + } + finally + { + target.selfRef().close(); + } // check that bloomfilter/summary ARE NOT regenerated and BF=AlwaysPresent when filter component is missing - Set components = SSTable.discoverComponentsFor(desc); - components.remove(Component.FILTER); + Set components = desc.discoverComponents(); + components.remove(Components.FILTER); target = SSTableReader.openNoValidation(desc, components, store); - - assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis()); - assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis()); - assertEquals(FilterFactory.AlwaysPresent, target.getBloomFilter()); - - target.selfRef().release(); + try + { + assertEquals(bloomModified, bloomFile.lastModified()); + assertEquals(summaryModified, summaryFile.lastModified()); + assertEquals(0, ((SSTableReaderWithFilter) target).getFilterOffHeapSize()); + } + finally + { + target.selfRef().close(); + } // #### online tests #### // check that summary & bloomfilter are not regenerated when SSTable is opened and BFFP has been changed - target = SSTableReader.open(desc, store.metadata); - - assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis()); - assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis()); - - target.selfRef().release(); + target = SSTableReader.open(store, desc, store.metadata); + try + { + assertEquals(bloomModified, bloomFile.lastModified()); + assertEquals(summaryModified, summaryFile.lastModified()); + } + finally + { + target.selfRef().close(); + } // check that bloomfilter is recreated when it doesn't exist and this causes the summary to be recreated - components = SSTable.discoverComponentsFor(desc); - components.remove(Component.FILTER); + components = desc.discoverComponents(); + components.remove(Components.FILTER); + components.remove(Components.SUMMARY); - target = SSTableReader.open(desc, components, store.metadata); - - assertTrue("Bloomfilter was not recreated", bloomModified < Files.getLastModifiedTime(bloomPath).toMillis()); - assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis()); - - target.selfRef().release(); + target = SSTableReader.open(store, desc, components, store.metadata); + try { + assertTrue("Bloomfilter was not recreated", bloomModified < bloomFile.lastModified()); + assertTrue("Summary was not recreated", summaryModified < summaryFile.lastModified()); + } + finally + { + target.selfRef().close(); + } // check that only the summary is regenerated when it is deleted - components.add(Component.FILTER); - summaryModified = Files.getLastModifiedTime(summaryPath).toMillis(); + components.add(Components.FILTER); + summaryModified = summaryFile.lastModified(); summaryFile.tryDelete(); TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different - bloomModified = Files.getLastModifiedTime(bloomPath).toMillis(); + bloomModified = bloomFile.lastModified(); - target = SSTableReader.open(desc, components, store.metadata); - - assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis()); - assertTrue("Summary was not recreated", summaryModified < Files.getLastModifiedTime(summaryPath).toMillis()); - - target.selfRef().release(); + target = SSTableReader.open(store, desc, components, store.metadata); + try + { + assertEquals(bloomModified, bloomFile.lastModified()); + assertTrue("Summary was not recreated", summaryModified < summaryFile.lastModified()); + } + finally + { + target.selfRef().close(); + } // check that summary and bloomfilter is not recreated when the INDEX is missing - components.add(Component.SUMMARY); - components.remove(Component.PRIMARY_INDEX); + components.add(Components.SUMMARY); + components.remove(Components.PRIMARY_INDEX); - summaryModified = Files.getLastModifiedTime(summaryPath).toMillis(); - target = SSTableReader.open(desc, components, store.metadata, false, false); - - TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different - assertEquals(bloomModified, Files.getLastModifiedTime(bloomPath).toMillis()); - assertEquals(summaryModified, Files.getLastModifiedTime(summaryPath).toMillis()); - - target.selfRef().release(); + summaryModified = summaryFile.lastModified(); + target = SSTableReader.open(store, desc, components, store.metadata, false, false); + try + { + TimeUnit.MILLISECONDS.sleep(1000); // sleep to ensure modified time will be different + assertEquals(bloomModified, bloomFile.lastModified()); + assertEquals(summaryModified, summaryFile.lastModified()); + } + finally + { + target.selfRef().close(); + } } @Test - public void testLoadingSummaryUsesCorrectPartitioner() + public void testLoadingSummaryUsesCorrectPartitioner() throws Exception { ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_INDEXED); @@ -555,20 +621,25 @@ public class SSTableReaderTest Util.flush(store); - for(ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores()) + for (ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores()) { assert indexCfs.isIndex(); SSTableReader sstable = indexCfs.getLiveSSTables().iterator().next(); assert sstable.first.getToken() instanceof LocalToken; - SSTableReader.saveSummary(sstable.descriptor, sstable.first, sstable.last, sstable.indexSummary); - SSTableReader reopened = SSTableReader.open(sstable.descriptor); - assert reopened.first.getToken() instanceof LocalToken; - reopened.selfRef().release(); + if (sstable instanceof IndexSummarySupport) + { + new IndexSummaryComponent(((IndexSummarySupport) sstable).getIndexSummary(), sstable.first, sstable.last).save(sstable.descriptor.fileFor(Components.SUMMARY), true); + SSTableReader reopened = SSTableReader.open(store, sstable.descriptor); + assert reopened.first.getToken() instanceof LocalToken; + reopened.selfRef().release(); + } } } - /** see CASSANDRA-5407 */ + /** + * see CASSANDRA-5407 + */ @Test public void testGetScannerForNoIntersectingRanges() throws Exception { @@ -583,11 +654,15 @@ public class SSTableReaderTest Util.flush(store); boolean foundScanner = false; - for (SSTableReader s : store.getLiveSSTables()) + + Set liveSSTables = store.getLiveSSTables(); + assertEquals("The table should have only one sstable", 1, liveSSTables.size()); + + for (SSTableReader s : liveSSTables) { - try (ISSTableScanner scanner = s.getScanner(new Range(t(0), t(1)))) + try (ISSTableScanner scanner = s.getScanner(new Range<>(t(0), t(1)))) { - scanner.next(); // throws exception pre 5407 + Assertions.assertThat(scanner).isInstanceOf(EmptySSTableScanner.class); foundScanner = true; } } @@ -623,13 +698,13 @@ public class SSTableReaderTest SSTableReader sstable = store.getLiveSSTables().iterator().next(); List sections = sstable.getPositionsForRanges(ranges); - assert sections.size() == 1 : "Expected to find range in sstable" ; + assert sections.size() == 1 : "Expected to find range in sstable"; // re-open the same sstable as it would be during bulk loading - Set components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX); - if (sstable.components.contains(Component.COMPRESSION_INFO)) - components.add(Component.COMPRESSION_INFO); - SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components, store.metadata); + Set components = Sets.newHashSet(sstable.descriptor.getFormat().primaryComponents()); + if (sstable.components.contains(Components.COMPRESSION_INFO)) + components.add(Components.COMPRESSION_INFO); + SSTableReader bulkLoaded = SSTableReader.openForBatch(store, sstable.descriptor, components, store.metadata); sections = bulkLoaded.getPositionsForRanges(ranges); assert sections.size() == 1 : "Expected to find range in sstable opened for bulk loading"; bulkLoaded.selfRef().release(); @@ -638,6 +713,7 @@ public class SSTableReaderTest @Test public void testIndexSummaryReplacement() throws IOException, ExecutionException, InterruptedException { + assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD_LOW_INDEX_INTERVAL); // index interval of 8, no key caching final int NUM_PARTITIONS = 512; @@ -648,7 +724,6 @@ public class SSTableReaderTest .add("val", format("%3d", j)) .build() .applyUnsafe(); - } Util.flush(store); CompactionManager.instance.performMaximal(store, false); @@ -678,7 +753,7 @@ public class SSTableReaderTest public void run() { Iterable results = store.keySamples( - new Range<>(sstable.getPartitioner().getMinimumToken(), sstable.getPartitioner().getToken(key))); + new Range<>(sstable.getPartitioner().getMinimumToken(), sstable.getPartitioner().getToken(key))); assertTrue(results.iterator().hasNext()); } })); @@ -687,7 +762,7 @@ public class SSTableReaderTest SSTableReader replacement; try (LifecycleTransaction txn = store.getTracker().tryModify(Collections.singletonList(sstable), OperationType.UNKNOWN)) { - replacement = sstable.cloneWithNewSummarySamplingLevel(store, 1); + replacement = ((IndexSummarySupport) sstable).cloneWithNewSummarySamplingLevel(store, 1); txn.update(replacement, true); txn.finish(); } @@ -700,6 +775,7 @@ public class SSTableReaderTest @Test public void testIndexSummaryUpsampleAndReload() throws Exception { + assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE; MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments @@ -713,7 +789,7 @@ public class SSTableReaderTest } } - private void testIndexSummaryUpsampleAndReload0() throws Exception + private > void testIndexSummaryUpsampleAndReload0() throws Exception { ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD_LOW_INDEX_INTERVAL); // index interval of 8, no key caching @@ -725,23 +801,22 @@ public class SSTableReaderTest .add("val", format("%3d", j)) .build() .applyUnsafe(); - } Util.flush(store); CompactionManager.instance.performMaximal(store, false); - Collection sstables = store.getLiveSSTables(); + Collection sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(store); assert sstables.size() == 1; - final SSTableReader sstable = sstables.iterator().next(); + final R sstable = sstables.iterator().next(); try (LifecycleTransaction txn = store.getTracker().tryModify(Collections.singletonList(sstable), OperationType.UNKNOWN)) { - SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(store, sstable.getIndexSummarySamplingLevel() + 1); + SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(store, sstable.getIndexSummary().getSamplingLevel() + 1); txn.update(replacement, true); txn.finish(); } - SSTableReader reopen = SSTableReader.open(sstable.descriptor); - assert reopen.getIndexSummarySamplingLevel() == sstable.getIndexSummarySamplingLevel() + 1; + R reopen = (R) SSTableReader.open(store, sstable.descriptor); + assert reopen.getIndexSummary().getSamplingLevel() == sstable.getIndexSummary().getSamplingLevel() + 1; } private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS) @@ -811,19 +886,19 @@ public class SSTableReaderTest // make sure the new directory is empty and that the old files exist: for (Component c : sstable.components) { - File f = new File(notLiveDesc.filenameFor(c)); + File f = notLiveDesc.fileFor(c); assertFalse(f.exists()); - assertTrue(new File(sstable.descriptor.filenameFor(c)).exists()); + assertTrue(sstable.descriptor.fileFor(c).exists()); } SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components, false); // make sure the files were moved: for (Component c : sstable.components) { - File f = new File(notLiveDesc.filenameFor(c)); + File f = notLiveDesc.fileFor(c); assertTrue(f.exists()); assertTrue(f.toString().contains(format("-%s-", id))); f.deleteOnExit(); - assertFalse(new File(sstable.descriptor.filenameFor(c)).exists()); + assertFalse(sstable.descriptor.fileFor(c).exists()); } } @@ -867,40 +942,40 @@ public class SSTableReaderTest Descriptor desc = setUpForTestVerfiyCompressionInfoExistence(); // delete the compression info, so it is corrupted. - File compressionInfoFile = new File(desc.filenameFor(Component.COMPRESSION_INFO)); + File compressionInfoFile = desc.fileFor(Components.COMPRESSION_INFO); compressionInfoFile.tryDelete(); assertFalse("CompressionInfo file should not exist", compressionInfoFile.exists()); // discovert the components on disk after deletion - Set components = SSTable.discoverComponentsFor(desc); + Set components = desc.discoverComponents(); expectedException.expect(CorruptSSTableException.class); expectedException.expectMessage("CompressionInfo.db"); - SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components); + CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components); } @Test public void testVerifyCompressionInfoExistenceWhenTOCUnableToOpen() { Descriptor desc = setUpForTestVerfiyCompressionInfoExistence(); - Set components = SSTable.discoverComponentsFor(desc); - SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components); + Set components = desc.discoverComponents(); + CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components); // mark the toc file not readable in order to trigger the FSReadError - File tocFile = new File(desc.filenameFor(Component.TOC)); + File tocFile = desc.fileFor(Components.TOC); tocFile.trySetReadable(false); expectedException.expect(FSReadError.class); expectedException.expectMessage("TOC.txt"); - SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components); + CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components); } @Test public void testVerifyCompressionInfoExistencePasses() { Descriptor desc = setUpForTestVerfiyCompressionInfoExistence(); - Set components = SSTable.discoverComponentsFor(desc); - SSTableReader.verifyCompressionInfoExistenceIfApplicable(desc, components); + Set components = desc.discoverComponents(); + CompressionInfoComponent.verifyCompressionInfoExistenceIfApplicable(desc, components); } private Descriptor setUpForTestVerfiyCompressionInfoExistence() @@ -911,8 +986,8 @@ public class SSTableReaderTest cfs.clearUnsafe(); Descriptor desc = sstable.descriptor; - File compressionInfoFile = new File(desc.filenameFor(Component.COMPRESSION_INFO)); - File tocFile = new File(desc.filenameFor(Component.TOC)); + File compressionInfoFile = desc.fileFor(Components.COMPRESSION_INFO); + File tocFile = desc.fileFor(Components.TOC); assertTrue("CompressionInfo file should exist", compressionInfoFile.exists()); assertTrue("TOC file should exist", tocFile.exists()); return desc; diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 363a000186..fe3ff271ca 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -19,7 +19,12 @@ package org.apache.cassandra.io.sstable; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @@ -27,8 +32,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import org.junit.Test; -import org.apache.cassandra.Util; import org.apache.cassandra.UpdateBuilder; +import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -36,24 +41,24 @@ import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.lifecycle.View; -import org.apache.cassandra.db.rows.EncodingStats; -import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; import org.apache.cassandra.db.compaction.CompactionController; import org.apache.cassandra.db.compaction.CompactionIterator; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.SSTableSplitter; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; import org.apache.cassandra.db.partitions.ImmutableBTreePartition; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -62,7 +67,10 @@ import static java.util.Collections.singletonList; import static org.apache.cassandra.db.compaction.OperationType.COMPACTION; import static org.apache.cassandra.utils.FBUtilities.nowInSeconds; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -884,39 +892,18 @@ public class SSTableRewriterTest extends SSTableWriterTestBase try { UnfilteredRowIterator uri = mock(UnfilteredRowIterator.class); - when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0,0)); + when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0, 0)); when(uri.partitionKey()).thenReturn(bopKeyFromInt(0)); // should not be able to append after buffer release on switch firstWriter.append(uri); fail("Expected AssertionError was not thrown."); } - catch(AssertionError ae) { - if (!ae.getMessage().contains("update is being called after releaseBuffers")) + catch (AssertionError ae) + { + if (!ae.getMessage().contains("update is being called after releaseBuffers") && !ae.getMessage().contains("Attempt to use a closed data output")) throw ae; } } - - // Can update a writer that is not eagerly cleared on switch - eagerWriterMetaRelease = false; - try (LifecycleTransaction txn = cfs.getTracker().tryModify(new HashSet<>(), OperationType.UNKNOWN); - SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, 1000000, false, eagerWriterMetaRelease) - ) - { - SSTableWriter firstWriter = getWriter(cfs, dir, txn); - rewriter.switchWriter(firstWriter); - - // At least one write so it's not aborted when switched out. - UnfilteredRowIterator uri = mock(UnfilteredRowIterator.class); - when(uri.partitionLevelDeletion()).thenReturn(new DeletionTime(0,0)); - when(uri.partitionKey()).thenReturn(bopKeyFromInt(0)); - rewriter.append(uri); - - rewriter.switchWriter(getWriter(cfs, dir, txn)); - - // should be able to append after switch, and assert is not tripped - when(uri.partitionKey()).thenReturn(bopKeyFromInt(1)); - firstWriter.append(uri); - } } static DecoratedKey bopKeyFromInt(int i) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java index 922200ab3b..c5c4ca6384 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableScannerTest.java @@ -24,14 +24,11 @@ import java.util.Collection; import java.util.List; import com.google.common.collect.Iterables; - import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; @@ -41,13 +38,14 @@ import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import static org.apache.cassandra.dht.AbstractBounds.isEmpty; diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index cdd3ee00e0..2ebbe3ffdd 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -19,23 +19,36 @@ package org.apache.cassandra.io.sstable; -import org.apache.cassandra.io.util.File; import java.io.IOException; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.stream.Stream; -import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableReader; - import org.apache.cassandra.Util; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; -import static org.apache.cassandra.service.ActiveRepairService.*; +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; import static org.junit.Assert.assertEquals; public class SSTableUtils @@ -80,7 +93,7 @@ public class SSTableUtils File cfDir = new File(tempdir, keyspaceName + File.pathSeparator() + cfname); cfDir.tryCreateDirectories(); cfDir.deleteOnExit(); - File datafile = new File(new Descriptor(cfDir, keyspaceName, cfname, id, SSTableFormat.Type.BIG).filenameFor(Component.DATA)); + File datafile = new Descriptor(cfDir, keyspaceName, cfname, id, BigFormat.getInstance().getType()).fileFor(Components.DATA); if (!datafile.createFileIfNotExists()) throw new IOException("unable to create file " + datafile); datafile.deleteOnExit(); @@ -216,11 +229,11 @@ public class SSTableUtils public Collection write(int expectedSize, Appender appender) throws IOException { - File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : new File(dest.filenameFor(Component.DATA)); + File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : dest.fileFor(Components.DATA); TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname); ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); SerializationHeader header = appender.header(); - SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFilename(datafile.absolutePath()), expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header); + SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, 0, header); while (appender.append(writer)) { /* pass */ } Collection readers = writer.finish(true); @@ -228,7 +241,7 @@ public class SSTableUtils if (cleanup) for (SSTableReader reader: readers) for (Component component : reader.components) - new File(reader.descriptor.filenameFor(component)).deleteOnExit(); + reader.descriptor.fileFor(component).deleteOnExit(); return readers; } } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java index 5d20367663..ad6bd61f57 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTest.java @@ -20,27 +20,29 @@ package org.apache.cassandra.io.sstable; import java.nio.ByteBuffer; -import org.apache.cassandra.io.util.File; import org.junit.Test; -import org.apache.cassandra.*; -import org.apache.cassandra.db.*; +import org.apache.cassandra.UpdateBuilder; +import org.apache.cassandra.Util; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.filter.*; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.TimeUUID; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SSTableWriterTest extends SSTableWriterTestBase { @@ -63,34 +65,38 @@ public class SSTableWriterTest extends SSTableWriterTestBase writer.append(builder.build().unfilteredIterator()); } - SSTableReader s = writer.setMaxDataAge(1000).openEarly(); - assert s != null; - assertFileCounts(dir.tryListNames()); - for (int i = 10000; i < 20000; i++) - { - UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1); - for (int j = 0; j < 100; j++) - builder.newRow("" + j).add("val", ByteBuffer.allocate(1000)); - writer.append(builder.build().unfilteredIterator()); - } - SSTableReader s2 = writer.setMaxDataAge(1000).openEarly(); - assertTrue(s.last.compareTo(s2.last) < 0); - assertFileCounts(dir.tryListNames()); - s.selfRef().release(); - s2.selfRef().release(); + writer.setMaxDataAge(1000); + writer.openEarly(s -> { + assert s != null; + assertFileCounts(dir.tryListNames()); + for (int i = 10000; i < 20000; i++) + { + UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), random(i, 10)).withTimestamp(1); + for (int j = 0; j < 100; j++) + builder.newRow("" + j).add("val", ByteBuffer.allocate(1000)); + writer.append(builder.build().unfilteredIterator()); + } + writer.setMaxDataAge(1000); + writer.openEarly(s2 -> { + assertTrue(s.last.compareTo(s2.last) < 0); + assertFileCounts(dir.tryListNames()); + s.selfRef().release(); + s2.selfRef().release(); - int datafiles = assertFileCounts(dir.tryListNames()); - assertEquals(datafiles, 1); + int datafiles = assertFileCounts(dir.tryListNames()); + assertEquals(datafiles, 1); - LifecycleTransaction.waitForDeletions(); - assertFileCounts(dir.tryListNames()); + LifecycleTransaction.waitForDeletions(); + assertFileCounts(dir.tryListNames()); - writer.abort(); - txn.abort(); - LifecycleTransaction.waitForDeletions(); - datafiles = assertFileCounts(dir.tryListNames()); - assertEquals(datafiles, 0); - validateCFS(cfs); + writer.abort(); + txn.abort(); + LifecycleTransaction.waitForDeletions(); + datafiles = assertFileCounts(dir.tryListNames()); + assertEquals(datafiles, 0); + validateCFS(cfs); + }); + }); } } @@ -293,4 +299,4 @@ public class SSTableWriterTest extends SSTableWriterTestBase assertInvalidRepairMetadata(1, NO_PENDING_REPAIR, true); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java index 83ad136137..91b44ed27e 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java @@ -25,7 +25,6 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.Uninterruptibles; -import org.apache.cassandra.io.util.File; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -42,6 +41,8 @@ import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.TimeUUID; @@ -139,7 +140,7 @@ public class SSTableWriterTestBase extends SchemaLoader { if (f.name().contains("Data")) { - Descriptor d = Descriptor.fromFilename(f.absolutePath()); + Descriptor d = Descriptor.fromFileWithComponent(f, false).left; assertTrue(d.toString(), liveDescriptors.contains(d.id)); } } @@ -155,7 +156,17 @@ public class SSTableWriterTestBase extends SchemaLoader public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn, long repairedAt, TimeUUID pendingRepair, boolean isTransient) { Descriptor desc = cfs.newSSTableDescriptor(directory); - return SSTableWriter.create(desc, 0, repairedAt, pendingRepair, isTransient, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS), cfs.indexManager.listIndexes(), txn); + return desc.getFormat().getWriterFactory().builder(desc) + .setTableMetadataRef(cfs.metadata) + .setKeyCount(0) + .setRepairedAt(repairedAt) + .setPendingRepair(pendingRepair) + .setTransientSSTable(isTransient) + .setSerializationHeader(new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS)) + .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType()) + .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) + .addDefaultComponents() + .build(txn, cfs); } public static SSTableWriter getWriter(ColumnFamilyStore cfs, File directory, LifecycleTransaction txn) diff --git a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java similarity index 78% rename from test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java rename to test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java index 14d3c5ef6f..e65c410ae7 100644 --- a/test/unit/org/apache/cassandra/io/sstable/BigTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java @@ -18,23 +18,26 @@ */ package org.apache.cassandra.io.sstable; -import org.apache.cassandra.io.util.File; import java.io.IOException; - -import org.junit.BeforeClass; +import java.util.Collection; import org.junit.Assert; +import org.junit.BeforeClass; + import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; -import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest; -public class BigTableWriterTest extends AbstractTransactionalTest +public class SSTableWriterTransactionTest extends AbstractTransactionalTest { public static final String KEYSPACE1 = "BigTableWriterTest"; public static final String CF_STANDARD = "Standard1"; @@ -78,7 +81,7 @@ public class BigTableWriterTest extends AbstractTransactionalTest private TestableBTW(Descriptor desc, SSTableTxnWriter sw) { super(sw); - this.file = new File(desc.filenameFor(Component.DATA)); + this.file = desc.fileFor(Components.DATA); this.descriptor = desc; this.writer = sw; @@ -93,19 +96,21 @@ public class BigTableWriterTest extends AbstractTransactionalTest protected void assertInProgress() throws Exception { - assertExists(Component.DATA, Component.PRIMARY_INDEX); - assertNotExists(Component.FILTER, Component.SUMMARY); + assertExists(descriptor.formatType.info.primaryComponents()); + assertNotExists(descriptor.formatType.info.generatedOnLoadComponents()); Assert.assertTrue(file.length() > 0); } protected void assertPrepared() throws Exception { - assertExists(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.SUMMARY); + assertExists(descriptor.formatType.info.primaryComponents()); + assertExists(descriptor.formatType.info.generatedOnLoadComponents()); } protected void assertAborted() throws Exception { - assertNotExists(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.SUMMARY); + assertNotExists(descriptor.formatType.info.primaryComponents()); + assertNotExists(descriptor.formatType.info.generatedOnLoadComponents()); Assert.assertFalse(file.exists()); } @@ -120,17 +125,16 @@ public class BigTableWriterTest extends AbstractTransactionalTest return true; } - private void assertExists(Component ... components) + private void assertExists(Collection components) { for (Component component : components) - Assert.assertTrue(new File(descriptor.filenameFor(component)).exists()); + Assert.assertTrue(descriptor.fileFor(component).exists()); } - private void assertNotExists(Component ... components) + private void assertNotExists(Collection components) { for (Component component : components) - Assert.assertFalse(component.toString(), new File(descriptor.filenameFor(component)).exists()); + Assert.assertFalse(component.toString(), descriptor.fileFor(component).exists()); } } - } diff --git a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java similarity index 86% rename from test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java rename to test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java index e6d20207df..11b92ac499 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/big/BigTableZeroCopyWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableZeroCopyWriterTest.java @@ -16,20 +16,16 @@ * limitations under the License. */ -package org.apache.cassandra.io.sstable.format.big; +package org.apache.cassandra.io.sstable; -import java.io.ByteArrayInputStream; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.Collection; +import java.util.HashSet; import java.util.Set; import java.util.function.Function; -import com.google.common.collect.ImmutableSet; -import org.apache.cassandra.io.util.File; import org.junit.BeforeClass; import org.junit.Test; @@ -46,11 +42,11 @@ import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.rows.UnfilteredRowIterator; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.net.AsyncStreamingInputPlus; import org.apache.cassandra.schema.CachingParams; @@ -60,11 +56,10 @@ import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -public class BigTableZeroCopyWriterTest +public class SSTableZeroCopyWriterTest { public static final String KEYSPACE1 = "BigTableBlockWriterTest"; public static final String CF_STANDARD = "Standard1"; @@ -126,7 +121,7 @@ public class BigTableZeroCopyWriterTest @Test public void writeDataFile_DataInputPlus() { - writeDataTestCycle(buffer -> new DataInputStreamPlus(new ByteArrayInputStream(buffer.array()))); + writeDataTestCycle(buffer -> new DataInputBuffer(buffer.array())); } @Test @@ -136,7 +131,9 @@ public class BigTableZeroCopyWriterTest { writeDataTestCycle(buffer -> { - input.append(Unpooled.wrappedBuffer(buffer)); + if (buffer.limit() > 0) { // skip empty files that would cause premature EOF + input.append(Unpooled.wrappedBuffer(buffer)); + } return input; }); @@ -152,14 +149,20 @@ public class BigTableZeroCopyWriterTest TableMetadataRef metadata = Schema.instance.getTableMetadataRef(desc); LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.STREAM); - Set componentsToWrite = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, - Component.STATS); + Set componentsToWrite = new HashSet<>(desc.getFormat().uploadComponents()); + if (!metadata.getLocal().params.compression.isEnabled()) + componentsToWrite.remove(Components.COMPRESSION_INFO); - BigTableZeroCopyWriter btzcw = new BigTableZeroCopyWriter(desc, metadata, txn, componentsToWrite); + SSTableZeroCopyWriter btzcw = desc.getFormat() + .getWriterFactory() + .builder(desc) + .setComponents(componentsToWrite) + .setTableMetadataRef(metadata) + .createZeroCopyWriter(txn, store); for (Component component : componentsToWrite) { - if (Files.exists(Paths.get(desc.filenameFor(component)))) + if (desc.fileFor(component).exists()) { Pair pair = getSSTableComponentData(sstable, component, bufferMapper); @@ -208,7 +211,7 @@ public class BigTableZeroCopyWriterTest private Pair getSSTableComponentData(SSTableReader sstable, Component component, Function bufferMapper) { - FileHandle componentFile = new FileHandle.Builder(sstable.descriptor.filenameFor(component)) + FileHandle componentFile = new FileHandle.Builder(sstable.descriptor.fileFor(component)) .bufferSize(1024).complete(); ByteBuffer buffer = ByteBuffer.allocate((int) componentFile.channel.size()); componentFile.channel.read(buffer, 0); diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java similarity index 63% rename from test/unit/org/apache/cassandra/db/ScrubTest.java rename to test/unit/org/apache/cassandra/io/sstable/ScrubTest.java index dfe916556b..ce3b2c2afa 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java @@ -7,42 +7,49 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable; import java.io.IOError; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.SortedSet; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.cassandra.io.util.File; -import org.apache.commons.lang3.StringUtils; - +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.commons.lang3.ArrayUtils; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import net.openhft.chronicle.core.util.ThrowingBiConsumer; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; @@ -51,9 +58,17 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.CounterMutation; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.compaction.Scrubber; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.marshal.Int32Type; @@ -68,22 +83,20 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.compress.CompressionMetadata; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.SSTableMultiWriter; -import org.apache.cassandra.io.sstable.SSTableRewriter; -import org.apache.cassandra.io.sstable.SSTableTxnWriter; -import org.apache.cassandra.io.sstable.SimpleSSTableMultiWriter; +import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.io.sstable.format.big.BigTableWriter; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.OutputHandler; +import org.apache.cassandra.utils.Throwables; +import org.jboss.byteman.contrib.bmunit.BMRule; +import org.jboss.byteman.contrib.bmunit.BMUnitRunner; import static org.apache.cassandra.SchemaLoader.counterCFMD; import static org.apache.cassandra.SchemaLoader.createKeyspace; @@ -93,12 +106,16 @@ import static org.apache.cassandra.SchemaLoader.standardCFMD; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; +import static org.junit.Assume.assumeFalse; +@RunWith(BMUnitRunner.class) public class ScrubTest { + private final static Logger logger = LoggerFactory.getLogger(ScrubTest.class); + public static final String INVALID_LEGACY_SSTABLE_ROOT_PROP = "invalid-legacy-sstable-root"; public static final String CF = "Standard1"; @@ -121,7 +138,10 @@ public class ScrubTest @BeforeClass public static void defineSchema() throws ConfigurationException { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setFileCacheEnabled(false); loadSchema(); + assertNull(ChunkCache.instance); } @Before @@ -150,7 +170,7 @@ public class ScrubTest } @Test - public void testScrubOnePartition() throws ExecutionException, InterruptedException + public void testScrubOnePartition() { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); @@ -159,14 +179,14 @@ public class ScrubTest fillCF(cfs, 1); assertOrderedAll(cfs, 1); - CompactionManager.instance.performScrub(cfs, false, true, false, 2); + performScrub(cfs, false, true, false, 2); // check data is still there assertOrderedAll(cfs, 1); } @Test - public void testScrubLastBrokenPartition() throws ExecutionException, InterruptedException, IOException + public void testScrubLastBrokenPartition() throws IOException { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ksName, CF); @@ -179,9 +199,8 @@ public class ScrubTest assertThat(liveSSTables).hasSize(1); String fileName = liveSSTables.iterator().next().getFilename(); Files.write(Paths.get(fileName), new byte[10], StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - ChunkCache.instance.invalidateFile(fileName); - CompactionManager.instance.performScrub(cfs, true, true, false, 2); + performScrub(cfs, true, true, false, 2); // check data is still there assertOrderedAll(cfs, 0); @@ -206,95 +225,165 @@ public class ScrubTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); //make sure to override at most 1 chunk when compression is enabled - overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1")); + //use 0x00 instead of the usual 0x7A because if by any chance it's able to iterate over the corrupt + //section, then we get many out-of-order errors, which we don't want + overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x00); // with skipCorrupted == false, the scrub is expected to fail try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB); - Scrubber scrubber = new Scrubber(cfs, txn, false, true)) + IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build())) { scrubber.scrub(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (IOError err) { - assertTrue(err.getCause() instanceof CorruptSSTableException); + catch (IOError err) + { + Throwables.assertAnyCause(err, CorruptSSTableException.class); } // with skipCorrupted == true, the corrupt rows will be skipped - Scrubber.ScrubResult scrubResult; + IScrubber.ScrubResult scrubResult; try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB); - Scrubber scrubber = new Scrubber(cfs, txn, true, true)) + IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().skipCorrupted().checkData().build())) { scrubResult = scrubber.scrubWithResult(); } assertNotNull(scrubResult); - boolean compression = Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")); + boolean compression = sstable.compression; assertEquals(0, scrubResult.emptyPartitions); if (compression) { assertEquals(numPartitions, scrubResult.badPartitions + scrubResult.goodPartitions); - //because we only corrupted 1 chunk and we chose enough partitions to cover at least 3 chunks + //because we only corrupted 1 chunk, and we chose enough partitions to cover at least 3 chunks assertTrue(scrubResult.goodPartitions >= scrubResult.badPartitions * 2); } else { assertEquals(1, scrubResult.badPartitions); - assertEquals(numPartitions-1, scrubResult.goodPartitions); + assertEquals(numPartitions - 1, scrubResult.goodPartitions); } assertEquals(1, cfs.getLiveSSTables().size()); assertOrderedAll(cfs, scrubResult.goodPartitions); } - @Test - public void testScrubCorruptedRowInSmallFile() throws IOException, WriteTimeoutException + private File sstableIndexPath(SSTableReader reader) { - // cannot test this with compression - assumeTrue(!Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))); + if (reader.descriptor.getFormat() == BigFormat.getInstance()) + return reader.descriptor.fileFor(Components.PRIMARY_INDEX); + else + throw Util.testMustBeImplementedForSSTableFormat(); + } + @Test + public void testScrubCorruptedRowInSmallFile() throws Throwable + { + // overwrite one row with garbage + testCorruptionInSmallFile((sstable, keys) -> + overrideWithGarbage(sstable, + ByteBufferUtil.bytes(keys[0]), + ByteBufferUtil.bytes(keys[1]), + (byte) 0x7A), + false, + 4); + } + + + @Test + public void testScrubCorruptedIndex() throws Throwable + { + // overwrite a part of the index with garbage + testCorruptionInSmallFile((sstable, keys) -> + overrideWithGarbage(sstableIndexPath(sstable), + 5, + 6, + (byte) 0x7A), + true, + 5); + } + + @Test + public void testScrubCorruptedIndexOnOpen() throws Throwable + { + // overwrite the whole index with garbage + testCorruptionInSmallFile((sstable, keys) -> + overrideWithGarbage(sstableIndexPath(sstable), + 0, + 60, + (byte) 0x7A), + true, + 5); + } + + @Test + public void testScrubCorruptedRowCorruptedIndex() throws Throwable + { + // overwrite one row, and the index with garbage + testCorruptionInSmallFile((sstable, keys) -> + { + overrideWithGarbage(sstable, + ByteBufferUtil.bytes(keys[2]), + ByteBufferUtil.bytes(keys[3]), + (byte) 0x7A); + overrideWithGarbage(sstableIndexPath(sstable), + 5, + 6, + (byte) 0x7A); + }, + false, + 2); // corrupt after the second partition, no way to resync + } + + public void testCorruptionInSmallFile(ThrowingBiConsumer corrupt, boolean isFullyRecoverable, int expectedPartitions) throws IOException, WriteTimeoutException + { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF); + cfs.clearUnsafe(); - fillCounterCF(cfs, 2); + String[] keys = fillCounterCF(cfs, 5); - assertOrderedAll(cfs, 2); + assertOrderedAll(cfs, 5); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); + // cannot test this with compression + assumeFalse(sstable.metadata().params.compression.isEnabled()); + // overwrite one row with garbage - overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1")); + corrupt.accept(sstable, keys); // with skipCorrupted == false, the scrub is expected to fail - try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB); - Scrubber scrubber = new Scrubber(cfs, txn, false, true)) + if (!isFullyRecoverable) { - // with skipCorrupted == true, the corrupt row will be skipped - scrubber.scrub(); - fail("Expected a CorruptSSTableException to be thrown"); - } - catch (IOError err) { - assertTrue(err.getCause() instanceof CorruptSSTableException); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB); + IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build())) + { + // with skipCorrupted == true, the corrupt row will be skipped + scrubber.scrub(); + fail("Expected a CorruptSSTableException to be thrown"); + } + catch (IOError expected) + { + } } try (LifecycleTransaction txn = cfs.getTracker().tryModify(Collections.singletonList(sstable), OperationType.SCRUB); - Scrubber scrubber = new Scrubber(cfs, txn, true, true)) + IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, txn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().skipCorrupted().build())) { // with skipCorrupted == true, the corrupt row will be skipped scrubber.scrub(); } assertEquals(1, cfs.getLiveSSTables().size()); - // verify that we can read all of the rows, and there is now one less row - assertOrderedAll(cfs, 1); + // verify that we can read all the rows, and there is now the expected number of rows + assertOrderedAll(cfs, expectedPartitions); } @Test - public void testScrubOneRowWithCorruptedKey() throws IOException, ExecutionException, InterruptedException, ConfigurationException + public void testScrubOneRowWithCorruptedKey() throws IOException, ConfigurationException { - // cannot test this with compression - assumeTrue(!Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"))); - CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); @@ -303,9 +392,12 @@ public class ScrubTest assertOrderedAll(cfs, 4); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - overrideWithGarbage(sstable, 0, 2); + // cannot test this with compression + assumeFalse(sstable.metadata().params.compression.isEnabled()); - CompactionManager.instance.performScrub(cfs, false, true, 2); + overrideWithGarbage(sstable, 0, 2, (byte) 0x7A); + + performScrub(cfs, false, true, false, 2); // check data is still there assertOrderedAll(cfs, 4); @@ -327,7 +419,7 @@ public class ScrubTest } @Test - public void testScrubMultiRow() throws ExecutionException, InterruptedException + public void testScrubMultiRow() { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); @@ -336,14 +428,14 @@ public class ScrubTest fillCF(cfs, 10); assertOrderedAll(cfs, 10); - CompactionManager.instance.performScrub(cfs, false, true, 2); + performScrub(cfs, false, true, false, 2); // check data is still there assertOrderedAll(cfs, 10); } @Test - public void testScrubNoIndex() throws ExecutionException, InterruptedException, ConfigurationException + public void testScrubNoIndex() throws ConfigurationException { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); @@ -353,17 +445,24 @@ public class ScrubTest assertOrderedAll(cfs, 10); for (SSTableReader sstable : cfs.getLiveSSTables()) - assertTrue(new File(sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)).tryDelete()); + assertTrue(sstableIndexPath(sstable).tryDelete()); - CompactionManager.instance.performScrub(cfs, false, true, 2); + performScrub(cfs, false, true, false, 2); // check data is still there assertOrderedAll(cfs, 10); } @Test + @BMRule(name = "skip partition order verification", targetClass = "SortedTableWriter", targetMethod = "verifyPartition", action = "return true") public void testScrubOutOfOrder() { + // Run only for Big Table format because Big Table Format does not complain if partitions are given in invalid + // order. Legacy SSTables with out-of-order partitions exist in production systems and must be corrected + // by scrubbing. + + Assume.assumeTrue(BigFormat.isDefault()); + // This test assumes ByteOrderPartitioner to create out-of-order SSTable IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner(); DatabaseDescriptor.setPartitionerUnsafe(new ByteOrderedPartitioner()); @@ -382,7 +481,7 @@ public class ScrubTest Descriptor desc = cfs.newSSTableDescriptor(tempDataDir); try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, keys.size(), cfs.metadata, txn))) + SSTableTxnWriter writer = new SSTableTxnWriter(txn, createTestWriter(desc, keys.size(), cfs, txn))) { for (String k : keys) { @@ -397,7 +496,7 @@ public class ScrubTest try { - SSTableReader.open(desc, cfs.metadata); + SSTableReader.open(cfs, desc, cfs.metadata); fail("SSTR validation should have caught the out-of-order rows"); } catch (CorruptSSTableException ise) @@ -405,21 +504,21 @@ public class ScrubTest // open without validation for scrubbing Set components = new HashSet<>(); - if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) - components.add(Component.COMPRESSION_INFO); - components.add(Component.DATA); - components.add(Component.PRIMARY_INDEX); - components.add(Component.FILTER); - components.add(Component.STATS); - components.add(Component.SUMMARY); - components.add(Component.TOC); + if (desc.fileFor(Components.COMPRESSION_INFO).exists()) + components.add(Components.COMPRESSION_INFO); + components.add(Components.DATA); + components.add(Components.PRIMARY_INDEX); + components.add(Components.FILTER); + components.add(Components.STATS); + components.add(Components.SUMMARY); + components.add(Components.TOC); SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs); - if (sstable.last.compareTo(sstable.first) < 0) - sstable.last = sstable.first; +// if (sstable.last.compareTo(sstable.first) < 0) +// sstable.last = sstable.first; try (LifecycleTransaction scrubTxn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); - Scrubber scrubber = new Scrubber(cfs, scrubTxn, false, true)) + IScrubber scrubber = sstable.descriptor.getFormat().getScrubber(cfs, scrubTxn, new OutputHandler.LogOutput(), new IScrubber.Options.Builder().checkData().build())) { scrubber.scrub(); } @@ -435,19 +534,24 @@ public class ScrubTest } } - static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException + public static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException { - boolean compression = Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")); + overrideWithGarbage(sstable, key1, key2, (byte) 'z'); + } + + public static void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2, byte junk) throws IOException + { + boolean compression = sstable.metadata().params.compression.isEnabled(); long startPosition, endPosition; if (compression) { // overwrite with garbage the compression chunks from key1 to key2 - CompressionMetadata compData = CompressionMetadata.create(sstable.getFilename()); + CompressionMetadata compData = CompressionInfoComponent.load(sstable.descriptor); CompressionMetadata.Chunk chunk1 = compData.chunkFor( - sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position); + sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ)); CompressionMetadata.Chunk chunk2 = compData.chunkFor( - sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position); + sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ)); startPosition = Math.min(chunk1.offset, chunk2.offset); endPosition = Math.max(chunk1.offset + chunk1.length, chunk2.offset + chunk2.length); @@ -456,27 +560,40 @@ public class ScrubTest } else { // overwrite with garbage from key1 to key2 - long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position; - long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position; + long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ); + long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ); startPosition = Math.min(row0Start, row1Start); endPosition = Math.max(row0Start, row1Start); } - overrideWithGarbage(sstable, startPosition, endPosition); + overrideWithGarbage(sstable, startPosition, endPosition, junk); } private static void overrideWithGarbage(SSTableReader sstable, long startPosition, long endPosition) throws IOException { - try (FileChannel fileChannel = new File(sstable.getFilename()).newReadWriteChannel()) - { - fileChannel.position(startPosition); - fileChannel.write(ByteBufferUtil.bytes(StringUtils.repeat('z', (int) (endPosition - startPosition)))); - } - if (ChunkCache.instance != null) - ChunkCache.instance.invalidateFile(sstable.getFilename()); + overrideWithGarbage(sstable, startPosition, endPosition, (byte) 'z'); } - static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize) + private static void overrideWithGarbage(SSTableReader sstable, long startPosition, long endPosition, byte junk) throws IOException + { + overrideWithGarbage(sstable.getDataChannel().file(), startPosition, endPosition, junk); + } + + private static void overrideWithGarbage(File path, long startPosition, long endPosition, byte junk) throws IOException + { + try (RandomAccessFile file = new RandomAccessFile(path.toJavaIOFile(), "rw")) + { + file.seek(startPosition); + int length = (int) (endPosition - startPosition); + byte[] buff = new byte[length]; + Arrays.fill(buff, junk); + file.write(buff, 0, length); + } + if (ChunkCache.instance != null) + ChunkCache.instance.invalidateFile(path.toString()); + } + + public static void assertOrderedAll(ColumnFamilyStore cfs, int expectedSize) { assertOrdered(Util.cmd(cfs).build(), expectedSize); } @@ -485,9 +602,13 @@ public class ScrubTest { int size = 0; DecoratedKey prev = null; + logger.info("Reading data from " + cmd); for (Partition partition : Util.getAllUnfiltered(cmd)) { DecoratedKey current = partition.partitionKey(); + logger.info("Read " + current.toString()); + if (!(prev == null || prev.compareTo(current) < 0)) + logger.error("key " + current + " does not sort after previous key " + prev); assertTrue("key " + current + " does not sort after previous key " + prev, prev == null || prev.compareTo(current) < 0); prev = current; ++size; @@ -495,7 +616,7 @@ public class ScrubTest assertEquals(expectedSize, size); } - protected static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable) + public static void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable) { for (int i = 0; i < partitionsPerSSTable; i++) { @@ -510,10 +631,10 @@ public class ScrubTest Util.flush(cfs); } - public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long ... values) + public static void fillIndexCF(ColumnFamilyStore cfs, boolean composite, long... values) { assertEquals(0, values.length % 2); - for (int i = 0; i < values.length; i +=2) + for (int i = 0; i < values.length; i += 2) { UpdateBuilder builder = UpdateBuilder.create(cfs.metadata(), String.valueOf(i)); if (composite) @@ -534,21 +655,28 @@ public class ScrubTest Util.flush(cfs); } - protected static void fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException + public static String[] fillCounterCF(ColumnFamilyStore cfs, int partitionsPerSSTable) throws WriteTimeoutException { + SortedSet tokenSorted = Sets.newTreeSet(Comparator.comparing(a -> cfs.getPartitioner() + .decorateKey(ByteBufferUtil.bytes(a)))); for (int i = 0; i < partitionsPerSSTable; i++) { + if (i < 10) + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS); PartitionUpdate update = UpdateBuilder.create(cfs.metadata(), String.valueOf(i)) .newRow("r1").add("val", 100L) .build(); + tokenSorted.add(String.valueOf(i)); new CounterMutation(new Mutation(update), ConsistencyLevel.ONE).apply(); } Util.flush(cfs); + + return tokenSorted.toArray(ArrayUtils.EMPTY_STRING_ARRAY); } @Test - public void testScrubColumnValidation() throws InterruptedException, RequestExecutionException, ExecutionException + public void testScrubColumnValidation() throws RequestExecutionException { QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_static_columns (a bigint, b timeuuid, c boolean static, d text, PRIMARY KEY (a, b))", ksName), ConsistencyLevel.ONE); @@ -556,7 +684,7 @@ public class ScrubTest QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_static_columns (a, b, c, d) VALUES (123, c3db07e8-b602-11e3-bc6b-e0b9a54a6d93, true, 'foobar')", ksName)); Util.flush(cfs); - CompactionManager.instance.performScrub(cfs, false, true, 2); + performScrub(cfs, false, true, false, 2); QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_scrub_validation (a text primary key, b int)", ksName), ConsistencyLevel.ONE); ColumnFamilyStore cfs2 = keyspace.getColumnFamilyStore("test_scrub_validation"); @@ -564,7 +692,7 @@ public class ScrubTest new Mutation(UpdateBuilder.create(cfs2.metadata(), "key").newRow().add("b", Int32Type.instance.decompose(1)).build()).apply(); Util.flush(cfs2); - CompactionManager.instance.performScrub(cfs2, false, false, 2); + performScrub(cfs2, false, false, false, 2); } /** @@ -572,7 +700,7 @@ public class ScrubTest * we want as value for the clustering column, including something that would conflict with a CQL column definition. */ @Test - public void testValidationCompactStorage() throws Exception + public void testValidationCompactStorage() { QueryProcessor.process(String.format("CREATE TABLE \"%s\".test_compact_dynamic_columns (a int, b text, c text, PRIMARY KEY (a, b)) WITH COMPACT STORAGE", ksName), ConsistencyLevel.ONE); @@ -582,7 +710,7 @@ public class ScrubTest QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')", ksName)); QueryProcessor.executeInternal(String.format("INSERT INTO \"%s\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')", ksName)); Util.flush(cfs); - CompactionManager.instance.performScrub(cfs, true, true, 2); + performScrub(cfs, true, true, false, 2); // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away" UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".test_compact_dynamic_columns", ksName)); @@ -640,18 +768,18 @@ public class ScrubTest } @SuppressWarnings("SameParameterValue") - private void testScrubIndex(String cfName, String colName, boolean composite, boolean ... scrubs) - throws IOException, ExecutionException, InterruptedException + private void testScrubIndex(String cfName, String colName, boolean composite, boolean... scrubs) + throws IOException, ExecutionException, InterruptedException { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName); int numRows = 1000; - long[] colValues = new long [numRows * 2]; // each row has two columns - for (int i = 0; i < colValues.length; i+=2) + long[] colValues = new long[numRows * 2]; // each row has two columns + for (int i = 0; i < colValues.length; i += 2) { colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column - colValues[i+1] = 3L; //other column + colValues[i + 1] = 3L; //other column } fillIndexCF(cfs, composite, colValues); @@ -662,20 +790,20 @@ public class ScrubTest // scrub index Set indexCfss = cfs.indexManager.getAllIndexColumnFamilyStores(); assertEquals(1, indexCfss.size()); - for(ColumnFamilyStore indexCfs : indexCfss) + for (ColumnFamilyStore indexCfs : indexCfss) { for (int i = 0; i < scrubs.length; i++) { boolean failure = !scrubs[i]; if (failure) { //make sure the next scrub fails - overrideWithGarbage(indexCfs.getLiveSSTables().iterator().next(), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(2L)); + overrideWithGarbage(indexCfs.getLiveSSTables().iterator().next(), ByteBufferUtil.bytes(1L), ByteBufferUtil.bytes(2L), (byte) 0x7A); } - CompactionManager.AllSSTableOpStatus result = indexCfs.scrub(false, false, false, true, false,0); + CompactionManager.AllSSTableOpStatus result = indexCfs.scrub(false, true, IScrubber.options().build(), 0); assertEquals(failure ? CompactionManager.AllSSTableOpStatus.ABORTED : CompactionManager.AllSSTableOpStatus.SUCCESSFUL, - result); + result); } } @@ -684,11 +812,25 @@ public class ScrubTest assertOrdered(Util.cmd(cfs).filterOn(colName, Operator.EQ, 1L).build(), numRows / 2); } - private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, TableMetadataRef metadata, LifecycleTransaction txn) + private static SSTableMultiWriter createTestWriter(Descriptor descriptor, long keyCount, ColumnFamilyStore cfs, LifecycleTransaction txn) { - SerializationHeader header = new SerializationHeader(true, metadata.get(), metadata.get().regularAndStaticColumns(), EncodingStats.NO_STATS); - MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(0); - return new TestMultiWriter(new TestWriter(descriptor, keyCount, 0, null, false, metadata, collector, header, txn), txn); + SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS); + MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator).sstableLevel(0); + SSTableWriter writer = descriptor.getFormat() + .getWriterFactory() + .builder(descriptor) + .setKeyCount(keyCount) + .setRepairedAt(0) + .setPendingRepair(null) + .setTransientSSTable(false) + .setTableMetadataRef(cfs.metadata) + .setMetadataCollector(collector) + .setSerializationHeader(header) + .setFlushObservers(Collections.emptyList()) + .addDefaultComponents() + .build(txn, cfs); + + return new TestMultiWriter(writer, txn); } private static class TestMultiWriter extends SimpleSSTableMultiWriter @@ -699,33 +841,17 @@ public class ScrubTest } } - /** - * Test writer that allows to write out of order SSTable. - */ - private static class TestWriter extends BigTableWriter - { - TestWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, TableMetadataRef metadata, - MetadataCollector collector, SerializationHeader header, LifecycleTransaction txn) - { - super(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, Collections.emptySet(), txn); - } - - @Override - protected long beforeAppend(DecoratedKey decoratedKey) - { - return dataFile.position(); - } - } - /** * Tests with invalid sstables (containing duplicate entries in 2.0 and 3.0 storage format), * that were caused by upgrading from 2.x with duplicate range tombstones. - * + *

* See CASSANDRA-12144 for details. */ @Test public void testFilterOutDuplicates() throws Exception { + Assume.assumeTrue(BigFormat.isDefault()); + IPartitioner oldPart = DatabaseDescriptor.getPartitioner(); try { @@ -752,7 +878,7 @@ public class ScrubTest cfs.loadNewSSTables(); - cfs.scrub(true, true, false, false, false, 1); + cfs.scrub(true, false, IScrubber.options().skipCorrupted().build(), 1); UntypedResultSet rs = QueryProcessor.executeInternal(String.format("SELECT * FROM \"%s\".cf_with_duplicates_3_0", ksName)); assertNotNull(rs); @@ -768,4 +894,14 @@ public class ScrubTest DatabaseDescriptor.setPartitionerUnsafe(oldPart); } } + + private static CompactionManager.AllSSTableOpStatus performScrub(ColumnFamilyStore cfs, boolean skipCorrupted, boolean checkData, boolean reinsertOverflowedTTL, int jobs) + { + IScrubber.Options options = IScrubber.options() + .skipCorrupted(skipCorrupted) + .checkData(checkData) + .reinsertOverflowedTTLRows(reinsertOverflowedTTL) + .build(); + return CompactionManager.instance.performScrub(cfs, options, jobs); + } } diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java similarity index 71% rename from test/unit/org/apache/cassandra/db/VerifyTest.java rename to test/unit/org/apache/cassandra/io/sstable/VerifyTest.java index a58837aa08..307e09c9a3 100644 --- a/test/unit/org/apache/cassandra/db/VerifyTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable; import java.io.BufferedWriter; import java.io.IOException; @@ -26,22 +26,29 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.zip.CRC32; import java.util.zip.CheckedInputStream; import com.google.common.base.Charsets; import org.apache.commons.lang3.StringUtils; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; import org.apache.cassandra.batchlog.Batch; import org.apache.cassandra.batchlog.BatchlogManager; import org.apache.cassandra.cache.ChunkCache; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.compaction.Verifier; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; @@ -50,9 +57,12 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; +import org.apache.cassandra.io.sstable.format.SortedTableVerifier.RangeOwnHelper; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileUtils; @@ -63,6 +73,7 @@ import org.apache.cassandra.schema.CompressionParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.OutputHandler; import static org.apache.cassandra.SchemaLoader.counterCFMD; import static org.apache.cassandra.SchemaLoader.createKeyspace; @@ -75,8 +86,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * Test for {@link Verifier}. - * + * Test for {@link IVerifier}. + *

* Note: the complete coverage is composed of: * - {@link org.apache.cassandra.tools.StandaloneVerifierOnSSTablesTest} * - {@link org.apache.cassandra.tools.StandaloneVerifierTest} @@ -84,6 +95,8 @@ import static org.junit.Assert.fail; */ public class VerifyTest { + private final static Logger logger = LoggerFactory.getLogger(VerifyTest.class); + public static final String KEYSPACE = "Keyspace1"; public static final String CF = "Standard1"; public static final String CF2 = "Standard2"; @@ -105,6 +118,8 @@ public class VerifyTest public static void defineSchema() throws ConfigurationException { CompressionParams compressionParameters = CompressionParams.snappy(32768); + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setColumnIndexSize(0); loadSchema(); createKeyspace(KEYSPACE, @@ -137,7 +152,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -157,7 +172,7 @@ public class VerifyTest fillCounterCF(cfs, 2); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -177,7 +192,7 @@ public class VerifyTest fillCF(cfs, 2); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -198,7 +213,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build())) { verifier.verify(); } @@ -219,7 +234,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -240,7 +255,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -261,7 +276,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -282,7 +297,7 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build())) { verifier.verify(); } @@ -307,26 +322,30 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST)))) + try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST))) { - Long correctChecksum = file.readLong(); + long correctChecksum = file.readLong(); - writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST)); + writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST)); } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) {} + catch (CorruptSSTableException expected) + { + } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build())) { verifier.verify(); fail("Expected a RuntimeException to be thrown"); } - catch (RuntimeException err) {} + catch (RuntimeException expected) + { + } } @@ -344,22 +363,22 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); // overwrite one row with garbage - long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position; - long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position; - long startPosition = row0Start < row1Start ? row0Start : row1Start; - long endPosition = row0Start < row1Start ? row1Start : row0Start; + long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ); + long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ); + long startPosition = Math.min(row0Start, row1Start); + long endPosition = Math.max(row0Start, row1Start); - FileChannel file = new File(sstable.getFilename()).newReadWriteChannel(); - file.position(startPosition); - file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); - file.close(); + try (FileChannel file = new File(sstable.getFilename()).newReadWriteChannel()) { + file.position(startPosition); + file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); + } if (ChunkCache.instance != null) ChunkCache.instance.invalidateFile(sstable.getFilename()); // Update the Digest to have the right Checksum - writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.filenameFor(Component.DIGEST)); + writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.fileFor(Components.DIGEST)); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { // First a simple verify checking digest, which should succeed try @@ -368,16 +387,16 @@ public class VerifyTest } catch (CorruptSSTableException err) { + logger.error("Unexpected exception", err); fail("Simple verify should have succeeded as digest matched"); } } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build())) { // Now try extended verify try { verifier.verify(); - } catch (CorruptSSTableException err) { @@ -400,26 +419,30 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS); - FileChannel file = new File(filenameToCorrupt).newReadWriteChannel(); + FileChannel file = sstable.descriptor.fileFor(Components.STATS).newReadWriteChannel(); file.position(0); file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2))); file.close(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) - {} - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(false).build())) + catch (CorruptSSTableException expected) + { + } + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build())) { verifier.verify(); fail("Expected a RuntimeException to be thrown"); } - catch (CorruptSSTableException err) { fail("wrong exception thrown"); } - catch (RuntimeException err) - {} + catch (CorruptSSTableException unexpected) + { + fail("wrong exception thrown"); + } + catch (RuntimeException expected) + { + } } @Test @@ -439,30 +462,32 @@ public class VerifyTest sstable.reloadSSTableMetadata(); // break the sstable: - Long correctChecksum; - try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST)))) + long correctChecksum; + try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST))) { correctChecksum = file.readLong(); } - writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST)); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build())) + writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST)); + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) - {} + catch (CorruptSSTableException expected) + { + } assertTrue(sstable.isRepaired()); // now the repair status should be changed: - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) - {} + catch (CorruptSSTableException expected) + { + } assertFalse(sstable.isRepaired()); } @@ -480,7 +505,7 @@ public class VerifyTest tmd.updateNormalToken(new ByteOrderedPartitioner.BytesToken(tk2), InetAddressAndPort.getByName("127.0.0.2")); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkOwnsTokens(true).extendedVerification(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).extendedVerification(true).build())) { verifier.verify(); } @@ -488,11 +513,10 @@ public class VerifyTest { StorageService.instance.getTokenMetadata().clearUnsafe(); } - } @Test - public void testMutateRepair() throws IOException, ExecutionException, InterruptedException + public void testMutateRepair() throws IOException { CompactionManager.instance.disableAutoCompaction(); Keyspace keyspace = Keyspace.open(KEYSPACE); @@ -508,37 +532,42 @@ public class VerifyTest cfs.forceMajorCompaction(); sstable = cfs.getLiveSSTables().iterator().next(); - Long correctChecksum; - try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST)))) + long correctChecksum; + try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST))) { correctChecksum = file.readLong(); } - writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST)); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build())) + writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST)); + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build())) { verifier.verify(); fail("should be corrupt"); } - catch (CorruptSSTableException e) - {} + catch (CorruptSSTableException expected) + { + } assertFalse(sstable.isRepaired()); } @Test public void testVerifyIndex() throws IOException { - testBrokenComponentHelper(Component.PRIMARY_INDEX); + Assume.assumeTrue(BigFormat.isDefault()); + testBrokenComponentHelper(Components.PRIMARY_INDEX); } + @Test public void testVerifyBf() throws IOException { - testBrokenComponentHelper(Component.FILTER); + Assume.assumeTrue(SSTableReaderWithFilter.class.isAssignableFrom(SSTableFormat.Type.current().info.getReaderFactory().getReaderClass())); + testBrokenComponentHelper(Components.FILTER); } @Test public void testVerifyIndexSummary() throws IOException { - testBrokenComponentHelper(Component.SUMMARY); + Assume.assumeTrue(BigFormat.isDefault()); + testBrokenComponentHelper(Components.SUMMARY); } private void testBrokenComponentHelper(Component componentToBreak) throws IOException @@ -550,22 +579,21 @@ public class VerifyTest fillCF(cfs, 2); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build())) { verifier.verify(); //still not corrupt, should pass } - String filenameToCorrupt = sstable.descriptor.filenameFor(componentToBreak); - try(FileChannel fileChannel = new File(filenameToCorrupt).newReadWriteChannel()) + try (FileChannel fileChannel = sstable.descriptor.fileFor(componentToBreak).newReadWriteChannel()) { fileChannel.truncate(3); } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("should throw exception"); } - catch(CorruptSSTableException e) + catch (CorruptSSTableException e) { //expected } @@ -585,31 +613,35 @@ public class VerifyTest SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (RandomAccessReader file = RandomAccessReader.open(new File(sstable.descriptor.filenameFor(Component.DIGEST)))) + try (RandomAccessReader file = RandomAccessReader.open(sstable.descriptor.fileFor(Components.DIGEST))) { - Long correctChecksum = file.readLong(); + long correctChecksum = file.readLong(); - writeChecksum(++correctChecksum, sstable.descriptor.filenameFor(Component.DIGEST)); + writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST)); } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a CorruptSSTableException to be thrown"); } - catch (CorruptSSTableException err) {} + catch (CorruptSSTableException expected) + { + } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest { verifier.verify(); } - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().invokeDiskFailurePolicy(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build())) { verifier.verify(); fail("Expected a RuntimeException to be thrown"); } - catch (CorruptSSTableException err) {} + catch (CorruptSSTableException expected) + { + } } @Test @@ -619,12 +651,12 @@ public class VerifyTest normalized.add(r(Long.MIN_VALUE, Long.MIN_VALUE + 1)); normalized.add(r(Long.MIN_VALUE + 5, Long.MIN_VALUE + 6)); normalized.add(r(Long.MIN_VALUE + 10, Long.MIN_VALUE + 11)); - normalized.add(r(0,10)); - normalized.add(r(10,11)); - normalized.add(r(20,25)); - normalized.add(r(26,200)); + normalized.add(r(0, 10)); + normalized.add(r(10, 11)); + normalized.add(r(20, 25)); + normalized.add(r(26, 200)); - Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized); + RangeOwnHelper roh = new RangeOwnHelper(normalized); roh.validate(dk(1)); roh.validate(dk(10)); @@ -647,8 +679,8 @@ public class VerifyTest public void testRangeOwnHelperBadToken() { List> normalized = new ArrayList<>(); - normalized.add(r(0,10)); - Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized); + normalized.add(r(0, 10)); + RangeOwnHelper roh = new RangeOwnHelper(normalized); roh.validate(dk(1)); // call with smaller token to get exception roh.validate(dk(0)); @@ -658,8 +690,8 @@ public class VerifyTest @Test public void testRangeOwnHelperNormalize() { - List> normalized = Range.normalize(Collections.singletonList(r(0,0))); - Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized); + List> normalized = Range.normalize(Collections.singletonList(r(0, 0))); + RangeOwnHelper roh = new RangeOwnHelper(normalized); roh.validate(dk(Long.MIN_VALUE)); roh.validate(dk(0)); roh.validate(dk(Long.MAX_VALUE)); @@ -668,8 +700,8 @@ public class VerifyTest @Test public void testRangeOwnHelperNormalizeWrap() { - List> normalized = Range.normalize(Collections.singletonList(r(Long.MAX_VALUE - 1000,Long.MIN_VALUE + 1000))); - Verifier.RangeOwnHelper roh = new Verifier.RangeOwnHelper(normalized); + List> normalized = Range.normalize(Collections.singletonList(r(Long.MAX_VALUE - 1000, Long.MIN_VALUE + 1000))); + RangeOwnHelper roh = new RangeOwnHelper(normalized); roh.validate(dk(Long.MIN_VALUE)); roh.validate(dk(Long.MAX_VALUE)); boolean gotException = false; @@ -687,7 +719,7 @@ public class VerifyTest @Test public void testEmptyRanges() { - new Verifier.RangeOwnHelper(Collections.emptyList()).validate(dk(1)); + new RangeOwnHelper(Collections.emptyList()).validate(dk(1)); } @Test @@ -707,7 +739,7 @@ public class VerifyTest for (SSTableReader sstable : cfs.getLiveSSTables()) { - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().checkOwnsTokens(true).build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).build())) { verifier.verify(); } @@ -724,9 +756,9 @@ public class VerifyTest assertEquals(1.0, cfs.metadata().params.bloomFilterFpChance, 0.0); for (SSTableReader sstable : cfs.getLiveSSTables()) { - File f = new File(sstable.descriptor.filenameFor(Component.FILTER)); + File f = sstable.descriptor.fileFor(Components.FILTER); assertFalse(f.exists()); - try (Verifier verifier = new Verifier(cfs, sstable, false, Verifier.options().build())) + try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build())) { verifier.verify(); } @@ -734,7 +766,6 @@ public class VerifyTest } - private DecoratedKey dk(long l) { return new BufferDecoratedKey(t(l), ByteBufferUtil.EMPTY_BYTE_BUFFER); @@ -778,37 +809,35 @@ public class VerifyTest protected long simpleFullChecksum(String filename) throws IOException { - try (FileInputStreamPlus inputStream = new FileInputStreamPlus(filename)) + try (FileInputStreamPlus inputStream = new FileInputStreamPlus(filename); + CheckedInputStream cinStream = new CheckedInputStream(inputStream, new CRC32())) { - CRC32 checksum = new CRC32(); - CheckedInputStream cinStream = new CheckedInputStream(inputStream, checksum); byte[] b = new byte[128]; - while (cinStream.read(b) >= 0) { + //noinspection StatementWithEmptyBody + while (cinStream.read(b) >= 0) + { } return cinStream.getChecksum().getValue(); } } - public static void writeChecksum(long checksum, String filePath) + public static void writeChecksum(long checksum, File file) { - File outFile = new File(filePath); BufferedWriter out = null; try { - out = Files.newBufferedWriter(outFile.toPath(), Charsets.UTF_8); + out = Files.newBufferedWriter(file.toPath(), Charsets.UTF_8); out.write(String.valueOf(checksum)); out.flush(); out.close(); } catch (IOException e) { - throw new FSWriteError(e, outFile); + throw new FSWriteError(e, file); } finally { FileUtils.closeQuietly(out); } - } - } diff --git a/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java similarity index 81% rename from test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java rename to test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java index afcf2a5659..3c954556f3 100644 --- a/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/filter/BloomFilterTrackerTest.java @@ -1,6 +1,4 @@ -package org.apache.cassandra.io; /* - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -9,22 +7,20 @@ package org.apache.cassandra.io; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package org.apache.cassandra.io.sstable.filter; + import org.junit.Test; -import org.apache.cassandra.io.sstable.BloomFilterTracker; - import static org.junit.Assert.assertEquals; public class BloomFilterTrackerTest diff --git a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java b/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java index 4e62b9c07c..8dc7425779 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java @@ -17,13 +17,24 @@ */ package org.apache.cassandra.io.sstable.format; +import java.util.NoSuchElementException; + +import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.config.DatabaseDescriptor; + import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.assertEquals; public class VersionAndTypeTest { + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.clientInitialization(); + } + @Test public void testValidInput() { @@ -37,10 +48,10 @@ public class VersionAndTypeTest .hasMessageContaining("should be of the form 'big-bc'"); assertThatThrownBy(() -> VersionAndType.fromString("mcc-")).isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("should be of the form 'big-bc'"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc-d")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("No Type constant mcc"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc-d-")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("No Type constant mcc"); + assertThatThrownBy(() -> VersionAndType.fromString("mcc-d")).isInstanceOf(NoSuchElementException.class) + .hasMessageContaining("mcc"); + assertThatThrownBy(() -> VersionAndType.fromString("mcc-d-")).isInstanceOf(NoSuchElementException.class) + .hasMessageContaining("mcc"); assertThatThrownBy(() -> VersionAndType.fromString("mcc")).isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("should be of the form 'big-bc'"); assertThatThrownBy(() -> VersionAndType.fromString("-")).isInstanceOf(IllegalArgumentException.class) diff --git a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java similarity index 90% rename from test/unit/org/apache/cassandra/db/RowIndexEntryTest.java rename to test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java index 871bedbe2d..512fa730f8 100644 --- a/test/unit/org/apache/cassandra/db/RowIndexEntryTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/big/RowIndexEntryTest.java @@ -15,38 +15,73 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable.format.big; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import com.google.common.primitives.Ints; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + import org.apache.cassandra.Util; import org.apache.cassandra.cache.IMeasurableMemory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; -import org.apache.cassandra.db.columniterator.AbstractSSTableIterator; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.partitions.ImmutableBTreePartition; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.ColumnData; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.SerializationHelper; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredSerializer; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.IndexInfo; -import org.apache.cassandra.io.sstable.format.SSTableFlushObserver; +import org.apache.cassandra.io.sstable.SSTableFlushObserver; import org.apache.cassandra.io.sstable.format.Version; -import org.apache.cassandra.io.sstable.format.big.BigFormat; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileHandle; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.LongSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.btree.BTree; -import org.junit.Assert; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -63,6 +98,12 @@ public class RowIndexEntryTest extends CQLTester return Util.clustering(comp, l); } + @BeforeClass + public static void beforeClass() + { + Assume.assumeTrue(BigFormat.isDefault()); + } + @Test public void testC11206AgainstPreviousArray() throws Exception { @@ -120,7 +161,7 @@ public class RowIndexEntryTest extends CQLTester CreateTableStatement.parse("CREATE TABLE pipe.dev_null (pk bigint, ck bigint, val text, PRIMARY KEY(pk, ck))", "foo") .build(); - Version version = BigFormat.latestVersion; + Version version = BigFormat.getInstance().getLatestVersion(); DeletionTime deletionInfo = new DeletionTime(FBUtilities.timestampMicros(), FBUtilities.nowInSeconds()); LivenessInfo primaryKeyLivenessInfo = LivenessInfo.EMPTY; @@ -129,7 +170,7 @@ public class RowIndexEntryTest extends CQLTester SerializationHeader header = new SerializationHeader(true, metadata, metadata.regularAndStaticColumns(), EncodingStats.NO_STATS); // create C-11206 + old serializer instances - RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(version, header); + RowIndexEntry.IndexSerializer rieSerializer = new RowIndexEntry.Serializer(version, header, null); Pre_C_11206_RowIndexEntry.Serializer oldSerializer = new Pre_C_11206_RowIndexEntry.Serializer(metadata, version, header); @SuppressWarnings({ "resource", "IOResourceOpenedButNotSafelyClosed" }) @@ -139,7 +180,7 @@ public class RowIndexEntryTest extends CQLTester final SequentialWriter dataWriterNew; final SequentialWriter dataWriterOld; - final org.apache.cassandra.db.ColumnIndex columnIndex; + final BigFormatPartitionWriter partitionWriter; RowIndexEntry rieNew; ByteBuffer rieNewSerialized; @@ -151,8 +192,7 @@ public class RowIndexEntryTest extends CQLTester SequentialWriterOption option = SequentialWriterOption.newBuilder().bufferSize(1024).build(); File f = FileUtils.createTempFile("RowIndexEntryTest-", "db"); dataWriterNew = new SequentialWriter(f, option); - columnIndex = new org.apache.cassandra.db.ColumnIndex(header, dataWriterNew, version, Collections.emptyList(), - rieSerializer.indexInfoSerializer()); + partitionWriter = new BigFormatPartitionWriter(header, dataWriterNew, version, rieSerializer.indexInfoSerializer()); f = FileUtils.createTempFile("RowIndexEntryTest-", "db"); dataWriterOld = new SequentialWriter(f, option); @@ -169,18 +209,25 @@ public class RowIndexEntryTest extends CQLTester { Iterator> clusteringIter = clusterings.iterator(); - columnIndex.buildRowIndex(makeRowIter(staticRow, partitionKey, clusteringIter, dataWriterNew)); + partitionWriter.start(partitionKey, DeletionTime.LIVE); + if (staticRow != null) + partitionWriter.addStaticRow(staticRow); + AbstractUnfilteredRowIterator rowIter = makeRowIter(staticRow, partitionKey, clusteringIter, dataWriterNew); + while (rowIter.hasNext()) + partitionWriter.addUnfiltered(rowIter.next()); + partitionWriter.finish(); + rieNew = RowIndexEntry.create(startPosition, 0L, - deletionInfo, columnIndex.headerLength, columnIndex.columnIndexCount, - columnIndex.indexInfoSerializedSize(), - columnIndex.indexSamples(), columnIndex.offsets(), + deletionInfo, partitionWriter.getHeaderLength(), partitionWriter.getColumnIndexCount(), + partitionWriter.indexInfoSerializedSize(), + partitionWriter.indexSamples(), partitionWriter.offsets(), rieSerializer.indexInfoSerializer()); - rieSerializer.serialize(rieNew, rieOutput, columnIndex.buffer()); + rieSerializer.serialize(rieNew, rieOutput, partitionWriter.buffer()); rieNewSerialized = rieOutput.buffer().duplicate(); Iterator> clusteringIter2 = clusterings.iterator(); ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(makeRowIter(staticRow, partitionKey, clusteringIter2, dataWriterOld), - dataWriterOld, header, Collections.emptySet(), BigFormat.latestVersion); + dataWriterOld, header, Collections.emptySet(), BigFormat.getInstance().getLatestVersion()); rieOld = Pre_C_11206_RowIndexEntry.create(startPosition, deletionInfo, columnIndex); oldSerializer.serialize(rieOld, oldOutput); rieOldSerialized = oldOutput.buffer().duplicate(); @@ -395,7 +442,7 @@ public class RowIndexEntryTest extends CQLTester DataOutputBuffer buffer = new DataOutputBuffer(); SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS); - Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata(), BigFormat.latestVersion, header); + Pre_C_11206_RowIndexEntry.Serializer serializer = new Pre_C_11206_RowIndexEntry.Serializer(cfs.metadata(), BigFormat.getInstance().getLatestVersion(), header); serializer.serialize(simple, buffer); @@ -410,9 +457,9 @@ public class RowIndexEntryTest extends CQLTester File tempFile = FileUtils.createTempFile("row_index_entry_test", null); tempFile.deleteOnExit(); SequentialWriter writer = new SequentialWriter(tempFile); - ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, Collections.emptySet(), BigFormat.latestVersion); + ColumnIndex columnIndex = RowIndexEntryTest.ColumnIndex.writeAndBuildIndex(partition.unfilteredIterator(), writer, header, Collections.emptySet(), BigFormat.getInstance().getLatestVersion()); Pre_C_11206_RowIndexEntry withIndex = Pre_C_11206_RowIndexEntry.create(0xdeadbeef, DeletionTime.LIVE, columnIndex); - IndexInfo.Serializer indexSerializer = IndexInfo.serializer(BigFormat.latestVersion, header); + IndexInfo.Serializer indexSerializer = IndexInfo.serializer(BigFormat.getInstance().getLatestVersion(), header); // sanity check assertTrue(columnIndex.columnsIndex.size() >= 3); @@ -449,7 +496,7 @@ public class RowIndexEntryTest extends CQLTester bb = buffer.buffer(); input = new DataInputBuffer(bb, false); - Pre_C_11206_RowIndexEntry.Serializer.skip(input, BigFormat.latestVersion); + Pre_C_11206_RowIndexEntry.Serializer.skip(input, BigFormat.getInstance().getLatestVersion()); Assert.assertEquals(0, bb.remaining()); } @@ -780,7 +827,7 @@ public class RowIndexEntryTest extends CQLTester } }; - AbstractSSTableIterator.IndexState indexState = new AbstractSSTableIterator.IndexState( + IndexState indexState = new IndexState( null, comp, rie, false, null ); @@ -794,7 +841,7 @@ public class RowIndexEntryTest extends CQLTester assertEquals(3, indexState.indexFor(cn(100L), 2)); assertEquals(3, indexState.indexFor(cn(100L), 3)); - indexState = new AbstractSSTableIterator.IndexState( + indexState = new IndexState( null, comp, rie, true, null ); diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java similarity index 75% rename from test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java rename to test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java index c494e3c3db..984e9f2475 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java @@ -15,18 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import com.google.common.base.Joiner; import com.google.common.collect.Sets; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -34,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.config.DatabaseDescriptor; @@ -43,10 +54,12 @@ import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionInfo; import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.schema.CachingParams; @@ -56,11 +69,10 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.utils.ByteBufferUtil; import static com.google.common.collect.ImmutableMap.of; -import static java.util.Arrays.asList; import static org.apache.cassandra.Util.assertOnDiskState; import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL; -import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.DOWNSAMPLE_THESHOLD; -import static org.apache.cassandra.io.sstable.IndexSummaryRedistribution.UPSAMPLE_THRESHOLD; +import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution.DOWNSAMPLE_THESHOLD; +import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution.UPSAMPLE_THRESHOLD; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -68,7 +80,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class IndexSummaryManagerTest +public class IndexSummaryManagerTest> { private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManagerTest.class); @@ -84,6 +96,10 @@ public class IndexSummaryManagerTest @BeforeClass public static void defineSchema() throws ConfigurationException { + DatabaseDescriptor.daemonInitialization(); + Assume.assumeTrue("This test make sense only if the default SSTable format support index summary", + IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); + SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), @@ -128,25 +144,25 @@ public class IndexSummaryManagerTest IndexSummaryManager.instance.setMemoryPoolCapacityInMB(originalCapacity); } - private static long totalOffHeapSize(List sstables) + private long totalOffHeapSize(List sstables) { long total = 0; - for (SSTableReader sstable : sstables) - total += sstable.getIndexSummaryOffHeapSize(); + for (R sstable : sstables) + total += sstable.getIndexSummary().getOffHeapSize(); return total; } - private static List resetSummaries(ColumnFamilyStore cfs, List sstables, long originalOffHeapSize) throws IOException + private List resetSummaries(ColumnFamilyStore cfs, List sstables, long originalOffHeapSize) throws IOException { - for (SSTableReader sstable : sstables) + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), originalOffHeapSize * sstables.size()); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), originalOffHeapSize * sstables.size()); } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel()); return sstables; } @@ -163,13 +179,7 @@ public class IndexSummaryManagerTest } } - private Comparator hotnessComparator = new Comparator() - { - public int compare(SSTableReader o1, SSTableReader o2) - { - return Double.compare(o1.getReadMeter().fifteenMinuteRate(), o2.getReadMeter().fifteenMinuteRate()); - } - }; + private final Comparator hotnessComparator = Comparator.comparingDouble(o -> o.getReadMeter().fifteenMinuteRate()); private void createSSTables(String ksname, String cfname, int numSSTables, int numPartition) { @@ -205,12 +215,12 @@ public class IndexSummaryManagerTest throw new RuntimeException(e); } } - assertEquals(numSSTables, cfs.getLiveSSTables().size()); + assertEquals(numSSTables, ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).size()); validateData(cfs, numPartition); } @Test - public void testChangeMinIndexInterval() throws IOException + public > void testChangeMinIndexInterval() throws IOException { String ksname = KEYSPACE1; String cfname = CF_STANDARDLOWiINTERVAL; // index interval of 8, no key caching @@ -220,78 +230,78 @@ public class IndexSummaryManagerTest int numRows = 256; createSSTables(ksname, cfname, numSSTables, numRows); - List sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); - for (SSTableReader sstable : sstables) - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); + for (R sstable : sstables) + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); // double the min_index_interval SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 2).build()); IndexSummaryManager.instance.redistributeSummaries(); - for (SSTableReader sstable : cfs.getLiveSSTables()) + for (R sstable : ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs)) { - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize()); + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); + assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size()); } // return min_index_interval to its original value SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build()); IndexSummaryManager.instance.redistributeSummaries(); - for (SSTableReader sstable : cfs.getLiveSSTables()) + for (R sstable : ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs)) { - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize()); + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); + assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size()); } // halve the min_index_interval, but constrain the available space to exactly what we have now; as a result, // the summary shouldn't change SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval / 2).build()); - SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - long summarySpace = sstable.getIndexSummaryOffHeapSize(); - try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN)) + R sstable = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).iterator().next(); + long summarySpace = sstable.getIndexSummary().getOffHeapSize(); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), summarySpace); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), summarySpace); } - sstable = cfs.getLiveSSTables().iterator().next(); - assertEquals(originalMinIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / originalMinIndexInterval, sstable.getIndexSummarySize()); + sstable = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).iterator().next(); + assertEquals(originalMinIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); + assertEquals(numRows / originalMinIndexInterval, sstable.getIndexSummary().size()); // keep the min_index_interval the same, but now give the summary enough space to grow by 50% - double previousInterval = sstable.getEffectiveIndexInterval(); - int previousSize = sstable.getIndexSummarySize(); - try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN)) + double previousInterval = sstable.getIndexSummary().getEffectiveIndexInterval(); + int previousSize = sstable.getIndexSummary().size(); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace * 1.5)); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace * 1.5)); } - sstable = cfs.getLiveSSTables().iterator().next(); - assertEquals(previousSize * 1.5, (double) sstable.getIndexSummarySize(), 1); - assertEquals(previousInterval * (1.0 / 1.5), sstable.getEffectiveIndexInterval(), 0.001); + sstable = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).iterator().next(); + assertEquals(previousSize * 1.5, (double) sstable.getIndexSummary().size(), 1); + assertEquals(previousInterval * (1.0 / 1.5), sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); // return min_index_interval to it's original value (double it), but only give the summary enough space // to have an effective index interval of twice the new min SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval).build()); - try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN)) + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace / 2.0)); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) Math.ceil(summarySpace / 2.0)); } - sstable = cfs.getLiveSSTables().iterator().next(); - assertEquals(originalMinIndexInterval * 2, sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / (originalMinIndexInterval * 2), sstable.getIndexSummarySize()); + sstable = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).iterator().next(); + assertEquals(originalMinIndexInterval * 2, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); + assertEquals(numRows / (originalMinIndexInterval * 2), sstable.getIndexSummary().size()); // raise the min_index_interval above our current effective interval, but set the max_index_interval lower // than what we actually have space for (meaning the index summary would ideally be smaller, but this would // result in an effective interval above the new max) SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().minIndexInterval(originalMinIndexInterval * 4).build()); SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(originalMinIndexInterval * 4).build()); - try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN)) + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10); } - sstable = cfs.getLiveSSTables().iterator().next(); - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); + sstable = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).iterator().next(); + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); } @Test @@ -305,41 +315,41 @@ public class IndexSummaryManagerTest int numRows = 256; createSSTables(ksname, cfname, numSSTables, numRows); - List sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10); } - sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) - assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01); + sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) + assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01); // halve the max_index_interval SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval / 2).build()); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 1); } - sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) { - assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01); - assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize()); + assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01); + assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().size()); } // return max_index_interval to its original value SchemaTestUtil.announceTableUpdate(cfs.metadata().unbuild().maxIndexInterval(cfs.metadata().params.maxIndexInterval * 2).build()); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 1); + redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 1); } - for (SSTableReader sstable : cfs.getLiveSSTables()) + for (R sstable : ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs)) { - assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getEffectiveIndexInterval(), 0.01); - assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummarySize()); + assertEquals(cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.01); + assertEquals(numRows / cfs.metadata().params.maxIndexInterval, sstable.getIndexSummary().size()); } } @@ -356,19 +366,19 @@ public class IndexSummaryManagerTest int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata().params.minIndexInterval) / cfs.metadata().params.maxIndexInterval; - List sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); - long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize(); + long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize(); // there should be enough space to not downsample anything try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables)); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables)); } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel()); assertEquals(singleSummaryOffHeapSpace * numSSTables, totalOffHeapSize(sstables)); validateData(cfs, numRows); @@ -376,38 +386,38 @@ public class IndexSummaryManagerTest assert sstables.size() == 4; try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2))); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2))); } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // everything should get cut to a quarter try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 4))); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 4))); } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL / 4, sstable.getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // upsample back up to half try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4)); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * (numSSTables / 2) + 4)); } assert sstables.size() == 4; - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // upsample back up to the original index summary try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables)); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * numSSTables)); } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // make two of the four sstables cold, only leave enough space for three full index summaries, @@ -416,13 +426,13 @@ public class IndexSummaryManagerTest sstables.get(1).overrideReadMeter(new RestorableMeter(50.0, 50.0)); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3)); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3)); } Collections.sort(sstables, hotnessComparator); - assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // small increases or decreases in the read rate don't result in downsampling or upsampling @@ -432,13 +442,13 @@ public class IndexSummaryManagerTest sstables.get(1).overrideReadMeter(new RestorableMeter(higherRate, higherRate)); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3)); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3)); } Collections.sort(sstables, hotnessComparator); - assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(0).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL / 2, sstables.get(1).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // reset, and then this time, leave enough space for one of the cold sstables to not get downsampled @@ -450,17 +460,17 @@ public class IndexSummaryManagerTest try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3) + 50); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (singleSummaryOffHeapSpace * 3) + 50); } Collections.sort(sstables, hotnessComparator); - if (sstables.get(0).getIndexSummarySamplingLevel() == minSamplingLevel) - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummarySamplingLevel()); + if (sstables.get(0).getIndexSummary().getSamplingLevel() == minSamplingLevel) + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummary().getSamplingLevel()); else - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummarySamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummary().getSamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummarySamplingLevel()); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(2).getIndexSummary().getSamplingLevel()); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); @@ -474,23 +484,23 @@ public class IndexSummaryManagerTest sstables.get(3).overrideReadMeter(new RestorableMeter(128.0, 128.0)); try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL)))); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL)))); } Collections.sort(sstables, hotnessComparator); - assertEquals(1, sstables.get(0).getIndexSummarySize()); // at the min sampling level - assertEquals(1, sstables.get(0).getIndexSummarySize()); // at the min sampling level - assertTrue(sstables.get(2).getIndexSummarySamplingLevel() > minSamplingLevel); - assertTrue(sstables.get(2).getIndexSummarySamplingLevel() < BASE_SAMPLING_LEVEL); - assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel()); + assertEquals(1, sstables.get(0).getIndexSummary().size()); // at the min sampling level + assertEquals(1, sstables.get(0).getIndexSummary().size()); // at the min sampling level + assertTrue(sstables.get(2).getIndexSummary().getSamplingLevel() > minSamplingLevel); + assertTrue(sstables.get(2).getIndexSummary().getSamplingLevel() < BASE_SAMPLING_LEVEL); + assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); // Don't leave enough space for even the minimal index summaries try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN)) { - sstables = redistributeSummaries(Collections.EMPTY_LIST, of(cfs.metadata.id, txn), 10); + sstables = redistributeSummaries(Collections.emptyList(), of(cfs.metadata.id, txn), 10); } - for (SSTableReader sstable : sstables) - assertEquals(1, sstable.getIndexSummarySize()); // at the min sampling level + for (R sstable : sstables) + assertEquals(1, sstable.getIndexSummary().size()); // at the min sampling level validateData(cfs, numRows); } @@ -519,19 +529,19 @@ public class IndexSummaryManagerTest Util.flush(cfs); - List sstables = new ArrayList<>(cfs.getLiveSSTables()); + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); assertEquals(1, sstables.size()); - SSTableReader original = sstables.get(0); + R original = sstables.get(0); - SSTableReader sstable = original; - try (LifecycleTransaction txn = cfs.getTracker().tryModify(asList(sstable), OperationType.UNKNOWN)) + R sstable = original; + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.UNKNOWN)) { for (int samplingLevel = 1; samplingLevel < BASE_SAMPLING_LEVEL; samplingLevel++) { sstable = sstable.cloneWithNewSummarySamplingLevel(cfs, samplingLevel); - assertEquals(samplingLevel, sstable.getIndexSummarySamplingLevel()); + assertEquals(samplingLevel, sstable.getIndexSummary().getSamplingLevel()); int expectedSize = (numRows * samplingLevel) / (cfs.metadata().params.minIndexInterval * BASE_SAMPLING_LEVEL); - assertEquals(expectedSize, sstable.getIndexSummarySize(), 1); + assertEquals(expectedSize, sstable.getIndexSummary().size(), 1); txn.update(sstable, true); txn.checkpoint(); } @@ -628,14 +638,14 @@ public class IndexSummaryManagerTest int numRows = 256; createSSTables(ksname, cfname, numSSTables, numRows); - List allSSTables = new ArrayList<>(cfs.getLiveSSTables()); - List sstables = allSSTables.subList(0, 4); - List compacting = allSSTables.subList(4, 8); + List allSSTables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + List sstables = allSSTables.subList(0, 4); + List compacting = allSSTables.subList(4, 8); - for (SSTableReader sstable : sstables) + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); - final long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize(); + final long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize(); // everything should get cut in half final AtomicReference exception = new AtomicReference<>(); @@ -701,9 +711,9 @@ public class IndexSummaryManagerTest assertNotNull("Expected compaction interrupted exception", exception.get()); assertTrue("Expected no active compactions", CompactionManager.instance.active.getCompactions().isEmpty()); - Set beforeRedistributionSSTables = new HashSet<>(allSSTables); - Set afterCancelSSTables = new HashSet<>(cfs.getLiveSSTables()); - Set disjoint = Sets.symmetricDifference(beforeRedistributionSSTables, afterCancelSSTables); + Set beforeRedistributionSSTables = new HashSet<>(allSSTables); + Set afterCancelSSTables = Sets.newHashSet(ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs)); + Set disjoint = Sets.symmetricDifference(beforeRedistributionSSTables, afterCancelSSTables); assertTrue(String.format("Mismatched files before and after cancelling redistribution: %s", Joiner.on(",").join(disjoint)), disjoint.isEmpty()); @@ -722,11 +732,11 @@ public class IndexSummaryManagerTest int numRows = 256; createSSTables(ksname, cfname, numSSTables, numRows); - List sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); - long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize(); + long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummary().getOffHeapSize(); // everything should get cut in half assert sstables.size() == numSSTables; @@ -742,18 +752,18 @@ public class IndexSummaryManagerTest { // expected } - for (SSTableReader sstable : sstables) - assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel()); + for (R sstable : sstables) + assertEquals(BASE_SAMPLING_LEVEL, sstable.getIndexSummary().getSamplingLevel()); validateData(cfs, numRows); assertOnDiskState(cfs, numSSTables); } - private static List redistributeSummaries(List compacting, - Map transactions, - long memoryPoolBytes) + private List redistributeSummaries(List compacting, + Map transactions, + long memoryPoolBytes) throws IOException { - long nonRedistributingOffHeapSize = compacting.stream().mapToLong(SSTableReader::getIndexSummaryOffHeapSize).sum(); + long nonRedistributingOffHeapSize = compacting.stream().mapToLong(t -> t.getIndexSummary().getOffHeapSize()).sum(); return IndexSummaryManager.redistributeSummaries(new IndexSummaryRedistribution(transactions, nonRedistributingOffHeapSize, memoryPoolBytes)); @@ -772,7 +782,7 @@ public class IndexSummaryManagerTest this.barrier = barrier; } - public List redistributeSummaries() throws IOException + public > List redistributeSummaries() throws IOException { try { diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java similarity index 84% rename from test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java rename to test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java index 9c121bf7bf..2cda741d6a 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryRedistributionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistributionTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; import java.io.IOException; import java.nio.ByteBuffer; @@ -25,15 +25,19 @@ import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.ServerTestUtils; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.metrics.StorageMetrics; @@ -43,7 +47,7 @@ import org.apache.cassandra.schema.SchemaTestUtil; import static org.junit.Assert.assertEquals; -public class IndexSummaryRedistributionTest +public class IndexSummaryRedistributionTest> { private static final String KEYSPACE1 = "IndexSummaryRedistributionTest"; private static final String CF_STANDARD = "Standard"; @@ -51,6 +55,9 @@ public class IndexSummaryRedistributionTest @BeforeClass public static void defineSchema() throws ConfigurationException { + DatabaseDescriptor.daemonInitialization(); + Assume.assumeTrue("This test make sense only if the default SSTable format support index summary", + IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), @@ -77,16 +84,16 @@ public class IndexSummaryRedistributionTest createSSTables(ksname, cfname, numSSTables, numRows); - List sstables = new ArrayList<>(cfs.getLiveSSTables()); - for (SSTableReader sstable : sstables) + List sstables = ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs); + for (R sstable : sstables) sstable.overrideReadMeter(new RestorableMeter(100.0, 100.0)); long oldSize = 0; long oldSizeUncompressed = 0; - for (SSTableReader sstable : sstables) + for (R sstable : sstables) { - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); oldSize += sstable.bytesOnDisk(); oldSizeUncompressed += sstable.logicalBytesOnDisk(); } @@ -105,10 +112,10 @@ public class IndexSummaryRedistributionTest long newSize = 0; long newSizeUncompressed = 0; - for (SSTableReader sstable : cfs.getLiveSSTables()) + for (R sstable : ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs)) { - assertEquals(cfs.metadata().params.minIndexInterval, sstable.getEffectiveIndexInterval(), 0.001); - assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummarySize()); + assertEquals(cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().getEffectiveIndexInterval(), 0.001); + assertEquals(numRows / cfs.metadata().params.minIndexInterval, sstable.getIndexSummary().size()); newSize += sstable.bytesOnDisk(); newSizeUncompressed += sstable.logicalBytesOnDisk(); } @@ -156,6 +163,6 @@ public class IndexSummaryRedistributionTest throw new RuntimeException(e); } } - assertEquals(numSSTables, cfs.getLiveSSTables().size()); + assertEquals(numSSTables, ServerTestUtils.getLiveIndexSummarySupportingReaders(cfs).size()); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java similarity index 92% rename from test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java rename to test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java index d0680f8a5b..faa14c416d 100644 --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryTest.java @@ -15,35 +15,43 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.io.sstable; +package org.apache.cassandra.io.sstable.indexsummary; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.UUID; import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.Assume; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; -import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.Downsampling; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import org.apache.cassandra.io.util.DataOutputBuffer; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Pair; -import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.downsample; -import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.entriesAtSamplingLevel; import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL; +import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder.downsample; +import static org.apache.cassandra.io.sstable.indexsummary.IndexSummaryBuilder.entriesAtSamplingLevel; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class IndexSummaryTest { @@ -224,7 +232,7 @@ public class IndexSummaryTest dos.writeUTF("JUNK"); dos.writeUTF("JUNK"); FileUtils.closeQuietly(dos); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray())); + DataInputStreamPlus dis = new DataInputBuffer(dos.toByteArray()); IndexSummary is = IndexSummary.serializer.deserialize(dis, partitioner, 1, 1); for (int i = 0; i < 100; i++) assertEquals(i, is.binarySearch(random.left.get(i))); @@ -250,7 +258,7 @@ public class IndexSummaryTest DataOutputBuffer dos = new DataOutputBuffer(); IndexSummary.serializer.serialize(summary, dos); - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dos.toByteArray())); + DataInputStreamPlus dis = new DataInputBuffer(dos.toByteArray()); IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, 1, 1); assertEquals(1, loaded.size()); @@ -287,7 +295,7 @@ public class IndexSummaryTest @Test public void testDownsamplePatterns() { - assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(0)); + Assert.assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(0)); assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(1)); assertEquals(Arrays.asList(1, 0), Downsampling.getSamplingPattern(2)); @@ -328,7 +336,7 @@ public class IndexSummaryTest int downsamplingRound = 1; for (int samplingLevel = BASE_SAMPLING_LEVEL - 1; samplingLevel >= 1; samplingLevel--) { - try (IndexSummary downsampled = downsample(original, samplingLevel, 128, partitioner);) + try (IndexSummary downsampled = downsample(original, samplingLevel, 128, partitioner)) { assertEquals(entriesAtSamplingLevel(samplingLevel, original.getMaxNumberOfEntries()), downsampled.size()); @@ -381,10 +389,10 @@ public class IndexSummaryTest { for (DecoratedKey key : keys) { - long orig = SSTableReader.getIndexScanPositionFromBinarySearchResult(original.binarySearch(key), original); + long orig = original.getScanPosition(key); int binarySearch = downsampled.binarySearch(key); - int index = SSTableReader.getIndexSummaryIndexFromBinarySearchResult(binarySearch); - int scanFrom = (int) SSTableReader.getIndexScanPositionFromBinarySearchResult(index, downsampled); + int index = IndexSummary.getIndexFromBinarySearchResult(binarySearch); + int scanFrom = (int) downsampled.getScanPositionFromBinarySearchResult(index); assert scanFrom <= orig; int effectiveInterval = downsampled.getEffectiveIndexIntervalAfterIndex(index); DecoratedKey k = null; diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java similarity index 87% rename from test/unit/org/apache/cassandra/db/KeyCacheTest.java rename to test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java index c673458d86..a1efafc3ae 100644 --- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/keycache/KeyCacheTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.io.sstable.keycache; import java.io.IOException; import java.util.ArrayList; @@ -40,16 +40,25 @@ import org.apache.cassandra.cache.AutoSavingCache; import org.apache.cassandra.cache.ICache; import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.format.big.RowIndexEntry; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Refs; +import org.assertj.core.api.Assertions; import org.hamcrest.Matchers; import org.mockito.Mockito; import org.mockito.internal.stubbing.answers.AnswersWithDelay; @@ -63,6 +72,8 @@ import static org.mockito.Mockito.mock; public class KeyCacheTest { + private static boolean sstableImplCachesKeys; + private static final String KEYSPACE1 = "KeyCacheTest1"; private static final String KEYSPACE2 = "KeyCacheTest2"; private static final String COLUMN_FAMILY1 = "Standard1"; @@ -82,6 +93,8 @@ public class KeyCacheTest @BeforeClass public static void defineSchema() throws ConfigurationException { + DatabaseDescriptor.daemonInitialization(); + sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current()); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), @@ -141,7 +154,7 @@ public class KeyCacheTest assertKeyCacheSize(100, KEYSPACE1, cf); // really? our caches don't implement the map interface? (hence no .addAll) - Map savedMap = new HashMap<>(); + Map savedMap = new HashMap<>(); Map savedInfoMap = new HashMap<>(); for (Iterator iter = CacheService.instance.keyCache.keyIterator(); iter.hasNext();) @@ -149,10 +162,13 @@ public class KeyCacheTest KeyCacheKey k = iter.next(); if (k.desc.ksname.equals(KEYSPACE1) && k.desc.cfname.equals(cf)) { - RowIndexEntry rie = CacheService.instance.keyCache.get(k); + AbstractRowIndexEntry rie = CacheService.instance.keyCache.get(k); savedMap.put(k, rie); - SSTableReader sstr = readerForKey(k); - savedInfoMap.put(k, rie.openWithIndex(sstr.getIndexFile())); + if (rie instanceof RowIndexEntry) + { + BigTableReader sstr = (BigTableReader) readerForKey(k); + savedInfoMap.put(k, ((RowIndexEntry) rie).openWithIndex(sstr.getIndexFile())); + } } } @@ -166,19 +182,24 @@ public class KeyCacheTest assertKeyCacheSize(savedMap.size(), KEYSPACE1, cf); // probably it's better to add equals/hashCode to RowIndexEntry... - for (Map.Entry entry : savedMap.entrySet()) + for (Map.Entry entry : savedMap.entrySet()) { - RowIndexEntry expected = entry.getValue(); - RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey()); + AbstractRowIndexEntry expected = entry.getValue(); + AbstractRowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey()); assertEquals(expected.position, actual.position); assertEquals(expected.columnsIndexCount(), actual.columnsIndexCount()); + assertEquals(expected.getSSTableFormat(), actual.getSSTableFormat()); for (int i = 0; i < expected.columnsIndexCount(); i++) { SSTableReader actualSstr = readerForKey(entry.getKey()); - try (RowIndexEntry.IndexInfoRetriever actualIir = actual.openWithIndex(actualSstr.getIndexFile())) + Assertions.assertThat(actualSstr.descriptor.formatType).isEqualTo(expected.getSSTableFormat().getType()); + if (actual instanceof RowIndexEntry) { - RowIndexEntry.IndexInfoRetriever expectedIir = savedInfoMap.get(entry.getKey()); - assertEquals(expectedIir.columnsIndex(i), actualIir.columnsIndex(i)); + try (RowIndexEntry.IndexInfoRetriever actualIir = ((RowIndexEntry) actual).openWithIndex(((BigTableReader) actualSstr).getIndexFile())) + { + RowIndexEntry.IndexInfoRetriever expectedIir = savedInfoMap.get(entry.getKey()); + assertEquals(expectedIir.columnsIndex(i), actualIir.columnsIndex(i)); + } } } if (expected.isIndexed()) @@ -375,7 +396,7 @@ public class KeyCacheTest // be zero after load. assertKeyCacheSize(numberOfRows, KEYSPACE1, columnFamily1); assertKeyCacheSize(numberOfRows, KEYSPACE2, columnFamily2); - assertEquals(numberOfRows * tables.size(), CacheService.instance.keyCache.size()); + assertEquals(sstableImplCachesKeys ? numberOfRows * tables.size() : 0, CacheService.instance.keyCache.size()); } @SuppressWarnings({ "unchecked", "rawtypes" }) @@ -408,8 +429,16 @@ public class KeyCacheTest long keysLoaded = autoSavingCache.loadSaved(); assertThat(keysLoaded, Matchers.lessThanOrEqualTo(maxExpectedKeyCache)); - assertNotEquals(0, keysLoaded); - Mockito.verify(keyCacheSerializerSpy, Mockito.times(1)).cleanupAfterDeserialize(); + if (sstableImplCachesKeys) + { + assertNotEquals(0, keysLoaded); + Mockito.verify(keyCacheSerializerSpy, Mockito.times(1)).cleanupAfterDeserialize(); + } + else + { + assertEquals(0, keysLoaded); + Mockito.verify(keyCacheSerializerSpy, Mockito.never()).cleanupAfterDeserialize(); + } } private void createAndInvalidateCache(List> tables, int numberOfRows) throws ExecutionException, InterruptedException @@ -464,6 +493,6 @@ public class KeyCacheTest if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily)) size++; } - assertEquals(expected, size); + assertEquals(sstableImplCachesKeys ? expected : 0, size); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 1e85f34106..64a1a2ce14 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -17,31 +17,43 @@ */ package org.apache.cassandra.io.sstable.metadata; -import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; -import org.apache.cassandra.io.util.*; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; +import org.apache.cassandra.db.marshal.AsciiType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.RandomPartitioner; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.format.big.BigFormat; -import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.io.util.DataOutputStreamPlus; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.Throwables; import static org.junit.Assert.assertEquals; @@ -50,21 +62,26 @@ import static org.junit.Assert.assertTrue; public class MetadataSerializerTest { + private final static Logger logger = LoggerFactory.getLogger(MetadataSerializerTest.class); + + private static SSTableFormat format; + @BeforeClass public static void initDD() { DatabaseDescriptor.daemonInitialization(); + format = SSTableFormat.Type.current().info; } @Test public void testSerialization() throws IOException { - Map originalMetadata = constructMetadata(); + Map originalMetadata = constructMetadata(false); MetadataSerializer serializer = new MetadataSerializer(); - File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion); + File statsFile = serialize(originalMetadata, serializer, SSTableFormat.Type.current().info.getLatestVersion()); - Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG); + Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.current()); try (RandomAccessReader in = RandomAccessReader.open(statsFile)) { Map deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class)); @@ -79,7 +96,7 @@ public class MetadataSerializerTest @Test public void testHistogramSterilization() throws IOException { - Map originalMetadata = constructMetadata(); + Map originalMetadata = constructMetadata(false); // Modify the histograms to overflow: StatsMetadata originalStats = (StatsMetadata) originalMetadata.get(MetadataType.STATS); @@ -90,8 +107,8 @@ public class MetadataSerializerTest // Serialize w/ overflowed histograms: MetadataSerializer serializer = new MetadataSerializer(); - File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion); - Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG); + File statsFile = serialize(originalMetadata, serializer, format.getLatestVersion()); + Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), format.getType()); try (RandomAccessReader in = RandomAccessReader.open(statsFile)) { @@ -107,7 +124,7 @@ public class MetadataSerializerTest throws IOException { // Serialize to tmp file - File statsFile = FileUtils.createTempFile(Component.STATS.name, null); + File statsFile = FileUtils.createTempFile(Components.STATS.name, null); try (DataOutputStreamPlus out = new FileOutputStreamPlus(statsFile)) { serializer.serialize(metadata, out, version); @@ -115,36 +132,44 @@ public class MetadataSerializerTest return statsFile; } - public Map constructMetadata() + public Map constructMetadata(boolean withNulls) { CommitLogPosition club = new CommitLogPosition(11L, 12); CommitLogPosition cllb = new CommitLogPosition(9L, 12); - TableMetadata cfm = SchemaLoader.standardCFMD("ks1", "cf1").build(); + TableMetadata cfm = TableMetadata.builder("ks1", "cf1") + .addPartitionKeyColumn("k", AsciiType.instance) + .addClusteringColumn("c1", UTF8Type.instance) + .addClusteringColumn("c2", Int32Type.instance) + .addRegularColumn("v", Int32Type.instance) + .build(); MetadataCollector collector = new MetadataCollector(cfm.comparator) .commitLogIntervals(new IntervalSet<>(cllb, club)); String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - ByteBuffer first = ByteBufferUtil.bytes(1); - ByteBuffer last = ByteBufferUtil.bytes(2); + collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("abc"), Int32Type.instance.decompose(123))); + collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("cba"), withNulls ? null : Int32Type.instance.decompose(234))); + ByteBuffer first = AsciiType.instance.decompose("a"); + ByteBuffer last = AsciiType.instance.decompose("b"); return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last); } - private void testVersions(String... versions) throws Throwable + private void testVersions(List versions) throws Throwable { + logger.info("Testing minor versions {} compatibility for sstable format {}", versions, format.getClass().getName()); Throwable t = null; - for (int oldIdx = 0; oldIdx < versions.length; oldIdx++) + for (int oldIdx = 0; oldIdx < versions.size(); oldIdx++) { - for (int newIdx = oldIdx; newIdx < versions.length; newIdx++) + for (int newIdx = oldIdx; newIdx < versions.size(); newIdx++) { try { - testOldReadsNew(versions[oldIdx], versions[newIdx]); + testOldReadsNew(versions.get(oldIdx), versions.get(newIdx)); } catch (Exception | AssertionError e) { - t = Throwables.merge(t, new AssertionError("Failed to test " + versions[oldIdx] + " -> " + versions[newIdx], e)); + t = Throwables.merge(t, new AssertionError("Failed to test " + versions.get(oldIdx) + " -> " + versions.get(newIdx), e)); } } } @@ -155,28 +180,31 @@ public class MetadataSerializerTest } @Test - public void testMVersions() throws Throwable + public void testMinorVersionsCompatibilty() throws Throwable { - testVersions("ma", "mb", "mc", "md", "me"); - } + Map> supportedVersions = new LinkedHashMap<>(); + for (char major = 'a'; major <= 'z'; major++){ + for (char minor = 'a'; minor <= 'z'; minor++){ + Version version = format.getVersion(String.format("%s%s", major, minor)); + if (version.isCompatible()) + supportedVersions.computeIfAbsent(major, ignored -> new ArrayList<>()).add(version.getVersion()); + } + } - @Test - public void testNVersions() throws Throwable - { - testVersions("na", "nb", "nc"); + for (List minorVersions : supportedVersions.values()) + testVersions(minorVersions); } public void testOldReadsNew(String oldV, String newV) throws IOException { - Map originalMetadata = constructMetadata(); + Map originalMetadata = constructMetadata(true); MetadataSerializer serializer = new MetadataSerializer(); // Write metadata in two minor formats. - File statsFileLb = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(newV)); - File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV)); + File statsFileLb = serialize(originalMetadata, serializer, format.getVersion(newV)); + File statsFileLa = serialize(originalMetadata, serializer, format.getVersion(oldV)); // Reading both as earlier version should yield identical results. - SSTableFormat.Type stype = SSTableFormat.Type.current(); - Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0), stype); + Descriptor desc = new Descriptor(format.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0), format.getType()); try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb); RandomAccessReader inLa = RandomAccessReader.open(statsFileLa)) { @@ -196,35 +224,70 @@ public class MetadataSerializerTest @Test public void pendingRepairCompatibility() { - Arrays.asList("ma", "mb", "mc", "md", "me").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPendingRepair())); - Arrays.asList("na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPendingRepair())); + if (format == BigFormat.instance) + { + Arrays.asList("ma", "mb", "mc", "md", "me").forEach(v -> assertFalse(format.getVersion(v).hasPendingRepair())); + Arrays.asList("na", "nb", "nc").forEach(v -> assertTrue(format.getVersion(v).hasPendingRepair())); + } + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } @Test public void originatingHostCompatibility() { - Arrays.asList("ma", "mb", "mc", "md", "na").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasOriginatingHostId())); - Arrays.asList("me", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasOriginatingHostId())); + if (format == BigFormat.instance) + { + Arrays.asList("ma", "mb", "mc", "md", "na").forEach(v -> assertFalse(format.getVersion(v).hasOriginatingHostId())); + Arrays.asList("me", "nb").forEach(v -> assertTrue(format.getVersion(v).hasOriginatingHostId())); + } + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } @Test public void improvedMinMaxCompatibility() { - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasImprovedMinMax())); - Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + if (format == BigFormat.instance) + { + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + } + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } @Test public void legacyMinMaxCompatiblity() { - Arrays.asList("oa").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasLegacyMinMax())); - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + if (format == BigFormat.instance) + { + Arrays.asList("oa").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + } + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } @Test public void partitionLevelDeletionPresenceMarkerCompatibility() { - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); - Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + if (format == BigFormat.instance) + { + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + } + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } } diff --git a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java index bb54f25d5d..9f98fb0dfb 100644 --- a/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java +++ b/test/unit/org/apache/cassandra/io/util/BufferedRandomAccessFileTest.java @@ -19,9 +19,6 @@ */ package org.apache.cassandra.io.util; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.utils.ByteBufferUtil; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -29,6 +26,9 @@ import java.util.Arrays; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.utils.ByteBufferUtil; + import static org.apache.cassandra.Util.expectEOF; import static org.apache.cassandra.Util.expectException; import static org.junit.Assert.assertEquals; @@ -56,84 +56,82 @@ public class BufferedRandomAccessFileTest w.sync(); // reading small amount of data from file, this is handled by initial buffer - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath())) + FileHandle.Builder builder = new FileHandle.Builder(w.getFile()); + try (FileHandle fh = builder.complete(); + RandomAccessReader r = fh.createReader()) { - try (FileHandle fh = builder.complete(); - RandomAccessReader r = fh.createReader()) - { - byte[] buffer = new byte[data.length]; - assertEquals(data.length, r.read(buffer)); - assertTrue(Arrays.equals(buffer, data)); // we read exactly what we wrote - assertEquals(r.read(), -1); // nothing more to read EOF - assert r.bytesRemaining() == 0 && r.isEOF(); + byte[] buffer = new byte[data.length]; + assertEquals(data.length, r.read(buffer)); + assertTrue(Arrays.equals(buffer, data)); // we read exactly what we wrote + assertEquals(r.read(), -1); // nothing more to read EOF + assert r.bytesRemaining() == 0 && r.isEOF(); + } + + // writing buffer bigger than page size, which will trigger reBuffer() + byte[] bigData = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE + 10]; + + for (int i = 0; i < bigData.length; i++) + bigData[i] = 'd'; + + long initialPosition = w.position(); + w.write(bigData); // writing data + assertEquals(w.position(), initialPosition + bigData.length); + assertEquals(w.length(), initialPosition + bigData.length); // file size should equals to last position + + w.sync(); + + // re-opening file in read-only mode + try (FileHandle fh = builder.complete(); + RandomAccessReader r = fh.createReader()) + { + + // reading written buffer + r.seek(initialPosition); // back to initial (before write) position + data = new byte[bigData.length]; + long sizeRead = 0; + for (int i = 0; i < data.length; i++) + { + data[i] = (byte) r.read(); + sizeRead++; } - // writing buffer bigger than page size, which will trigger reBuffer() - byte[] bigData = new byte[RandomAccessReader.DEFAULT_BUFFER_SIZE + 10]; + assertEquals(sizeRead, data.length); // read exactly data.length bytes + assertEquals(r.getFilePointer(), initialPosition + data.length); + assertEquals(r.length(), initialPosition + bigData.length); + assertTrue(Arrays.equals(bigData, data)); + assertTrue(r.bytesRemaining() == 0 && r.isEOF()); // we are at the of the file - for (int i = 0; i < bigData.length; i++) - bigData[i] = 'd'; + // test readBytes(int) method + r.seek(0); + ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length()); + assertEquals(fileContent.limit(), w.length()); + assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData)); - long initialPosition = w.position(); - w.write(bigData); // writing data - assertEquals(w.position(), initialPosition + bigData.length); - assertEquals(w.length(), initialPosition + bigData.length); // file size should equals to last position + // read the same buffer but using readFully(int) + data = new byte[bigData.length]; + r.seek(initialPosition); + r.readFully(data); + assert r.bytesRemaining() == 0 && r.isEOF(); // we should be at EOF + assertTrue(Arrays.equals(bigData, data)); - w.sync(); + // try to read past mark (all methods should return -1) + data = new byte[10]; + assertEquals(r.read(), -1); + assertEquals(r.read(data), -1); + assertEquals(r.read(data, 0, data.length), -1); - // re-opening file in read-only mode - try (FileHandle fh = builder.complete(); - RandomAccessReader r = fh.createReader()) + // test read(byte[], int, int) + r.seek(0); + data = new byte[20]; + assertEquals(15, r.read(data, 0, 15)); + assertTrue(new String(data).contains("Hellodddddddddd")); + for (int i = 16; i < data.length; i++) { - - // reading written buffer - r.seek(initialPosition); // back to initial (before write) position - data = new byte[bigData.length]; - long sizeRead = 0; - for (int i = 0; i < data.length; i++) - { - data[i] = (byte) r.read(); - sizeRead++; - } - - assertEquals(sizeRead, data.length); // read exactly data.length bytes - assertEquals(r.getFilePointer(), initialPosition + data.length); - assertEquals(r.length(), initialPosition + bigData.length); - assertTrue(Arrays.equals(bigData, data)); - assertTrue(r.bytesRemaining() == 0 && r.isEOF()); // we are at the of the file - - // test readBytes(int) method - r.seek(0); - ByteBuffer fileContent = ByteBufferUtil.read(r, (int) w.length()); - assertEquals(fileContent.limit(), w.length()); - assert ByteBufferUtil.string(fileContent).equals("Hello" + new String(bigData)); - - // read the same buffer but using readFully(int) - data = new byte[bigData.length]; - r.seek(initialPosition); - r.readFully(data); - assert r.bytesRemaining() == 0 && r.isEOF(); // we should be at EOF - assertTrue(Arrays.equals(bigData, data)); - - // try to read past mark (all methods should return -1) - data = new byte[10]; - assertEquals(r.read(), -1); - assertEquals(r.read(data), -1); - assertEquals(r.read(data, 0, data.length), -1); - - // test read(byte[], int, int) - r.seek(0); - data = new byte[20]; - assertEquals(15, r.read(data, 0, 15)); - assertTrue(new String(data).contains("Hellodddddddddd")); - for (int i = 16; i < data.length; i++) - { - assert data[i] == 0; - } - - w.finish(); + assert data[i] == 0; } + + w.finish(); } } @@ -147,8 +145,7 @@ public class BufferedRandomAccessFileTest byte[] in = generateByteArray(RandomAccessReader.DEFAULT_BUFFER_SIZE); w.write(in); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader r = fh.createReader()) { // Read it into a same size array. @@ -190,8 +187,7 @@ public class BufferedRandomAccessFileTest w.finish(); // will use cachedlength - try (FileHandle.Builder builder = new FileHandle.Builder(tmpFile.path()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(tmpFile).complete(); RandomAccessReader r = fh.createReader()) { assertEquals(lessThenBuffer.length + biggerThenBuffer.length, r.length()); @@ -214,8 +210,7 @@ public class BufferedRandomAccessFileTest w.write(data); w.sync(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader r = fh.createReader()) { @@ -245,8 +240,7 @@ public class BufferedRandomAccessFileTest w.write(data); w.finish(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader file = fh.createReader()) { file.seek(0); @@ -277,8 +271,7 @@ public class BufferedRandomAccessFileTest w.write(generateByteArray(RandomAccessReader.DEFAULT_BUFFER_SIZE * 2)); w.finish(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader file = fh.createReader()) { @@ -313,8 +306,7 @@ public class BufferedRandomAccessFileTest w.sync(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader r = fh.createReader()) { @@ -351,8 +343,7 @@ public class BufferedRandomAccessFileTest for (final int offset : Arrays.asList(0, 8)) { File file1 = writeTemporaryFile(new byte[16]); - try (FileHandle.Builder builder = new FileHandle.Builder(file1.path()).bufferSize(bufferSize); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(file1).bufferSize(bufferSize).complete(); RandomAccessReader file = fh.createReader()) { expectEOF(() -> { file.readFully(target, offset, 17); return null; }); @@ -363,8 +354,7 @@ public class BufferedRandomAccessFileTest for (final int n : Arrays.asList(1, 2, 4, 8)) { File file1 = writeTemporaryFile(new byte[16]); - try (FileHandle.Builder builder = new FileHandle.Builder(file1.path()).bufferSize(bufferSize); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(file1).bufferSize(bufferSize).complete(); RandomAccessReader file = fh.createReader()) { expectEOF(() -> { @@ -396,8 +386,7 @@ public class BufferedRandomAccessFileTest w.sync(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader r = fh.createReader()) { @@ -424,8 +413,7 @@ public class BufferedRandomAccessFileTest tmpFile.deleteOnExit(); // Create the BRAF by filename instead of by file. - try (FileHandle.Builder builder = new FileHandle.Builder(tmpFile.path()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(tmpFile).complete(); RandomAccessReader r = fh.createReader()) { assert tmpFile.path().equals(r.getPath()); @@ -459,7 +447,7 @@ public class BufferedRandomAccessFileTest //Writing to a BufferedOutputStream that is closed generates no error //Going to allow the NPE to throw to catch as a bug any use after close. Notably it won't throw NPE for a //write of a 0 length, but that is kind of a corner case - expectException(() -> { w.write(generateByteArray(1)); return null; }, NullPointerException.class); + expectException(() -> { w.write(generateByteArray(1)); return null; }, AssertionError.class); try (RandomAccessReader copy = RandomAccessReader.open(new File(r.getPath()))) { @@ -478,8 +466,7 @@ public class BufferedRandomAccessFileTest w.finish(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader file = fh.createReader()) { file.seek(10); @@ -514,8 +501,7 @@ public class BufferedRandomAccessFileTest w.write(new byte[30]); w.flush(); - try (FileHandle.Builder builder = new FileHandle.Builder(w.getPath()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(w.getFile()).complete(); RandomAccessReader r = fh.createReader()) { r.seek(10); @@ -620,4 +606,4 @@ public class BufferedRandomAccessFileTest return arr; } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java index 7194d3042f..e6b5dd0c09 100644 --- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java +++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java @@ -20,12 +20,13 @@ package org.apache.cassandra.io.util; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Random; import com.google.common.primitives.Ints; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,27 +35,37 @@ import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.io.compress.CompressedSequentialWriter; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.compress.CompressionMetadata.Chunk; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.schema.CompressionParams; +import org.apache.cassandra.utils.ByteBufferUtil; -import static org.junit.Assert.assertNull; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; - public class MmappedRegionsTest { private static final Logger logger = LoggerFactory.getLogger(MmappedRegionsTest.class); + private final int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + @BeforeClass public static void setupDD() { DatabaseDescriptor.daemonInitialization(); } + @After + public void resetMaxSegmentSize() + { + MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; + } + private static ByteBuffer allocateBuffer(int size) { ByteBuffer ret = ByteBuffer.allocate(Ints.checkedCast(size)); @@ -63,6 +74,11 @@ public class MmappedRegionsTest logger.info("Seed {}", seed); new Random(seed).nextBytes(ret.array()); + byte[] arr = ret.array(); + for (int i = 0; i < arr.length; i++) + { + arr[i] = (byte) (arr[i] & 0xf); + } return ret; } @@ -80,15 +96,14 @@ public class MmappedRegionsTest assert ret.exists(); assert ret.length() >= buffer.capacity(); return ret; - } @Test public void testEmpty() throws Exception { ByteBuffer buffer = allocateBuffer(1024); - try(ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testEmpty", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { assertTrue(regions.isEmpty()); assertTrue(regions.isValid(channel)); @@ -99,8 +114,8 @@ public class MmappedRegionsTest public void testTwoSegments() throws Exception { ByteBuffer buffer = allocateBuffer(2048); - try(ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testTwoSegments", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(1024); for (int i = 0; i < 1024; i++) @@ -133,12 +148,11 @@ public class MmappedRegionsTest @Test public void testSmallSegmentSize() throws Exception { - int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; MmappedRegions.MAX_SEGMENT_SIZE = 1024; ByteBuffer buffer = allocateBuffer(4096); - try(ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testSmallSegmentSize", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(1024); regions.extend(2048); @@ -153,22 +167,17 @@ public class MmappedRegionsTest assertEquals(SIZE + (SIZE * (i / SIZE)), region.end()); } } - finally - { - MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; - } } @Test public void testAllocRegions() throws Exception { - int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; MmappedRegions.MAX_SEGMENT_SIZE = 1024; ByteBuffer buffer = allocateBuffer(MmappedRegions.MAX_SEGMENT_SIZE * MmappedRegions.REGION_ALLOC_SIZE * 3); - try(ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testAllocRegions", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(buffer.capacity()); @@ -181,10 +190,6 @@ public class MmappedRegionsTest assertEquals(SIZE + (SIZE * (i / SIZE)), region.end()); } } - finally - { - MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; - } } @Test @@ -195,8 +200,8 @@ public class MmappedRegionsTest MmappedRegions snapshot; ChannelProxy channelCopy; - try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer)); - MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshot", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, buffer.capacity() / 4)) { // create 3 more segments, one per quater capacity regions.extend(buffer.capacity() / 2); @@ -237,8 +242,8 @@ public class MmappedRegionsTest MmappedRegions snapshot; ChannelProxy channelCopy; - try(ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testSnapshotCannotExtend", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(buffer.capacity() / 2); @@ -264,8 +269,8 @@ public class MmappedRegionsTest public void testExtendOutOfOrder() throws Exception { ByteBuffer buffer = allocateBuffer(4096); - try(ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testExtendOutOfOrder", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(4096); regions.extend(1024); @@ -285,8 +290,8 @@ public class MmappedRegionsTest public void testNegativeExtend() throws Exception { ByteBuffer buffer = allocateBuffer(1024); - try(ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer)); - MmappedRegions regions = MmappedRegions.empty(channel)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testNegativeExtend", buffer)); + MmappedRegions regions = MmappedRegions.empty(channel)) { regions.extend(-1); } @@ -295,7 +300,6 @@ public class MmappedRegionsTest @Test public void testMapForCompressionMetadata() throws Exception { - int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; MmappedRegions.MAX_SEGMENT_SIZE = 1024; ByteBuffer buffer = allocateBuffer(128 * 1024); @@ -306,41 +310,29 @@ public class MmappedRegionsTest cf.deleteOnExit(); MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); - try(SequentialWriter writer = new CompressedSequentialWriter(f, cf.absolutePath(), - null, SequentialWriterOption.DEFAULT, - CompressionParams.snappy(), sstableMetadataCollector)) + try (SequentialWriter writer = new CompressedSequentialWriter(f, cf, + null, SequentialWriterOption.DEFAULT, + CompressionParams.snappy(), sstableMetadataCollector)) { writer.write(buffer); writer.finish(); } - CompressionMetadata metadata = new CompressionMetadata(cf.absolutePath(), f.length(), true); - try(ChannelProxy channel = new ChannelProxy(f); - MmappedRegions regions = MmappedRegions.map(channel, metadata)) + CompressionMetadata metadata = CompressionMetadata.open(cf, f.length(), true); + try (ChannelProxy channel = new ChannelProxy(f); + MmappedRegions regions = MmappedRegions.map(channel, metadata)) { assertFalse(regions.isEmpty()); - int i = 0; - while(i < buffer.capacity()) + int dataOffset = 0; + while (dataOffset < buffer.capacity()) { - CompressionMetadata.Chunk chunk = metadata.chunkFor(i); - - MmappedRegions.Region region = regions.floor(chunk.offset); - assertNotNull(region); - - ByteBuffer compressedChunk = region.buffer.duplicate(); - assertNotNull(compressedChunk); - assertEquals(chunk.length + 4, compressedChunk.capacity()); - - assertEquals(chunk.offset, region.offset()); - assertEquals(chunk.offset + chunk.length + 4, region.end()); - - i += metadata.chunkLength(); + verifyChunks(f, metadata, dataOffset, regions); + dataOffset += metadata.chunkLength(); } } finally { - MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; metadata.close(); } } @@ -349,8 +341,8 @@ public class MmappedRegionsTest public void testIllegalArgForMap1() throws Exception { ByteBuffer buffer = allocateBuffer(1024); - try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer)); - MmappedRegions regions = MmappedRegions.map(channel, 0)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap1", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, 0)) { assertTrue(regions.isEmpty()); } @@ -360,8 +352,8 @@ public class MmappedRegionsTest public void testIllegalArgForMap2() throws Exception { ByteBuffer buffer = allocateBuffer(1024); - try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer)); - MmappedRegions regions = MmappedRegions.map(channel, -1L)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap2", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, -1L)) { assertTrue(regions.isEmpty()); } @@ -371,11 +363,131 @@ public class MmappedRegionsTest public void testIllegalArgForMap3() throws Exception { ByteBuffer buffer = allocateBuffer(1024); - try(ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer)); - MmappedRegions regions = MmappedRegions.map(channel, null)) + try (ChannelProxy channel = new ChannelProxy(writeFile("testIllegalArgForMap3", buffer)); + MmappedRegions regions = MmappedRegions.map(channel, null)) { assertTrue(regions.isEmpty()); } } + @Test + public void testExtendForCompressionMetadata() throws Exception + { + testExtendForCompressionMetadata(8, 4, 4, 8, 12); + testExtendForCompressionMetadata(4, 4, 4, 8, 12); + testExtendForCompressionMetadata(2, 4, 4, 8, 12); + } + + public void testExtendForCompressionMetadata(int maxSegmentSize, int chunkSize, int... writeSizes) throws Exception + { + MmappedRegions.MAX_SEGMENT_SIZE = maxSegmentSize << 10; + int size = Arrays.stream(writeSizes).sum() << 10; + + ByteBuffer buffer = allocateBuffer(size); + File f = FileUtils.createTempFile("testMapForCompressionMetadata", "1"); + f.deleteOnExit(); + + File cf = FileUtils.createTempFile(f.name() + ".metadata", "1"); + cf.deleteOnExit(); + + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); + try (CompressedSequentialWriter writer = new CompressedSequentialWriter(f, cf, null, + SequentialWriterOption.DEFAULT, + CompressionParams.deflate(chunkSize << 10), + sstableMetadataCollector)) + { + ByteBuffer slice = buffer.slice(); + slice.limit(writeSizes[0] << 10); + writer.write(slice); + writer.sync(); + + try (ChannelProxy channel = new ChannelProxy(f); + CompressionMetadata metadata = writer.open(writer.getLastFlushOffset()); + MmappedRegions regions = MmappedRegions.map(channel, metadata)) + { + assertFalse(regions.isEmpty()); + int dataOffset = 0; + while (dataOffset < metadata.dataLength) + { + verifyChunks(f, metadata, dataOffset, regions); + dataOffset += metadata.chunkLength(); + } + + int idx = 1; + int pos = writeSizes[0] << 10; + while (idx < writeSizes.length) + { + slice = buffer.slice(); + slice.position(pos).limit(pos + (writeSizes[idx] << 10)); + writer.write(slice); + writer.sync(); + + // verify that calling extend for the same (first iteration) or some previous metadata (further iterations) has no effect + assertFalse(regions.extend(metadata)); + + logger.info("Checking extend on compressed chunk for range={} {}..{} / {}", idx, pos, pos + (writeSizes[idx] << 10), size); + checkExtendOnCompressedChunks(f, writer, regions); + pos += writeSizes[idx] << 10; + idx++; + } + } + } + } + + private void checkExtendOnCompressedChunks(File f, CompressedSequentialWriter writer, MmappedRegions regions) + { + int dataOffset; + try (CompressionMetadata metadata = writer.open(writer.getLastFlushOffset())) + { + regions.extend(metadata); + assertFalse(regions.isEmpty()); + dataOffset = 0; + while (dataOffset < metadata.dataLength) + { + logger.info("Checking chunk {}..{}", dataOffset, dataOffset + metadata.chunkLength()); + verifyChunks(f, metadata, dataOffset, regions); + dataOffset += metadata.chunkLength(); + } + } + } + + private ByteBuffer fromRegions(MmappedRegions regions, int offset, int size) + { + ByteBuffer buf = ByteBuffer.allocate(size); + + while (buf.remaining() > 0) + { + MmappedRegions.Region region = regions.floor(offset); + ByteBuffer regBuf = region.buffer.slice(); + int regBufOffset = (int) (offset - region.offset); + regBuf.position(regBufOffset); + regBuf.limit(regBufOffset + Math.min(buf.remaining(), regBuf.remaining())); + offset += regBuf.remaining(); + buf.put(regBuf); + } + + buf.flip(); + return buf; + } + + private Chunk verifyChunks(File f, CompressionMetadata metadata, long dataOffset, MmappedRegions regions) + { + Chunk chunk = metadata.chunkFor(dataOffset); + + ByteBuffer compressedChunk = fromRegions(regions, (int) chunk.offset, chunk.length + 4); + assertThat(compressedChunk.capacity()).isEqualTo(chunk.length + 4); + + try (ChannelProxy channel = new ChannelProxy(f)) + { + ByteBuffer buf = ByteBuffer.allocate(compressedChunk.remaining()); + long len = channel.read(buf, chunk.offset); + assertThat(len).isEqualTo(chunk.length + 4); + buf.flip(); + String mmappedHex = ByteBufferUtil.bytesToHex(compressedChunk); + String fileHex = ByteBufferUtil.bytesToHex(buf); + assertThat(fileHex).isEqualTo(mmappedHex); + } + + return chunk; + } } diff --git a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java index f933cf1a18..853564688f 100644 --- a/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java +++ b/test/unit/org/apache/cassandra/io/util/RandomAccessReaderTest.java @@ -38,17 +38,19 @@ import java.util.concurrent.TimeUnit; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.junit.Assert.*; - import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.utils.ByteBufferUtil; +import static org.apache.cassandra.utils.Clock.Global.nanoTime; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class RandomAccessReaderTest { private static final Logger logger = LoggerFactory.getLogger(RandomAccessReaderTest.class); @@ -142,17 +144,16 @@ public class RandomAccessReaderTest Parameters params = new Parameters(SIZE, 1 << 20); // 1MiB - try (ChannelProxy channel = new ChannelProxy("abc", new FakeFileChannel(SIZE)); - FileHandle.Builder builder = new FileHandle.Builder(channel) - .bufferType(params.bufferType).bufferSize(params.bufferSize); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(new File("abc")).bufferType(params.bufferType) + .bufferSize(params.bufferSize) + .complete(f -> new ChannelProxy(f, new FakeFileChannel(SIZE))); RandomAccessReader reader = fh.createReader()) { - assertEquals(channel.size(), reader.length()); - assertEquals(channel.size(), reader.bytesRemaining()); + assertEquals(fh.channel.size(), reader.length()); + assertEquals(fh.channel.size(), reader.bytesRemaining()); assertEquals(Integer.MAX_VALUE, reader.available()); - assertEquals(channel.size(), reader.skip(channel.size())); + assertEquals(fh.channel.size(), reader.skip(fh.channel.size())); assertTrue(reader.isEOF()); assertEquals(0, reader.bytesRemaining()); @@ -290,30 +291,28 @@ public class RandomAccessReaderTest private static void testReadFully(Parameters params) throws IOException { final File f = writeFile(params); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()) - .bufferType(params.bufferType).bufferSize(params.bufferSize)) + FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType) + .bufferSize(params.bufferSize); + builder.mmapped(params.mmappedRegions); + try (FileHandle fh = builder.complete(); + RandomAccessReader reader = fh.createReader()) { - builder.mmapped(params.mmappedRegions); - try (FileHandle fh = builder.complete(); - RandomAccessReader reader = fh.createReader()) + assertEquals(f.absolutePath(), reader.getPath()); + assertEquals(f.length(), reader.length()); + assertEquals(f.length(), reader.bytesRemaining()); + assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available()); + + byte[] b = new byte[params.expected.length]; + long numRead = 0; + while (numRead < params.fileLength) { - assertEquals(f.absolutePath(), reader.getPath()); - assertEquals(f.length(), reader.length()); - assertEquals(f.length(), reader.bytesRemaining()); - assertEquals(Math.min(Integer.MAX_VALUE, f.length()), reader.available()); - - byte[] b = new byte[params.expected.length]; - long numRead = 0; - while (numRead < params.fileLength) - { - reader.readFully(b); - assertTrue(Arrays.equals(params.expected, b)); - numRead += b.length; - } - - assertTrue(reader.isEOF()); - assertEquals(0, reader.bytesRemaining()); + reader.readFully(b); + assertTrue(Arrays.equals(params.expected, b)); + numRead += b.length; } + + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); } } @@ -331,8 +330,7 @@ public class RandomAccessReaderTest assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(f).complete(); RandomAccessReader reader = fh.createReader()) { assertEquals(f.absolutePath(), reader.getPath()); @@ -362,8 +360,7 @@ public class RandomAccessReaderTest assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()); - FileHandle fh = builder.complete(); + try (FileHandle fh = new FileHandle.Builder(f).complete(); RandomAccessReader reader = fh.createReader()) { assertEquals(expected.length() * numIterations, reader.length()); @@ -442,61 +439,59 @@ public class RandomAccessReaderTest assert f.exists(); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path())) + FileHandle.Builder builder = new FileHandle.Builder(f); + final Runnable worker = () -> { - final Runnable worker = () -> + try (FileHandle fh = builder.complete(); + RandomAccessReader reader = fh.createReader()) { - try (FileHandle fh = builder.complete(); - RandomAccessReader reader = fh.createReader()) + assertEquals(expected.length, reader.length()); + + ByteBuffer b = ByteBufferUtil.read(reader, expected.length); + assertTrue(Arrays.equals(expected, b.array())); + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + + reader.seek(0); + b = ByteBufferUtil.read(reader, expected.length); + assertTrue(Arrays.equals(expected, b.array())); + assertTrue(reader.isEOF()); + assertEquals(0, reader.bytesRemaining()); + + for (int i = 0; i < 10; i++) { - assertEquals(expected.length, reader.length()); + int pos = r.nextInt(expected.length); + reader.seek(pos); + assertEquals(pos, reader.getPosition()); - ByteBuffer b = ByteBufferUtil.read(reader, expected.length); - assertTrue(Arrays.equals(expected, b.array())); - assertTrue(reader.isEOF()); - assertEquals(0, reader.bytesRemaining()); + ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos) + .order(ByteOrder.BIG_ENDIAN); - reader.seek(0); - b = ByteBufferUtil.read(reader, expected.length); - assertTrue(Arrays.equals(expected, b.array())); - assertTrue(reader.isEOF()); - assertEquals(0, reader.bytesRemaining()); - - for (int i = 0; i < 10; i++) - { - int pos = r.nextInt(expected.length); - reader.seek(pos); - assertEquals(pos, reader.getPosition()); - - ByteBuffer buf = ByteBuffer.wrap(expected, pos, expected.length - pos) - .order(ByteOrder.BIG_ENDIAN); - - while (reader.bytesRemaining() > 4) - assertEquals(buf.getInt(), reader.readInt()); - } - - reader.close(); + while (reader.bytesRemaining() > 4) + assertEquals(buf.getInt(), reader.readInt()); } - catch (Exception ex) - { - ex.printStackTrace(); - fail(ex.getMessage()); - } - }; - if (numThreads == 1) - { - worker.run(); + reader.close(); } - else + catch (Exception ex) { - ExecutorService executor = Executors.newFixedThreadPool(numThreads); - for (int i = 0; i < numThreads; i++) - executor.submit(worker); + ex.printStackTrace(); + fail(ex.getMessage()); + } + }; - executor.shutdown(); - Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); - } + if (numThreads == 1) + { + worker.run(); + } + else + { + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + for (int i = 0; i < numThreads; i++) + executor.submit(worker); + + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES)); } } @@ -519,16 +514,14 @@ public class RandomAccessReaderTest { Parameters params = new Parameters(8192, 4096); final File f = writeFile(params); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()) - .bufferType(params.bufferType).bufferSize(params.bufferSize)) + FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType) + .bufferSize(params.bufferSize) + .mmapped(params.mmappedRegions); + try (FileHandle fh = builder.complete(); + RandomAccessReader reader = fh.createReader()) { - builder.mmapped(params.mmappedRegions); - try (FileHandle fh = builder.complete(); - RandomAccessReader reader = fh.createReader()) - { - assertEquals(0, reader.skipBytes(0)); - assertEquals(0, reader.skipBytes(-1)); - } + assertEquals(0, reader.skipBytes(0)); + assertEquals(0, reader.skipBytes(-1)); } } @@ -537,42 +530,38 @@ public class RandomAccessReaderTest { Parameters params = new Parameters(8192, 4096); final File f = writeFile(params); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()) - .bufferType(params.bufferType).bufferSize(params.bufferSize)) + FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType) + .bufferSize(params.bufferSize); + try (FileHandle fh = builder.complete(); + RandomAccessReader reader = fh.createReader()) { - try (FileHandle fh = builder.complete(); - RandomAccessReader reader = fh.createReader()) - { - reader.close(); - reader.skipBytes(31415); - } + reader.close(); + reader.skipBytes(31415); } } private static void testSkipBytes(Parameters params, int expectationMultiples) throws IOException { final File f = writeFile(params); - try (FileHandle.Builder builder = new FileHandle.Builder(f.path()) - .bufferType(params.bufferType).bufferSize(params.bufferSize)) + FileHandle.Builder builder = new FileHandle.Builder(f).bufferType(params.bufferType) + .bufferSize(params.bufferSize) + .mmapped(params.mmappedRegions); + try (FileHandle fh = builder.complete(); + RandomAccessReader reader = fh.createReader()) { - builder.mmapped(params.mmappedRegions); - try (FileHandle fh = builder.complete(); - RandomAccessReader reader = fh.createReader()) - { - int toSkip = expectationMultiples * params.expected.length; - byte[] b = new byte[params.expected.length]; - long numRead = 0; + int toSkip = expectationMultiples * params.expected.length; + byte[] b = new byte[params.expected.length]; + long numRead = 0; - while (numRead < params.fileLength) - { - reader.readFully(b); - assertTrue(Arrays.equals(params.expected, b)); - numRead += b.length; - int skipped = reader.skipBytes(toSkip); - long expectedSkipped = Math.max(Math.min(toSkip, params.fileLength - numRead), 0); - assertEquals(expectedSkipped, skipped); - numRead += skipped; - } + while (numRead < params.fileLength) + { + reader.readFully(b); + assertTrue(Arrays.equals(params.expected, b)); + numRead += b.length; + int skipped = reader.skipBytes(toSkip); + long expectedSkipped = Math.max(Math.min(toSkip, params.fileLength - numRead), 0); + assertEquals(expectedSkipped, skipped); + numRead += skipped; } } } diff --git a/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java new file mode 100644 index 0000000000..0d448cde5f --- /dev/null +++ b/test/unit/org/apache/cassandra/io/util/WrappingRebuffererTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.nio.ByteBuffer; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class WrappingRebuffererTest +{ + @Test + public void testRebufferRelease() + { + TestRebufferer mock = new TestRebufferer(); + try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock) + { + }) + { + Rebufferer.BufferHolder ret = rebufferer.rebuffer(0); + assertNotNull(ret); + assertEquals(mock.buffer(), ret.buffer()); + assertEquals(mock.offset(), ret.offset()); + + ret.release(); + assertTrue(mock.released); + } + } + + @Test + public void testRebufferReleaseFailingContract() + { + TestRebufferer mock = new TestRebufferer(); + try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock) + { + }) + { + Rebufferer.BufferHolder ret1 = rebufferer.rebuffer(0); + assertNotNull(ret1); + assertEquals(mock.buffer(), ret1.buffer()); + assertEquals(mock.offset(), ret1.offset()); + + assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> rebufferer.rebuffer(1)); + ret1.release(); + + assertTrue(mock.released); + assertThatExceptionOfType(AssertionError.class).isThrownBy(ret1::buffer); + assertThatExceptionOfType(AssertionError.class).isThrownBy(ret1::offset); + } + } + + @Test + public void testRebufferReleaseFailingContractWhenClosing() + { + assertThatExceptionOfType(AssertionError.class).isThrownBy(() -> { + TestRebufferer mock = new TestRebufferer(); + try (WrappingRebufferer rebufferer = new WrappingRebufferer(mock) + { + }) + { + Rebufferer.BufferHolder ret1 = rebufferer.rebuffer(0); + assertNotNull(ret1); + assertEquals(mock.buffer(), ret1.buffer()); + assertEquals(mock.offset(), ret1.offset()); + } + }); + } + + + private static class TestRebufferer implements Rebufferer, Rebufferer.BufferHolder + { + final ByteBuffer buffer; + boolean released; + long offset; + + TestRebufferer() + { + this.buffer = ByteBuffer.allocate(0); + this.released = false; + this.offset = 0; + } + + @Override + public ChannelProxy channel() + { + return null; + } + + @Override + public ByteBuffer buffer() + { + return buffer; + } + + public long fileLength() + { + return buffer.remaining(); + } + + public double getCrcCheckChance() + { + return 0; + } + + public BufferHolder rebuffer(long position) + { + offset = position; + return this; + } + + public long offset() + { + return offset; + } + + public void release() + { + released = true; + } + + public long adjustExternal(long position) + { + return position; + } + + public long adjustInternal(long position) + { + return position; + } + + public void close() + { + // nothing + } + + public void closeReader() + { + // nothing + } + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java index e1556b7a52..16ef782e44 100644 --- a/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java +++ b/test/unit/org/apache/cassandra/schema/MigrationManagerTest.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.function.Supplier; import com.google.common.collect.ImmutableMap; -import org.apache.cassandra.io.util.File; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -43,8 +42,9 @@ import org.apache.cassandra.db.marshal.ByteType; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.utils.FBUtilities; @@ -473,7 +473,7 @@ public class MigrationManagerTest // check assertTrue(cfs.indexManager.listIndexes().isEmpty()); LifecycleTransaction.waitForDeletions(); - assertFalse(new File(desc.filenameFor(Component.DATA)).exists()); + assertFalse(desc.fileFor(Components.DATA).exists()); } @Test diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 7c095649c1..0da6e1f092 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -33,28 +33,36 @@ import com.google.common.collect.ImmutableSet; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.memtable.Memtable; import org.apache.cassandra.db.memtable.SkipListMemtable; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.IndexSummary; -import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; +import org.apache.cassandra.io.sstable.format.big.BigTableReader; +import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; +import org.apache.cassandra.io.sstable.keycache.KeyCache; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; -import org.apache.cassandra.io.util.ChannelProxy; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.Memory; -import org.apache.cassandra.utils.AlwaysPresentFilter; +import org.apache.cassandra.service.CacheService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FilterFactory; import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; @@ -117,6 +125,7 @@ public class MockSchema { return sstable(generation, size, false, cfs); } + public static SSTableReader sstable(int generation, int size, boolean keepRef, ColumnFamilyStore cfs) { return sstable(generation, size, keepRef, generation, generation, cfs); @@ -154,52 +163,66 @@ public class MockSchema public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime, long timestamp) { + SSTableFormat format = SSTableFormat.Type.current().info; Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getTableName(), - sstableId(generation), SSTableFormat.Type.BIG); - Set components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC); - for (Component component : components) + sstableId(generation), + format.getType()); + + if (format == BigFormat.getInstance()) { - File file = new File(descriptor.filenameFor(component)); - file.createFileIfNotExists(); - } - // .complete() with size to make sstable.onDiskLength work - try (FileHandle.Builder builder = new FileHandle.Builder(new ChannelProxy(tempFile)).bufferSize(size); - FileHandle fileHandle = builder.complete(size)) - { - if (size > 0) + Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); + for (Component component : components) { - try - { - File file = new File(descriptor.filenameFor(Component.DATA)); - Util.setFileLength(file, size); - } - catch (IOException e) - { - throw new RuntimeException(e); - } + File file = descriptor.fileFor(component); + file.createFileIfNotExists(); + } + // .complete() with size to make sstable.onDiskLength work + try (FileHandle fileHandle = new FileHandle.Builder(tempFile).bufferSize(size).withLengthOverride(size).complete()) + { + if (size > 0) + { + try + { + File file = descriptor.fileFor(Components.DATA); + Util.setFileLength(file, size); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); + MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator); + collector.update(new DeletionTime(timestamp, minLocalDeletionTime)); + BufferDecoratedKey first = readerBounds(firstToken); + BufferDecoratedKey last = readerBounds(lastToken); StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, first.retainable().getKey().slice(), last.retainable().getKey().slice()) + .get(MetadataType.STATS); + BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) + .setTableMetadataRef(cfs.metadata) + .setDataFile(fileHandle.sharedCopy()) + .setIndexFile(fileHandle.sharedCopy()) + .setIndexSummary(indexSummary.sharedCopy()) + .setFilter(FilterFactory.AlwaysPresent) + .setMaxDataAge(1L) + .setStatsMetadata(metadata) + .setOpenReason(SSTableReader.OpenReason.NORMAL) + .setSerializationHeader(header) + .setFirst(first) + .setLast(last) + .setKeyCache(cfs.metadata().params.caching.cacheKeys ? new KeyCache(CacheService.instance.keyCache) : KeyCache.NO_CACHE) + .build(cfs, false, false); + if (!keepRef) + reader.selfRef().release(); + return reader; } - SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); - MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator); - collector.update(new DeletionTime(timestamp, minLocalDeletionTime)); - - BufferDecoratedKey first = readerBounds(firstToken); - BufferDecoratedKey last = readerBounds(lastToken); - - StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, SSTable.getMinimalKey(first).getKey().slice(), SSTable.getMinimalKey(last).getKey().slice()) - .get(MetadataType.STATS); - SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, - fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(), - new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header); - reader.first = first ; - reader.last = last; - if (!keepRef) - reader.selfRef().release(); - return reader; } - + else + { + throw Util.testMustBeImplementedForSSTableFormat(); + } } public static ColumnFamilyStore newCFS() diff --git a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java index 5545d931f4..c4fe58a542 100644 --- a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java +++ b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java @@ -25,8 +25,11 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.junit.BeforeClass; import org.junit.Test; +import com.googlecode.concurrenttrees.common.Iterables; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; @@ -47,6 +50,12 @@ public class SSTablesGlobalTrackerTest private static final int MAX_VERSION_LIST_SIZE = 10; private static final int MAX_UPDATES_PER_GEN = 100; + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.clientInitialization(); + } + /** * Ensures that the tracker properly maintains the set of versions in use. * @@ -111,7 +120,7 @@ public class SSTablesGlobalTrackerTest private Gen sstableFormatTypes() { - return Generate.enumValues(SSTableFormat.Type.class); + return Generate.pick(Iterables.toList(SSTableFormat.Type.values())); } private Gen sstableVersionString() @@ -157,4 +166,4 @@ public class SSTablesGlobalTrackerTest '}'; } } -} \ No newline at end of file +} diff --git a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java index 619e0d92f3..391d589721 100644 --- a/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java +++ b/test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java @@ -17,14 +17,14 @@ */ package org.apache.cassandra.streaming.compression; -import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.EOFException; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; -import org.apache.cassandra.io.util.File; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -33,18 +33,21 @@ import org.junit.rules.TemporaryFolder; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.streaming.CompressedInputStream; +import org.apache.cassandra.db.streaming.CompressionInfo; import org.apache.cassandra.io.compress.CompressedSequentialWriter; import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; +import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.schema.CompressionParams; -import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.metadata.MetadataCollector; -import org.apache.cassandra.db.streaming.CompressedInputStream; -import org.apache.cassandra.db.streaming.CompressionInfo; import org.apache.cassandra.utils.ChecksumType; import static org.junit.Assert.assertEquals; @@ -121,12 +124,12 @@ public class CompressedInputStreamTest // write compressed data file of longs File parentDir = new File(tempFolder.newFolder()); Descriptor desc = new Descriptor(parentDir, "ks", "cf", new SequenceBasedSSTableId(1)); - File tmp = new File(desc.filenameFor(Component.DATA)); + File tmp = desc.fileFor(Components.DATA); MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); CompressionParams param = CompressionParams.snappy(32, minCompressRatio); Map index = new HashMap(); try (CompressedSequentialWriter writer = new CompressedSequentialWriter(tmp, - desc.filenameFor(Component.COMPRESSION_INFO), + desc.fileFor(Components.COMPRESSION_INFO), null, SequentialWriterOption.DEFAULT, param, collector)) @@ -139,7 +142,7 @@ public class CompressedInputStreamTest writer.finish(); } - CompressionMetadata comp = CompressionMetadata.create(tmp.absolutePath()); + CompressionMetadata comp = CompressionInfoComponent.load(desc); List sections = new ArrayList<>(); for (long l : valuesToCheck) { @@ -184,7 +187,7 @@ public class CompressedInputStreamTest testException(sections, info); return; } - CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(toRead)), info, ChecksumType.CRC32, () -> 1.0); + CompressedInputStream input = new CompressedInputStream(new DataInputBuffer(toRead), info, ChecksumType.CRC32, () -> 1.0); try (DataInputStream in = new DataInputStream(input)) { @@ -199,7 +202,7 @@ public class CompressedInputStreamTest private static void testException(List sections, CompressionInfo info) throws IOException { - CompressedInputStream input = new CompressedInputStream(new DataInputStreamPlus(new ByteArrayInputStream(new byte[0])), info, ChecksumType.CRC32, () -> 1.0); + CompressedInputStream input = new CompressedInputStream(new DataInputBuffer(new byte[0]), info, ChecksumType.CRC32, () -> 1.0); try (DataInputStream in = new DataInputStream(input)) { @@ -218,4 +221,3 @@ public class CompressedInputStreamTest } } } - diff --git a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java index e9d2070570..180e0239be 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java @@ -18,34 +18,31 @@ package org.apache.cassandra.tools; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.cassandra.io.util.File; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.io.sstable.LegacySSTableTest; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tools.ToolRunner.ToolResult; import org.assertj.core.api.Assertions; -import org.assertj.core.util.Lists; import static org.junit.Assert.assertEquals; /* * SStableUpdater should be run with the server shutdown, but we need to set up a certain env to be able to * load/swap/drop sstable files under the test's feet. Hence why we need a separate file vs StandaloneUpgraderTest. - * + * * Caution: heavy hacking ahead. */ public class StandaloneUpgraderOnSStablesTest @@ -76,9 +73,7 @@ public class StandaloneUpgraderOnSStablesTest "-k", "legacy_tables", "legacy_" + legacyId + "_simple"); - Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading."); - Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple"); - Assertions.assertThat(tool.getStdout()).contains("-Data.db"); + checkUpgradeToolOutput(tool, origFiles); tool.assertOnCleanExit(); List newFiles = getSStableFiles("legacy_tables", "legacy_" + legacyId + "_simple"); @@ -105,13 +100,19 @@ public class StandaloneUpgraderOnSStablesTest "wrongsnapshot"); Assertions.assertThat(tool.getStdout()).contains("Found 0 sstables that need upgrading."); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists("legacy_tables", "legacy_" + legacyId + "_simple"); + List names = cfs.getDirectories() + .sstableLister(Directories.OnTxnErr.IGNORE) + .snapshots("testsnapshot").list().keySet().stream() + .map(descriptor -> descriptor.baseFile().toString()) + .collect(Collectors.toList()); + tool = ToolRunner.invokeClass(StandaloneUpgrader.class, "legacy_tables", "legacy_" + legacyId + "_simple", "testsnapshot"); - Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading."); - Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple"); - Assertions.assertThat(tool.getStdout()).contains("-Data.db"); + checkUpgradeToolOutput(tool, names); + tool.assertOnCleanExit(); } @@ -125,9 +126,8 @@ public class StandaloneUpgraderOnSStablesTest ToolResult tool = ToolRunner.invokeClass(StandaloneUpgrader.class, "legacy_tables", "legacy_" + legacyId + "_simple"); - Assertions.assertThat(tool.getStdout()).contains("Found 1 sstables that need upgrading."); - Assertions.assertThat(tool.getStdout()).contains("legacy_tables/legacy_" + legacyId + "_simple"); - Assertions.assertThat(tool.getStdout()).contains("-Data.db"); + + checkUpgradeToolOutput(tool, origFiles); tool.assertOnCleanExit(); List newFiles = getSStableFiles("legacy_tables", "legacy_" + legacyId + "_simple"); @@ -138,22 +138,25 @@ public class StandaloneUpgraderOnSStablesTest Keyspace.open("legacy_tables").getColumnFamilyStore("legacy_" + legacyId + "_simple").loadNewSSTables(); } + private static void checkUpgradeToolOutput(ToolResult tool, List names) + { + Assertions.assertThat(tool.getStdout()).contains("Found " + names.size() + " sstables that need upgrading."); + for (String name : names) + { + Assertions.assertThat(tool.getStdout()).matches("(?s).*Upgrading.*" + Pattern.quote(name) + ".*"); + Assertions.assertThat(tool.getStdout()).matches("(?s).*Upgrade of.*" + Pattern.quote(name) + ".*complete.*"); + } + } + private List getSStableFiles(String ks, String table) throws StartupException { ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(table); org.apache.cassandra.Util.flush(cfs); ColumnFamilyStore.scrubDataDirectories(cfs.metadata()); - Set sstables = cfs.getLiveSSTables(); - if (sstables.isEmpty()) - return Lists.emptyList(); - - String sstableFileName = sstables.iterator().next().getFilename(); - File sstablesDir = new File(sstableFileName).parent(); - return Arrays.asList(sstablesDir.tryList()) - .stream() - .filter(f -> f.isFile()) - .map(file -> file.toString()) - .collect(Collectors.toList()); + return cfs.getDirectories() + .sstableLister(Directories.OnTxnErr.IGNORE).list().keySet().stream() + .map(descriptor -> descriptor.baseFile().toString()) + .collect(Collectors.toList()); } } diff --git a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java index 65dd1255be..8512abf76c 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java @@ -33,8 +33,10 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.VerifyTest; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigTableVerifier; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tools.ToolRunner.ToolResult; @@ -47,13 +49,13 @@ import static org.junit.Assert.assertEquals; /** * Class that tests tables for {@link StandaloneVerifier} by updating using {@link SchemaLoader} * Similar in vein to other {@link SchemaLoader} type tests, as well as {@link StandaloneUpgraderOnSStablesTest}. - * Since the tool mainly exercises the {@link org.apache.cassandra.db.compaction.Verifier}, we elect to - * not run every conceivable option as many tests are already covered by {@link org.apache.cassandra.db.VerifyTest}. + * Since the tool mainly exercises the {@link BigTableVerifier}, we elect to + * not run every conceivable option as many tests are already covered by {@link VerifyTest}. * * Note: the complete coverage is composed of: * - {@link StandaloneVerifierOnSSTablesTest} * - {@link StandaloneVerifierTest} - * - {@link org.apache.cassandra.db.VerifyTest} + * - {@link VerifyTest} */ public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils { @@ -129,7 +131,7 @@ public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils String corruptStatsTable = "corruptStatsTable"; createAndPopulateTable(keyspaceName, corruptStatsTable, cfs -> { SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.STATS), "rw")) + try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.fileFor(Components.STATS).toJavaIOFile(), "rw")) { file.seek(0); file.writeBytes(StringUtils.repeat('z', 2)); @@ -150,8 +152,8 @@ public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils createAndPopulateTable(keyspaceName, corruptDataTable, cfs -> { SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position; - long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position; + long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ); + long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ); long startPosition = Math.min(row0Start, row1Start); try (RandomAccessFile file = new RandomAccessFile(sstable.getFilename(), "rw")) diff --git a/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java b/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java index 9d8f797982..f94a5d6b3b 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import org.junit.Test; +import org.apache.cassandra.io.sstable.VerifyTest; import org.apache.cassandra.tools.ToolRunner.ToolResult; import org.assertj.core.api.Assertions; import org.hamcrest.CoreMatchers; @@ -33,7 +34,7 @@ import static org.junit.Assert.assertThat; * Note: the complete coverage is composed of: * - {@link StandaloneVerifierOnSSTablesTest} * - {@link StandaloneVerifierTest} - * - {@link org.apache.cassandra.db.VerifyTest} + * - {@link VerifyTest} */ public class StandaloneVerifierTest extends OfflineToolUtils { @@ -219,4 +220,4 @@ public class StandaloneVerifierTest extends OfflineToolUtils assertEquals(1, tool.getExitCode()); }); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/ScrubToolTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java similarity index 86% rename from test/unit/org/apache/cassandra/db/ScrubToolTest.java rename to test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java index 280810c27b..7c69839d5f 100644 --- a/test/unit/org/apache/cassandra/db/ScrubToolTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/ScrubToolTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.tools.nodetool; import java.io.IOError; import java.io.IOException; @@ -27,17 +27,21 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.marshal.UUIDType; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.ScrubTest; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.tools.StandaloneScrubber; import org.apache.cassandra.tools.ToolRunner; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Throwables; import org.assertj.core.api.Assertions; import static org.apache.cassandra.SchemaLoader.counterCFMD; @@ -45,19 +49,18 @@ import static org.apache.cassandra.SchemaLoader.createKeyspace; import static org.apache.cassandra.SchemaLoader.getCompressionParameters; import static org.apache.cassandra.SchemaLoader.loadSchema; import static org.apache.cassandra.SchemaLoader.standardCFMD; -import static org.apache.cassandra.db.ScrubTest.CF_INDEX1; -import static org.apache.cassandra.db.ScrubTest.CF_INDEX1_BYTEORDERED; -import static org.apache.cassandra.db.ScrubTest.CF_INDEX2; -import static org.apache.cassandra.db.ScrubTest.CF_INDEX2_BYTEORDERED; -import static org.apache.cassandra.db.ScrubTest.CF_UUID; -import static org.apache.cassandra.db.ScrubTest.COMPRESSION_CHUNK_LENGTH; -import static org.apache.cassandra.db.ScrubTest.COUNTER_CF; -import static org.apache.cassandra.db.ScrubTest.assertOrderedAll; -import static org.apache.cassandra.db.ScrubTest.fillCF; -import static org.apache.cassandra.db.ScrubTest.fillCounterCF; -import static org.apache.cassandra.db.ScrubTest.overrideWithGarbage; +import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX1; +import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX1_BYTEORDERED; +import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX2; +import static org.apache.cassandra.io.sstable.ScrubTest.CF_INDEX2_BYTEORDERED; +import static org.apache.cassandra.io.sstable.ScrubTest.CF_UUID; +import static org.apache.cassandra.io.sstable.ScrubTest.COMPRESSION_CHUNK_LENGTH; +import static org.apache.cassandra.io.sstable.ScrubTest.COUNTER_CF; +import static org.apache.cassandra.io.sstable.ScrubTest.assertOrderedAll; +import static org.apache.cassandra.io.sstable.ScrubTest.fillCF; +import static org.apache.cassandra.io.sstable.ScrubTest.fillCounterCF; +import static org.apache.cassandra.io.sstable.ScrubTest.overrideWithGarbage; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class ScrubToolTest @@ -121,7 +124,7 @@ public class ScrubToolTest assertEquals(1, cfs.getLiveSSTables().size()); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1")); + ScrubTest.overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x7A); // with skipCorrupted == true, the corrupt rows will be skipped ToolRunner.ToolResult tool = ToolRunner.invokeClass(StandaloneScrubber.class, "-s", ksName, COUNTER_CF); @@ -143,7 +146,9 @@ public class ScrubToolTest assertEquals(1, cfs.getLiveSSTables().size()); SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); - overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1")); + //use 0x00 instead of the usual 0x7A because if by any chance it's able to iterate over the corrupt + //section, then we get many out-of-order errors, which we don't want + overrideWithGarbage(sstable, ByteBufferUtil.bytes("0"), ByteBufferUtil.bytes("1"), (byte) 0x0); // with skipCorrupted == false, the scrub is expected to fail try @@ -151,8 +156,9 @@ public class ScrubToolTest ToolRunner.invokeClass(StandaloneScrubber.class, ksName, COUNTER_CF); fail("Expected a CorruptSSTableException to be thrown"); } - catch (IOError err) { - assertTrue(err.getCause() instanceof CorruptSSTableException); + catch (IOError err) + { + Throwables.assertAnyCause(err, CorruptSSTableException.class); } } @@ -246,4 +252,4 @@ public class ScrubToolTest tool.assertOnCleanExit(); assertOrderedAll(cfs, 1); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java index 96464c91f1..7f08d6ccf2 100644 --- a/test/unit/org/apache/cassandra/utils/BloomFilterTest.java +++ b/test/unit/org/apache/cassandra/utils/BloomFilterTest.java @@ -18,10 +18,7 @@ */ package org.apache.cassandra.utils; -import org.apache.cassandra.io.util.*; - import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashSet; @@ -29,10 +26,20 @@ import java.util.Iterator; import java.util.Random; import java.util.Set; -import org.junit.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.IFilter.FilterKey; import org.apache.cassandra.utils.KeyGenerator.RandomStringGenerator; import org.apache.cassandra.utils.obs.IBitSet; @@ -55,11 +62,11 @@ public class BloomFilterTest } else { - BloomFilterSerializer.serialize((BloomFilter) f, out); + BloomFilterSerializer.forVersion(false).serialize((BloomFilter) f, out); } ByteArrayInputStream in = new ByteArrayInputStream(out.getData(), 0, out.getLength()); - IFilter f2 = BloomFilterSerializer.deserialize(new DataInputStream(in), oldBfFormat); + IFilter f2 = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(Util.DataInputStreamPlusImpl.wrap(in)); assert f2.isPresent(FilterTestHelper.bytes("a")); assert !f2.isPresent(FilterTestHelper.bytes("b")); @@ -210,13 +217,14 @@ public class BloomFilterTest File file = FileUtils.createDeletableTempFile("bloomFilterTest-", ".dat"); BloomFilter filter = (BloomFilter) FilterFactory.getFilter(((long) Integer.MAX_VALUE / 8) + 1, 0.01d); filter.add(FilterTestHelper.wrap(test)); - DataOutputStreamPlus out = new FileOutputStreamPlus(file); - BloomFilterSerializer.serialize(filter, out); + FileOutputStreamPlus out = file.newOutputStream(File.WriteMode.OVERWRITE); + BloomFilterSerializer serializer = BloomFilterSerializer.forVersion(false); + serializer.serialize(filter, out); out.close(); filter.close(); - DataInputStream in = new DataInputStream(new FileInputStreamPlus(file)); - BloomFilter filter2 = BloomFilterSerializer.deserialize(in, false); + FileInputStreamPlus in = file.newInputStream(); + BloomFilter filter2 = BloomFilterSerializer.forVersion(false).deserialize(in); Assert.assertTrue(filter2.isPresent(FilterTestHelper.wrap(test))); FileUtils.closeQuietly(in); filter2.close(); diff --git a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java index 7693b45a02..f400461150 100644 --- a/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java +++ b/test/unit/org/apache/cassandra/utils/BytesReadTrackerTest.java @@ -18,10 +18,6 @@ */ package org.apache.cassandra.utils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -29,11 +25,18 @@ import java.io.DataOutputStream; import org.junit.Test; +import org.apache.cassandra.Util; import org.apache.cassandra.io.util.BytesReadTracker; +import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import org.apache.cassandra.io.util.TrackedDataInputPlus; import org.apache.cassandra.io.util.TrackedInputStream; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class BytesReadTrackerTest { @@ -99,9 +102,9 @@ public class BytesReadTrackerTest out.close(); } - DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData)); - BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in); - DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker; + DataInputStreamPlus in = new DataInputBuffer(testData); + BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in); + DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker; try { @@ -172,9 +175,9 @@ public class BytesReadTrackerTest out.close(); } - DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData)); - BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in); - DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker; + DataInputStreamPlus in = new DataInputBuffer(testData); + BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in); + DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker; try { @@ -200,9 +203,9 @@ public class BytesReadTrackerTest String testStr = "1234567890"; byte[] testData = testStr.getBytes(); - DataInputPlus.DataInputStreamPlus in = new DataInputPlus.DataInputStreamPlus(new ByteArrayInputStream(testData)); - BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in); - DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker; + DataInputStreamPlus in = new DataInputBuffer(testData); + BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in); + DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker; try { @@ -233,8 +236,8 @@ public class BytesReadTrackerTest public void internalTestReadLine(boolean inputStream) throws Exception { DataInputStream in = new DataInputStream(new ByteArrayInputStream("1".getBytes())); - BytesReadTracker tracker = inputStream? new TrackedInputStream(in) : new TrackedDataInputPlus(in); - DataInputPlus reader = inputStream? new DataInputPlus.DataInputStreamPlus((TrackedInputStream)tracker) : (DataInputPlus) tracker; + BytesReadTracker tracker = inputStream ? new TrackedInputStream(in) : new TrackedDataInputPlus(in); + DataInputPlus reader = inputStream ? Util.DataInputStreamPlusImpl.wrap((TrackedInputStream) tracker) : (DataInputPlus) tracker; try { diff --git a/test/unit/org/apache/cassandra/utils/SerializationsTest.java b/test/unit/org/apache/cassandra/utils/SerializationsTest.java index be235e6886..e23bb38832 100644 --- a/test/unit/org/apache/cassandra/utils/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/utils/SerializationsTest.java @@ -18,10 +18,8 @@ */ package org.apache.cassandra.utils; -import java.io.DataInputStream; import java.io.IOException; -import org.apache.cassandra.io.util.FileInputStreamPlus; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -31,12 +29,12 @@ import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.DataOutputStreamPlus; -import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.utils.obs.OffHeapBitSet; - import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.utils.obs.OffHeapBitSet; public class SerializationsTest extends AbstractSerializationsTester { @@ -65,7 +63,7 @@ public class SerializationsTest extends AbstractSerializationsTester if (oldBfFormat) serializeOldBfFormat((BloomFilter) bf, out); else - BloomFilterSerializer.serialize((BloomFilter) bf, out); + BloomFilterSerializer.forVersion(false).serialize((BloomFilter) bf, out); } } } @@ -80,7 +78,7 @@ public class SerializationsTest extends AbstractSerializationsTester } try (FileInputStreamPlus in = getInput("4.0", "utils.BloomFilter1000.bin"); - IFilter filter = BloomFilterSerializer.deserialize(in, false)) + IFilter filter = BloomFilterSerializer.forVersion(false).deserialize(in)) { boolean present; for (int i = 0 ; i < 1000 ; i++) @@ -96,7 +94,7 @@ public class SerializationsTest extends AbstractSerializationsTester } try (FileInputStreamPlus in = getInput("3.0", "utils.BloomFilter1000.bin"); - IFilter filter = BloomFilterSerializer.deserialize(in, true)) + IFilter filter = BloomFilterSerializer.forVersion(true).deserialize(in)) { boolean present; for (int i = 0 ; i < 1000 ; i++) @@ -118,12 +116,12 @@ public class SerializationsTest extends AbstractSerializationsTester testBloomFilterTable("test/data/bloom-filter/la/foo/la-1-big-Filter.db", true); } - private static void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception + private void testBloomFilterTable(String file, boolean oldBfFormat) throws Exception { Murmur3Partitioner partitioner = new Murmur3Partitioner(); - try (DataInputStream in = new DataInputStream(new FileInputStreamPlus(new File(file))); - IFilter filter = BloomFilterSerializer.deserialize(in, oldBfFormat)) + try (FileInputStreamPlus in = new File(file).newInputStream(); + IFilter filter = BloomFilterSerializer.forVersion(oldBfFormat).deserialize(in)) { for (int i = 1; i <= 10; i++) { diff --git a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java index e99d3683d9..d50e408d4d 100644 --- a/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java +++ b/test/unit/org/apache/cassandra/utils/concurrent/RefCountedTest.java @@ -18,28 +18,41 @@ */ package org.apache.cassandra.utils.concurrent; -import org.junit.Test; - -import org.junit.Assert; - -import org.apache.cassandra.io.util.File; import java.lang.ref.WeakReference; +import java.time.Duration; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ObjectSizes; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.concurrent.Ref.Visitor; +import org.awaitility.Awaitility; + +import static org.assertj.core.api.Assertions.assertThat; @SuppressWarnings({"unused", "unchecked", "rawtypes"}) public class RefCountedTest @@ -406,4 +419,134 @@ public class RefCountedTest Assert.assertTrue(visitor.haveLoops.isEmpty()); } + + static class LambdaTestClassTidier implements RefCounted.Tidy + { + Runnable runOnClose; + + @Override + public void tidy() throws Exception + { + runOnClose.run(); + } + + @Override + public String name() + { + return "42"; + } + } + + static class LambdaTestClass + { + String a = "x"; + Ref ref; + + Runnable getRunOnCloseLambda() + { + return () -> System.out.println("aaa"); + } + + Runnable getRunOnCloseInner() + { + return new Runnable() + { + @Override + public void run() + { + System.out.println("aaa"); + } + }; + } + + Runnable getRunOnCloseLambdaWithThis() + { + return () -> System.out.println(a); + } + } + + private Set testCycles(Function runOnCloseSupplier) + { + LambdaTestClass test = new LambdaTestClass(); + Runnable weakRef = runOnCloseSupplier.apply(test); + RefCounted.Tidy tidier = new RefCounted.Tidy() + { + Runnable ref = weakRef; + + public void tidy() + { + } + + public String name() + { + return "42"; + } + }; + + Ref ref = new Ref(test, tidier); + test.ref = ref; + + Visitor visitor = new Visitor(); + visitor.haveLoops = new HashSet<>(); + visitor.run(); + ref.close(); + + return visitor.haveLoops; + } + + /** + * The intention of this test is to confirm that lambda without a reference to `this`, does not implicitly + * include a reference to the enclosing class. If it did, we would detect cycles, as we do when we deal with + * anonymous inner classes or lamba which references `this` explicitly (see the sanity checks). + *

+ * This test aims to confirm that JVM works as we assumed because we couldn't find that in the specification. + */ + @Test + public void testCycles() + { + assertThat(testCycles(LambdaTestClass::getRunOnCloseLambdaWithThis)).isNotEmpty(); // sanity test + assertThat(testCycles(LambdaTestClass::getRunOnCloseInner)).isNotEmpty(); // sanity test + + assertThat(testCycles(LambdaTestClass::getRunOnCloseLambda)).isEmpty(); + } + + @Test + public void testSSTableReaderLeakIsDetected() + { + DatabaseDescriptor.clientInitialization(); + DatabaseDescriptor.setPartitionerUnsafe(ByteOrderedPartitioner.instance); + Descriptor descriptor = Descriptor.fromFileWithComponent(new File("test/data/legacy-sstables/nb/legacy_tables/legacy_nb_simple/nb-1-big-Data.db"), false).left; + TableMetadata tm = TableMetadata.builder("legacy_tables", "legacy_nb_simple").addPartitionKeyColumn("pk", UTF8Type.instance).addRegularColumn("val", UTF8Type.instance).build(); + AtomicBoolean leakDetected = new AtomicBoolean(); + AtomicBoolean runOnCloseExecuted1 = new AtomicBoolean(); + AtomicBoolean runOnCloseExecuted2 = new AtomicBoolean(); + Ref.OnLeak prevOnLeak = Ref.ON_LEAK; + + try + { + Ref.ON_LEAK = state -> { + leakDetected.set(true); + }; + { + SSTableReader reader = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(tm)); + reader.runOnClose(() -> runOnCloseExecuted1.set(true)); + reader.runOnClose(() -> runOnCloseExecuted2.set(true)); // second time to actually create lambda referencing to lambda, see runOnClose impl + //noinspection UnusedAssignment + reader = null; // this is required, otherwise GC will not attempt to collect the created reader + } + Awaitility.await().atMost(Duration.ofSeconds(30)).pollDelay(Duration.ofSeconds(1)).untilAsserted(() -> { + System.gc(); + System.gc(); + System.gc(); + System.gc(); + assertThat(leakDetected.get()).isTrue(); + }); + assertThat(runOnCloseExecuted1.get()).isTrue(); + assertThat(runOnCloseExecuted2.get()).isTrue(); + } + finally + { + Ref.ON_LEAK = prevOnLeak; + } + } } diff --git a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java index 038488085e..3397426500 100644 --- a/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java +++ b/test/unit/org/apache/cassandra/utils/vint/VIntCodingTest.java @@ -18,20 +18,18 @@ */ package org.apache.cassandra.utils.vint; -import com.google.common.primitives.UnsignedInteger; -import org.apache.cassandra.io.util.DataInputBuffer; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputBuffer; -import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus; -import org.junit.Test; - -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; +import com.google.common.primitives.UnsignedInteger; +import org.junit.Test; + +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -120,8 +118,7 @@ public class VIntCodingTest // read as ByteBuffer assertEquals(val, VIntCoding.getUnsignedVInt(out.buffer(), 0)); // read as DataInput - InputStream is = new ByteArrayInputStream(out.toByteArray()); - assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is))); + assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(out.toByteArray()))); } } } @@ -142,8 +139,7 @@ public class VIntCodingTest // read as ByteBuffer assertEquals(val, VIntCoding.getUnsignedVInt(ByteBuffer.wrap(baos.toByteArray()), 0)); // read as DataInput - InputStream is = new ByteArrayInputStream(baos.toByteArray()); - assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is))); + assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(baos.toByteArray()))); } } } @@ -160,8 +156,7 @@ public class VIntCodingTest // read as ByteBuffer assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0)); // read as DataInput - InputStream is = new ByteArrayInputStream(bb.array()); - assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is))); + assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(bb.array()))); } } @@ -175,8 +170,7 @@ public class VIntCodingTest // read as ByteBuffer assertEquals(val, VIntCoding.getUnsignedVInt(bb, 0)); // read as DataInput - InputStream is = new ByteArrayInputStream(bb.array()); - assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputPlus.DataInputStreamPlus(is))); + assertEquals(val, VIntCoding.readUnsignedVInt(new DataInputBuffer(bb.array()))); } @Test diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java index 0e7eec5a7f..9829d56021 100644 --- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java +++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java @@ -21,7 +21,12 @@ package org.apache.cassandra.stress; import java.io.File; import java.io.IOError; import java.net.URI; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -31,7 +36,11 @@ import javax.inject.Inject; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Uninterruptibles; -import io.airlift.airline.*; +import io.airlift.airline.Cli; +import io.airlift.airline.Command; +import io.airlift.airline.Help; +import io.airlift.airline.HelpOption; +import io.airlift.airline.Option; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; @@ -41,9 +50,10 @@ import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.StressCQLSSTableWriter; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.StressCQLSSTableWriter; +import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; @@ -131,7 +141,7 @@ public abstract class CompactionStress implements Runnable for (Map.Entry> entry : lister.list().entrySet()) { Set components = entry.getValue(); - if (!components.contains(Component.DATA)) + if (!components.contains(Components.DATA)) continue; try