diff --git a/CHANGES.txt b/CHANGES.txt index c1735755c8..5fdc11b760 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Fix sstable formats configuration (CASSANDRA-18441) * Add guardrail to bound timestamps (CASSANDRA-18352) * Add keyspace_name column to system_views.clients (CASSANDRA-18525) * Moved system properties and envs to CassandraRelevantProperties and CassandraRelevantEnv respectively (CASSANDRA-17797) @@ -160,6 +161,7 @@ Merged from 3.0: * Do not remove SSTables when cause of FSReadError is OutOfMemoryError while using best_effort disk failure policy (CASSANDRA-18336) * Do not remove truncated_at entry in system.local while dropping an index (CASSANDRA-18105) + 4.0.9 * Update zstd-jni library to version 1.5.5 (CASSANDRA-18429) * Backport CASSANDRA-17205 to 4.0 branch - Remove self-reference in SSTableTidier (CASSANDRA-18332) @@ -176,7 +178,8 @@ Merged from 3.0: * Do not remove SSTables when cause of FSReadError is OutOfMemoryError while using best_effort disk failure policy (CASSANDRA-18336) * Do not remove truncated_at entry in system.local while dropping an index (CASSANDRA-18105) * Save host id to system.local and flush immediately after startup (CASSANDRA-18153) - + + 4.1.1 * Deprecate org.apache.cassandra.hadoop code (CASSANDRA-16984) * Fix too early schema version change in sysem local table (CASSANDRA-18291) @@ -220,6 +223,7 @@ Merged from 3.0: * Avoid anticompaction mixing data from two different time windows with TWCS (CASSANDRA-17970) * Do not spam the logs with MigrationCoordinator not being able to pull schemas (CASSANDRA-18096) + 4.1.0 * Fix ContentionStrategy backoff and Clock.waitUntil (CASSANDRA-18086) Merged from 4.0: @@ -256,6 +260,7 @@ Merged from 3.0: * Suppress CVE-2019-2684 (CASSANDRA-17965) * Fix auto-completing "WITH" when creating a materialized view (CASSANDRA-17879) + 4.1-beta1 * We should not emit deprecation warning on startup for `key_cache_save_period`, `row_cache_save_period`, `counter_cache_save_period` (CASSANDRA-17904) * upsert with adder support is not consistent with numbers and strings in LWT (CASSANDRA-17857) @@ -318,6 +323,7 @@ Merged from 3.0: * Fix scrubber falling into infinite loop when the last partition is broken (CASSANDRA-17862) * Fix resetting schema (CASSANDRA-17819) + 4.1-alpha1 * Handle config parameters upper bound on startup; Fix auto_snapshot_ttl and paxos_purge_grace_period min unit validations (CASSANDRA-17571) * Fix leak of non-standard Java types in our Exceptions as clients using JMX are unable to handle them. @@ -496,6 +502,7 @@ Merged from 3.0: * Suppress CVE-2022-24823 (CASSANDRA-17633) * fsync TOC and digest files (CASSANDRA-10709) + 4.0.4 * Optimise BTree build,update and transform operations (CASSANDRA-15510) * Clean up schema migration coordinator and tests (CASSANDRA-17533) @@ -535,6 +542,7 @@ Merged from 3.0: * Lazy transaction log replica creation allows incorrect replica content divergence during anticompaction (CASSANDRA-17273) * LeveledCompactionStrategy disk space check improvements (CASSANDRA-17272) + 4.0.3 * Deprecate otc_coalescing_strategy, otc_coalescing_window_us, otc_coalescing_enough_coalesced_messages, otc_backlog_expiration_interval_ms (CASSANDRA-17377) @@ -645,6 +653,7 @@ Merged from 3.0: * Ensure java executable is on the path (CASSANDRA-14325) * Clean transaction log leftovers at the beginning of sstablelevelreset and sstableofflinerelevel (CASSANDRA-12519) + 4.0.0 * Avoid signaling DigestResolver until the minimum number of responses are guaranteed to be visible (CASSANDRA-16807) * Fix pre-4.0 FWD_FRM parameter serializer (CASSANDRA-16808) @@ -652,6 +661,7 @@ Merged from 3.0: * Fix CassandraVersion::compareTo (CASSANDRA-16794) * BinLog does not close chronicle queue leaving this to GC to cleanup (CASSANDRA-16774) + 4.0-rc2 * Improved password obfuscation (CASSANDRA-16801) * Avoid memoizing the wrong min cluster version during upgrades (CASSANDRA-16759) @@ -687,6 +697,7 @@ Merged from 3.0: * Prevent loss of commit log data when moving sstables between nodes (CASSANDRA-16619) * Fix materialized view builders inserting truncated data (CASSANDRA-16567) + 4.0-rc1 * Allow for setting buffer max capacity to increase it dynamically as needed (CASSANDRA-16524) * Harden internode message resource limit accounting against serialization failures (CASSANDRA-16616) @@ -768,6 +779,7 @@ Merged from 2.2: * Fix centos packaging for arm64, >=4.0 rpm's now require python3 (CASSANDRA-16477) * Make TokenMetadata's ring version increments atomic (CASSANDRA-16286) + 4.0-beta4 * DROP COMPACT STORAGE should invalidate prepared statements still using CompactTableMetadata (CASSANDRA-16361) * Update default num_tokens to 16 and allocate_tokens_for_local_replication_factor to 3 (CASSANDRA-13701) @@ -815,6 +827,7 @@ Merged from 3.0: Merged from 2.2: * Fix the histogram merge of the table metrics (CASSANDRA-16259) + 4.0-beta3 * Segregate Network and Chunk Cache BufferPools and Recirculate Partially Freed Chunks (CASSANDRA-15229) * Fail truncation requests when they fail on a replica (CASSANDRA-16208) @@ -857,6 +870,7 @@ Merged from 2.2: * Automatically drop compact storage on tables for which it is safe (CASSANDRA-16048) * Fixed NullPointerException for COMPACT STORAGE tables with null clustering (CASSANDRA-16241) + 4.0-beta2 * Add addition incremental repair visibility to nodetool repair_admin (CASSANDRA-14939) * Always access system properties and environment variables via the new CassandraRelevantProperties and CassandraRelevantEnv classes (CASSANDRA-15876) @@ -895,6 +909,7 @@ Merged from 2.2: Merged from 2.1: * Only allow strings to be passed to JMX authentication (CASSANDRA-16077) + 4.0-beta1 * Remove BackPressureStrategy (CASSANDRA-15375) * Improve messaging on indexing frozen collections (CASSANDRA-15908) @@ -984,6 +999,7 @@ Merged from 2.2: Merged from 2.1: * Fix writing of snapshot manifest when the table has table-backed secondary indexes (CASSANDRA-10968) + 4.0-alpha4 * Add client request size server metrics (CASSANDRA-15704) * Add additional logging around FileUtils and compaction leftover cleanup (CASSANDRA-15705) @@ -1040,6 +1056,7 @@ Merged from 2.1: * Fix Red Hat init script on newer systemd versions (CASSANDRA-15273) * Allow EXTRA_CLASSPATH to work on tar/source installations (CASSANDRA-15567) + 4.0-alpha3 * Restore monotonic read consistency guarantees for blocking read repair (CASSANDRA-14740) * Separate exceptions for CAS write timeout exceptions caused by contention and unkown result (CASSANDRA-15350) @@ -1091,6 +1108,7 @@ Merged from 2.2: * In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429) * Added data modeling documentation (CASSANDRA-15443) + 4.0-alpha2 * Fix SASI non-literal string comparisons (range operators) (CASSANDRA-15169) * Upgrade Guava to 27, and to java-driver 3.6.0 (from 3.4.0-SNAPSHOT) (CASSANDRA-14655) @@ -1482,6 +1500,7 @@ Merged from 3.0: * Multi-version in-JVM dtests (CASSANDRA-14937) * Allow instance class loaders to be garbage collected for inJVM dtest (CASSANDRA-15170) + 3.11.6 * Fix bad UDT sstable metadata serialization headers written by C* 3.0 on upgrade and in sstablescrub (CASSANDRA-15035) * Fix nodetool compactionstats showing extra pending task for TWCS - patch implemented (CASSANDRA-15409) @@ -1512,6 +1531,7 @@ Merged from 2.2: * In-JVM DTest: Support NodeTool in dtest (CASSANDRA-15429) * Fix NativeLibrary.tryOpenDirectory callers for Windows (CASSANDRA-15426) + 3.11.5 * Fix cassandra-env.sh to use $CASSANDRA_CONF to find cassandra-jaas.config (CASSANDRA-14305) * Fixed nodetool cfstats printing index name twice (CASSANDRA-14903) @@ -2648,7 +2668,6 @@ Merged from 2.1: * Sane default (200Mbps) for inter-DC streaming througput (CASSANDRA-8708) - 3.2 * Make sure tokens don't exist in several data directories (CASSANDRA-6696) * Add requireAuthorization method to IAuthorizer (CASSANDRA-10852) @@ -7681,7 +7700,6 @@ Merged from 0.8: - Similarly, merged batch_insert_super into batch_insert. - 0.4.0 beta * On-disk data format has changed to allow billions of keys/rows per node instead of only millions @@ -7718,7 +7736,6 @@ Merged from 0.8: * Rename configuration "table" to "keyspace" * Moved to crash-only design; no more shutdown (just kill the process) * Lots of bug fixes - Full list of issues resolved in 0.4 is at https://issues.apache.org/jira/secure/IssueNavigator.jspa?reset=true&&pid=12310865&fixfor=12313862&resolution=1&sorter/field=issuekey&sorter/order=DESC @@ -7749,3 +7766,5 @@ Full list of issues resolved in 0.4 is at https://issues.apache.org/jira/secure/ * Combined blocking and non-blocking versions of insert APIs * Added FlushPeriodInMinutes configuration parameter to force flushing of infrequently-updated ColumnFamilies + + diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 85aee95318..a8c11bfb92 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1935,14 +1935,12 @@ drop_compact_storage_enabled: false # excluded_keyspaces: # comma separated list of keyspaces to exclude from the check # excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check -# Supported sstable formats -# This is a list of elements consisting of class_name and parameters, where class_name should point to the class -# implementing org.apache.cassandra.io.sstable.format.SSTableFormat. Parameters must include unique 'id' integer -# which is used in some serialization to denote the format type in a compact way (such as local key cache); and 'name' -# which will be used to recognize the format type - in particular that name will be used in sstable file names and in -# stream headers so the name has to be the same for the same format across all the nodes in the cluster. -sstable_formats: - - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat - parameters: - id: 0 - name: big +# The sstable formats configuration. SSTable formats implementations are loaded using the service loader mechanism. +# In this section, one can select the format for created sstables and pass additional parameters for the formats +# available on the classpath. +#sstable: +# selected_format: big +# format: +# big: +# param1: value1 +# ... diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java index 243c070a32..dde36f1eac 100644 --- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java +++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java @@ -22,10 +22,12 @@ import java.io.IOException; import java.nio.file.NoSuchFileException; import java.util.ArrayDeque; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.Set; import java.util.UUID; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import javax.annotation.concurrent.NotThreadSafe; import org.cliffc.high_scale_lib.NonBlockingHashSet; import org.slf4j.Logger; @@ -48,6 +50,8 @@ import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.DataOutputStreamPlus; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.SequentialWriterOption; import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus; @@ -133,12 +137,17 @@ public class AutoSavingCache extends InstrumentingCache extends InstrumentingCache extends InstrumentingCache 0) { - //tableId and indexName are serialized by the serializers in CacheService - //That is delegated there because there are serializer specific conditions - //where a cache key is skipped and not written - TableId tableId = TableId.deserialize(in); - String indexName = in.readUTF(); - if (indexName.isEmpty()) - indexName = null; - - ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId); - if (indexName != null && cfs != null) - cfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElse(null); - - Future> entryFuture = cacheLoader.deserialize(in, cfs); + Future> entryFuture = cacheLoader.deserialize(in); // Key cache entry can return null, if the SSTable doesn't exist. if (entryFuture == null) continue; @@ -355,8 +358,12 @@ public class AutoSavingCache extends InstrumentingCache cacheFilePaths = tempCacheFiles(); - try (WrappedDataOutputStreamPlus writer = new WrappedDataOutputStreamPlus(streamFactory.getOutputStream(cacheFilePaths.left, cacheFilePaths.right))) + File dataTmpFile = getTempCacheFile(getCacheDataPath(CURRENT_VERSION)); + File crcTmpFile = getTempCacheFile(getCacheCrcPath(CURRENT_VERSION)); + File metadataTmpFile = getTempCacheFile(getCacheMetadataPath(CURRENT_VERSION)); + + try (WrappedDataOutputStreamPlus writer = new WrappedDataOutputStreamPlus(streamFactory.getOutputStream(dataTmpFile, crcTmpFile)); + FileOutputStreamPlus metadataWriter = metadataTmpFile.newOutputStream(File.WriteMode.OVERWRITE)) { //Need to be able to check schema version because CF names are ambiguous @@ -380,6 +387,9 @@ public class AutoSavingCache extends InstrumentingCache= keysEstimate) break; } + + cacheLoader.serializeMetadata(metadataWriter); + metadataWriter.sync(); } catch (FileNotFoundException | NoSuchFileException e) { @@ -387,30 +397,36 @@ public class AutoSavingCache extends InstrumentingCache tempCacheFiles() + private File getTempCacheFile(File cacheFile) { - File dataPath = getCacheDataPath(CURRENT_VERSION); - File crcPath = getCacheCrcPath(CURRENT_VERSION); - return Pair.create(FileUtils.createTempFile(dataPath.name(), null, dataPath.parent()), - FileUtils.createTempFile(crcPath.name(), null, crcPath.parent())); + return FileUtils.createTempFile(cacheFile.name(), null, cacheFile.parent()); } private void deleteOldCacheFiles() @@ -446,12 +462,91 @@ public class AutoSavingCache extends InstrumentingCache + /** + * A base cache serializer that is used to serialize/deserialize a cache to/from disk. + *

+ * It expects the following lifecycle: + * Serializations: + * 1. {@link #serialize(CacheKey, DataOutputPlus, ColumnFamilyStore)} is called for each key in the cache. + * 2. {@link #serializeMetadata(DataOutputPlus)} is called to serialize any metadata. + * 3. {@link #cleanupAfterSerialize()} is called to clean up any resources allocated for serialization. + *

+ * Deserializations: + * 1. {@link #deserializeMetadata(DataInputPlus)} is called to deserialize any metadata. + * 2. {@link #deserialize(DataInputPlus)} is called for each key in the cache. + * 3. {@link #cleanupAfterDeserialize()} is called to clean up any resources allocated for deserialization. + *

+ * This abstract class provides the default implementation for the metadata serialization/deserialization. + * The metadata includes a dictionary of column family stores collected during serialization whenever + * {@link #writeCFS(DataOutputPlus, ColumnFamilyStore)} or {@link #getOrCreateCFSOrdinal(ColumnFamilyStore)} + * are called. When such metadata is deserialized, the implementation of {@link #deserialize(DataInputPlus)} may + * use {@link #readCFS(DataInputPlus)} method to read the ColumnFamilyStore stored with + * {@link #writeCFS(DataOutputPlus, ColumnFamilyStore)}. + */ + @NotThreadSafe + public static abstract class CacheSerializer { - void serialize(K key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException; + private ColumnFamilyStore[] cfStores; - Future> deserialize(DataInputPlus in, ColumnFamilyStore cfs) throws IOException; + private final LinkedHashMap, Integer> cfsOrdinals = new LinkedHashMap<>(); - default void cleanupAfterDeserialize() { } + protected final int getOrCreateCFSOrdinal(ColumnFamilyStore cfs) + { + Integer ordinal = cfsOrdinals.putIfAbsent(Pair.create(cfs.metadata().id, cfs.metadata().indexName().orElse("")), cfsOrdinals.size()); + if (ordinal == null) + ordinal = cfsOrdinals.size() - 1; + return ordinal; + } + + protected ColumnFamilyStore readCFS(DataInputPlus in) throws IOException + { + return cfStores[in.readUnsignedVInt32()]; + } + + protected void writeCFS(DataOutputPlus out, ColumnFamilyStore cfs) throws IOException + { + out.writeUnsignedVInt32(getOrCreateCFSOrdinal(cfs)); + } + + public void serializeMetadata(DataOutputPlus out) throws IOException + { + // write the table ids + out.writeUnsignedVInt32(cfsOrdinals.size()); + for (Pair tableAndIndex : cfsOrdinals.keySet()) + { + tableAndIndex.left.serialize(out); + out.writeUTF(tableAndIndex.right); + } + } + + public void deserializeMetadata(DataInputPlus in) throws IOException + { + int tableEntries = in.readUnsignedVInt32(); + if (tableEntries == 0) + return; + cfStores = new ColumnFamilyStore[tableEntries]; + for (int i = 0; i < tableEntries; i++) + { + TableId tableId = TableId.deserialize(in); + String indexName = in.readUTF(); + cfStores[i] = Schema.instance.getColumnFamilyStoreInstance(tableId); + if (cfStores[i] != null && !indexName.isEmpty()) + cfStores[i] = cfStores[i].indexManager.getIndexByName(indexName).getBackingTable().orElse(null); + } + } + + public abstract void serialize(K key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException; + + public abstract Future> deserialize(DataInputPlus in) throws IOException; + + public void cleanupAfterSerialize() + { + cfsOrdinals.clear(); + } + + public void cleanupAfterDeserialize() + { + cfStores = null; + } } } diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 0411a862a6..fa83f51421 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -23,7 +23,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -31,8 +30,6 @@ import java.util.function.Supplier; import javax.annotation.Nullable; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.slf4j.Logger; @@ -77,9 +74,6 @@ public class Config */ public static final String PROPERTY_PREFIX = "cassandra."; - public static final String SSTABLE_FORMAT_ID = "id"; - public static final String SSTABLE_FORMAT_NAME = "name"; - public String cluster_name = "Test Cluster"; public String authenticator; public String authorizer; @@ -364,9 +358,13 @@ public class Config public String[] data_file_directories = new String[0]; - public List sstable_formats = ImmutableList.of(new ParameterizedClass(BigFormat.class.getName(),// "org.apache.cassandra.io.sstable.format.big.BigFormat", - ImmutableMap.of(SSTABLE_FORMAT_ID, "0", - SSTABLE_FORMAT_NAME, "big"))); + public static class SSTableConfig + { + public String selected_format = BigFormat.NAME; + public Map> format = new HashMap<>(); + } + + public final SSTableConfig sstable = new SSTableConfig(); /** * The directory to use for storing the system keyspaces data. diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index f6f59d0f40..2ab9ad6923 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -33,12 +33,12 @@ import java.util.Collection; import java.util.Comparator; import java.util.Enumeration; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; +import java.util.ServiceLoader; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -50,9 +50,10 @@ import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.RateLimiter; @@ -61,6 +62,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.googlecode.concurrenttrees.common.Iterables; import org.apache.cassandra.audit.AuditLogOptions; import org.apache.cassandra.auth.AllowAllInternodeAuthenticator; import org.apache.cassandra.auth.AuthConfig; @@ -83,6 +85,7 @@ import org.apache.cassandra.fql.FullQueryLoggerOptions; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.util.DiskOptimizationStrategy; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; @@ -206,15 +209,18 @@ public class DatabaseDescriptor public static volatile boolean allowUnlimitedConcurrentValidations = ALLOW_UNLIMITED_CONCURRENT_VALIDATIONS.getBoolean(); - /** The configuration for guardrails. */ + /** + * The configuration for guardrails. + */ private static GuardrailsOptions guardrails; private static StartupChecksOptions startupChecksOptions; - private static Map>> sstableFormatFactories; + private static ImmutableMap> sstableFormats; + private static SSTableFormat selectedSSTableFormat; private static Function commitLogSegmentMgrProvider = c -> DatabaseDescriptor.isCDCEnabled() - ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation()) - : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation()); + ? new CommitLogSegmentManagerCDC(c, DatabaseDescriptor.getCommitLogLocation()) + : new CommitLogSegmentManagerStandard(c, DatabaseDescriptor.getCommitLogLocation()); public static void daemonInitialization() throws ConfigurationException { @@ -1370,78 +1376,95 @@ public class DatabaseDescriptor paritionerName = partitioner.getClass().getCanonicalName(); } - @VisibleForTesting - public static Map>> loadSSTableFormatFactories(List configuredFormats) + private static void validateSSTableFormatFactories(Iterable factories) { - ImmutableMap.Builder>> sstableFormatFactories = ImmutableMap.builderWithExpectedSize(configuredFormats.size()); - Set names = new HashSet<>(configuredFormats.size()); - Set ids = new HashSet<>(configuredFormats.size()); - - for (ParameterizedClass formatConfig : configuredFormats) + Map factoryByName = new HashMap<>(); + for (SSTableFormat.Factory factory : factories) { - assert formatConfig.parameters != null; - Map params = new HashMap<>(formatConfig.parameters); + if (factory.name() == null) + throw new ConfigurationException(String.format("SSTable format name in %s cannot be null", factory.getClass().getCanonicalName())); - String name = params.get(Config.SSTABLE_FORMAT_NAME); - if (name == null) - throw new ConfigurationException("Missing 'name' parameter in sstable format configuration for " + formatConfig.class_name); - if (!name.matches("^[a-z]+$")) - throw new ConfigurationException("'name' parameter in sstable format configuration for " + formatConfig.class_name + " must be non-empty, lower-case letters only string"); - if (names.contains(name)) - throw new ConfigurationException("Name '" + name + "' of sstable format " + formatConfig.class_name + " is already defined for another sstable format"); - params.remove(Config.SSTABLE_FORMAT_NAME); + if (!factory.name().matches("^[a-z]+$")) + throw new ConfigurationException(String.format("SSTable format name for %s must be non-empty, lower-case letters only string", factory.getClass().getCanonicalName())); - String idString = params.get(Config.SSTABLE_FORMAT_ID); - if (idString == null) - throw new ConfigurationException("Missing 'id' parameter in sstable format configuration for " + formatConfig.class_name); - int id; - try - { - id = Integer.parseInt(idString); - } - catch (RuntimeException ex) - { - throw new ConfigurationException("'id' parameter in sstable format configuration for " + formatConfig.class_name + " must be an integer"); - } - if (id < 0 || id > 127) - throw new ConfigurationException("'id' parameter in sstable format configuration for " + formatConfig.class_name + " must be within bounds [0..127] range"); - if (ids.contains(id)) - throw new ConfigurationException("ID '" + id + "' of sstable format " + formatConfig.class_name + " is already defined for another sstable format"); - params.remove(Config.SSTABLE_FORMAT_ID); - - Supplier> factory = () -> { - Class> cls = FBUtilities.classForName(formatConfig.class_name, "sstable format"); - if (!SSTableFormat.class.isAssignableFrom(cls)) - throw new ConfigurationException(String.format("Class %s for sstable format %s does not implement %s", formatConfig.class_name, name, SSTableFormat.class.getName())); - - SSTableFormat sstableFormat = FBUtilities.instanceOrConstruct(cls.getName(), "sstable format"); - sstableFormat.setup(id, name, params); - return sstableFormat; - }; - sstableFormatFactories.put(name, factory); - names.add(name); - ids.add(id); + SSTableFormat.Factory prev = factoryByName.put(factory.name(), factory); + if (prev != null) + throw new ConfigurationException(String.format("Multiple sstable format implementations with the same name %s: %s and %s", factory.name(), factory.getClass().getCanonicalName(), prev.getClass().getCanonicalName())); } + } - return sstableFormatFactories.build(); + private static ImmutableMap>> validateAndMatchSSTableFormatOptions(Iterable factories, Map> options) + { + ImmutableMap.Builder>> providersBuilder = ImmutableMap.builder(); + if (options == null) + options = ImmutableMap.of(); + for (SSTableFormat.Factory factory : factories) + { + Map formatOptions = options.getOrDefault(factory.name(), ImmutableMap.of()); + providersBuilder.put(factory.name(), () -> factory.getInstance(ImmutableMap.copyOf(formatOptions))); + } + ImmutableMap>> providers = providersBuilder.build(); + if (options != null) + { + Sets.SetView unknownFormatNames = Sets.difference(options.keySet(), providers.keySet()); + if (!unknownFormatNames.isEmpty()) + throw new ConfigurationException(String.format("Configuration contains options of unknown sstable formats: %s", unknownFormatNames)); + } + return providers; + } + + private static SSTableFormat getAndValidateWriteFormat(Map> sstableFormats, String selectedFormatName) + { + SSTableFormat selectedFormat; + if (StringUtils.isBlank(selectedFormatName)) + selectedFormatName = BigFormat.NAME; + selectedFormat = sstableFormats.get(selectedFormatName); + if (selectedFormat == null) + throw new ConfigurationException(String.format("Selected sstable format '%s' is not available.", selectedFormatName)); + + return selectedFormat; } private static void applySSTableFormats() { - if (sstableFormatFactories != null) - logger.warn("Reinitializing SSTableFactories - this should happen only in tests"); + ServiceLoader loader = ServiceLoader.load(SSTableFormat.Factory.class, DatabaseDescriptor.class.getClassLoader()); + List factories = Iterables.toList(loader); + if (factories.isEmpty()) + factories = ImmutableList.of(new BigFormat.BigFormatFactory()); + applySSTableFormats(factories, conf.sstable); + } - sstableFormatFactories = loadSSTableFormatFactories(conf.sstable_formats); + private static void applySSTableFormats(Iterable factories, Config.SSTableConfig sstableFormatsConfig) + { + if (sstableFormats != null) + return; - Iterable types = SSTableFormat.Type.values(); // make sure we know where those types get initialized - types.forEach(t -> t.info.allComponents()); // make sure to reach all supported components for a type so that we know all of them are registered - logger.info("Supported sstable formats are: {}", Lists.newArrayList(types).stream().map(f -> f.name + " -> " + f.info.getClass().getName() + " with singleton components: " + f.info.allComponents()).collect(Collectors.joining(", "))); + validateSSTableFormatFactories(factories); + ImmutableMap>> providers = validateAndMatchSSTableFormatOptions(factories, sstableFormatsConfig.format); + + ImmutableMap.Builder> sstableFormatsBuilder = ImmutableMap.builder(); + providers.forEach((name, provider) -> { + try + { + sstableFormatsBuilder.put(name, provider.get()); + } + catch (RuntimeException | Error ex) + { + throw new ConfigurationException(String.format("Failed to instantiate sstable format '%s'", name), ex); + } + }); + sstableFormats = sstableFormatsBuilder.build(); + + selectedSSTableFormat = getAndValidateWriteFormat(sstableFormats, sstableFormatsConfig.selected_format); + + sstableFormats.values().forEach(SSTableFormat::allComponents); // make sure to reach all supported components for a type so that we know all of them are registered + logger.info("Supported sstable formats are: {}", sstableFormats.values().stream().map(f -> f.name() + " -> " + f.getClass().getName() + " with singleton components: " + f.allComponents()).collect(Collectors.joining(", "))); } /** * Computes the sum of the 2 specified positive values returning {@code Long.MAX_VALUE} if the sum overflow. * - * @param left the left operand + * @param left the left operand * @param right the right operand * @return the sum of the 2 specified positive values of {@code Long.MAX_VALUE} if the sum overflow. */ @@ -4702,9 +4725,22 @@ public class DatabaseDescriptor conf.client_request_size_metrics_enabled = enabled; } - public static Map>> getSSTableFormatFactories() + @VisibleForTesting + public static void resetSSTableFormats(Iterable factories, Config.SSTableConfig config) { - return Objects.requireNonNull(sstableFormatFactories, "Forgot to initialize DatabaseDescriptor?"); + sstableFormats = null; + selectedSSTableFormat = null; + applySSTableFormats(factories, config); + } + + public static ImmutableMap> getSSTableFormats() + { + return Objects.requireNonNull(sstableFormats, "Forgot to initialize DatabaseDescriptor?"); + } + + public static SSTableFormat getSelectedSSTableFormat() + { + return Objects.requireNonNull(selectedSSTableFormat, "Forgot to initialize DatabaseDescriptor?"); } public static boolean getDynamicDataMaskingEnabled() diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java index cf5e52f5dc..edff7be8ec 100644 --- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java +++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java @@ -200,6 +200,8 @@ public class YamlConfigurationLoader implements ConfigurationLoader Yaml rawYaml = new Yaml(loaderOptions); Map rawConfig = rawYaml.load(new ByteArrayInputStream(configBytes)); + if (rawConfig == null) + rawConfig = new HashMap<>(); verifyReplacements(replacements, rawConfig); } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 1b54710446..1be2085779 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -879,8 +879,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner descriptor.cfname, // Increment the generation until we find a filename that doesn't exist. This is needed because the new // SSTables that are being loaded might already use these generation numbers. - sstableIdGenerator.get(), - descriptor.formatType); + sstableIdGenerator.get()); } while (newDescriptor.fileFor(Components.DATA).exists()); return newDescriptor; @@ -926,22 +925,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner public Descriptor newSSTableDescriptor(File directory) { - return newSSTableDescriptor(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current()); + return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()); } - public Descriptor newSSTableDescriptor(File directory, SSTableFormat.Type format) + public Descriptor newSSTableDescriptor(File directory, SSTableFormat format) { - return newSSTableDescriptor(directory, format.info.getLatestVersion(), format); + return newSSTableDescriptor(directory, format.getLatestVersion()); } - public Descriptor newSSTableDescriptor(File directory, Version version, SSTableFormat.Type format) + public Descriptor newSSTableDescriptor(File directory, Version version) { Descriptor newDescriptor = new Descriptor(version, directory, keyspace.getName(), name, - sstableIdGenerator.get(), - format); + sstableIdGenerator.get()); assert !newDescriptor.fileFor(Components.DATA).exists(); return newDescriptor; } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java index 8d02da9d4a..88a7705997 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionLogger.java @@ -21,13 +21,16 @@ package org.apache.cassandra.db.compaction; import java.io.IOException; import java.io.OutputStreamWriter; import java.lang.ref.WeakReference; -import java.nio.file.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; @@ -173,7 +176,7 @@ public class CompactionLogger { ObjectNode node = json.objectNode(); node.put("generation", sstable.descriptor.id.toString()); - node.put("version", sstable.descriptor.version.getVersion()); + node.put("version", sstable.descriptor.version.version); node.put("size", sstable.onDiskLength()); JsonNode logResult = strategy.strategyLogger().sstable(sstable); if (logResult != null) diff --git a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java index 7a23a7a250..6b2b71ba4b 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/LogFile.java +++ b/src/java/org/apache/cassandra/db/lifecycle/LogFile.java @@ -41,11 +41,13 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LogRecord.Type; import org.apache.cassandra.io.sstable.SSTable; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; @@ -85,6 +87,8 @@ final class LogFile implements AutoCloseable // The unique id of the transaction private final TimeUUID id; + private final Version version = DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion(); + static LogFile make(File logReplica) { return make(logReplica.name(), Collections.singletonList(logReplica)); @@ -504,7 +508,10 @@ final class LogFile implements AutoCloseable private String getFileName() { - return StringUtils.join(SSTableFormat.Type.current().info.getLatestVersion(), LogFile.SEP, // remove version and separator when downgrading to 4.x is becomes unsupported + // For pre-5.0 versions, only BigFormat is supported, and the file name includes only the version string. + // To retain the ability to downgrade to 4.x, we keep the old file naming scheme for BigFormat sstables + // and add format names for other formats as they are supported only in 5.0 and above. + return StringUtils.join(BigFormat.is(version.format) ? version.toString() : version.toFormatAndVersionString(), LogFile.SEP, // remove version and separator when downgrading to 4.x is becomes unsupported "txn", LogFile.SEP, type.fileName, LogFile.SEP, id.toString(), LogFile.EXT); diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java index 0f89624a38..afe7a12000 100644 --- a/src/java/org/apache/cassandra/db/memtable/Flushing.java +++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java @@ -27,6 +27,7 @@ import com.google.common.base.Throwables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -97,12 +98,12 @@ public class Flushing Directories.DataDirectory flushLocation) { Memtable.FlushablePartitionSet flushSet = memtable.getFlushSet(from, to); - SSTableFormat.Type formatType = SSTableFormat.Type.current(); - long estimatedSize = formatType.info.getWriterFactory().estimateSize(flushSet); + SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); + long estimatedSize = format.getWriterFactory().estimateSize(flushSet); Descriptor descriptor = flushLocation == null - ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), formatType) - : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), formatType); + ? cfs.newSSTableDescriptor(cfs.getDirectories().getWriteableLocationAsFile(estimatedSize), format) + : cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(flushLocation), format); SSTableMultiWriter writer = createFlushWriter(cfs, flushSet, diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java index 005a9aaa6c..5a6e710518 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java @@ -76,7 +76,7 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader { TrackedDataInputPlus in = new TrackedDataInputPlus(cis); deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata())); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); String filename = writer.getFilename(); String sectionName = filename + '-' + fileSeqNum; int sectionIdx = 0; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java index bafe53979f..9c5e048a12 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java @@ -178,7 +178,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader LifecycleNewTracker lifecycleNewTracker = CassandraStreamReceiver.fromReceiver(session.getAggregator(tableId)).createLifecycleNewTracker(); - Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version, header.format); + Descriptor desc = cfs.newSSTableDescriptor(dataDir, header.version); IOOptions ioOptions = new IOOptions(DatabaseDescriptor.getDiskOptimizationStrategy(), DatabaseDescriptor.getDiskAccessMode(), diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index 6a712276bb..88ecff8a44 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -31,9 +31,9 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.OutgoingStream; -import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamSession; +import org.apache.cassandra.streaming.StreamingDataOutputPlus; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Ref; @@ -81,7 +81,6 @@ public class CassandraOutgoingFile implements OutgoingStream : null; return CassandraStreamHeader.builder() - .withSSTableFormat(sstable.descriptor.formatType) .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(operation.keepSSTableLevel() ? sstable.getSSTableLevel() : 0) .withEstimatedKeys(estimatedKeys) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java index 1ba5d13d02..2b1223bed9 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamHeader.java @@ -26,6 +26,7 @@ import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; @@ -47,8 +48,6 @@ public class CassandraStreamHeader /** SSTable version */ public final Version version; - /** SSTable format **/ - public final SSTableFormat.Type format; public final long estimatedKeys; public final List sections; public final CompressionInfo compressionInfo; @@ -68,7 +67,6 @@ public class CassandraStreamHeader private CassandraStreamHeader(Builder builder) { version = builder.version; - format = builder.format; estimatedKeys = builder.estimatedKeys; sections = builder.sections; compressionInfo = builder.compressionInfo; @@ -124,7 +122,6 @@ public class CassandraStreamHeader sstableLevel == that.sstableLevel && isEntireSSTable == that.isEntireSSTable && Objects.equals(version, that.version) && - format == that.format && Objects.equals(sections, that.sections) && Objects.equals(compressionInfo, that.compressionInfo) && Objects.equals(serializationHeader, that.serializationHeader) && @@ -136,7 +133,7 @@ public class CassandraStreamHeader @Override public int hashCode() { - return Objects.hash(version, format, estimatedKeys, sections, compressionInfo, sstableLevel, serializationHeader, componentManifest, + return Objects.hash(version, estimatedKeys, sections, compressionInfo, sstableLevel, serializationHeader, componentManifest, isEntireSSTable, firstKey, tableId); } @@ -145,7 +142,7 @@ public class CassandraStreamHeader { return "CassandraStreamHeader{" + "version=" + version + - ", format=" + format + + ", format=" + version.format.name() + ", estimatedKeys=" + estimatedKeys + ", sections=" + sections + ", sstableLevel=" + sstableLevel + @@ -163,7 +160,7 @@ public class CassandraStreamHeader public void serialize(CassandraStreamHeader header, DataOutputPlus out, int version) throws IOException { out.writeUTF(header.version.toString()); - out.writeUTF(header.format.name); + out.writeUTF(header.version.format.name()); out.writeLong(header.estimatedKeys); out.writeInt(header.sections.size()); @@ -182,7 +179,7 @@ public class CassandraStreamHeader if (header.isEntireSSTable) { - ComponentManifest.serializers.get(header.format).serialize(header.componentManifest, out, version); + ComponentManifest.serializers.get(header.version.format.name()).serialize(header.componentManifest, out, version); ByteBufferUtil.writeWithVIntLength(header.firstKey.getKey(), out); } } @@ -202,8 +199,10 @@ public class CassandraStreamHeader public CassandraStreamHeader deserialize(DataInputPlus in, int version, Function partitionerMapper) throws IOException { String sstableVersionString = in.readUTF(); - SSTableFormat.Type format = SSTableFormat.Type.getByName(in.readUTF()); - Version sstableVersion = format.info.getVersion(sstableVersionString); + String formatName = in.readUTF(); + SSTableFormat format = Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(formatName), + String.format("Unknown SSTable format '%s'", formatName)); + Version sstableVersion = format.getVersion(sstableVersionString); long estimatedKeys = in.readLong(); int count = in.readInt(); @@ -222,7 +221,7 @@ public class CassandraStreamHeader if (isEntireSSTable) { - manifest = ComponentManifest.serializers.get(format).deserialize(in, version); + manifest = ComponentManifest.serializers.get(format.name()).deserialize(in, version); ByteBuffer keyBuf = ByteBufferUtil.readWithVIntLength(in); IPartitioner partitioner = partitionerMapper.apply(tableId); if (partitioner == null) @@ -230,8 +229,7 @@ public class CassandraStreamHeader firstKey = partitioner.decorateKey(keyBuf); } - return builder().withSSTableFormat(format) - .withSSTableVersion(sstableVersion) + return builder().withSSTableVersion(sstableVersion) .withSSTableLevel(sstableLevel) .withEstimatedKeys(estimatedKeys) .withSections(sections) @@ -248,7 +246,7 @@ public class CassandraStreamHeader { long size = 0; size += TypeSizes.sizeof(header.version.toString()); - size += TypeSizes.sizeof(header.format.name); + size += TypeSizes.sizeof(header.version.format.name()); size += TypeSizes.sizeof(header.estimatedKeys); size += TypeSizes.sizeof(header.sections.size()); @@ -268,7 +266,7 @@ public class CassandraStreamHeader if (header.isEntireSSTable) { - size += ComponentManifest.serializers.get(header.format).serializedSize(header.componentManifest, version); + size += ComponentManifest.serializers.get(header.version.format.name()).serializedSize(header.componentManifest, version); size += ByteBufferUtil.serializedSizeWithVIntLength(header.firstKey.getKey()); } return size; @@ -278,7 +276,6 @@ public class CassandraStreamHeader public static final class Builder { private Version version; - private SSTableFormat.Type format; private long estimatedKeys; private List sections; private CompressionInfo compressionInfo; @@ -289,12 +286,6 @@ public class CassandraStreamHeader private DecoratedKey firstKey; private TableId tableId; - public Builder withSSTableFormat(SSTableFormat.Type format) - { - this.format = format; - return this; - } - public Builder withSSTableVersion(Version version) { this.version = version; @@ -358,7 +349,6 @@ public class CassandraStreamHeader public CassandraStreamHeader build() { checkNotNull(version); - checkNotNull(format); checkNotNull(sections); checkNotNull(serializationHeader); checkNotNull(tableId); diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index a659c73e7c..2325828b02 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -73,7 +73,6 @@ public class CassandraStreamReader implements IStreamReader protected final Version inputVersion; protected final long repairedAt; protected final TimeUUID pendingRepair; - protected final SSTableFormat.Type format; protected final int sstableLevel; protected final SerializationHeader.Component header; protected final int fileSeqNum; @@ -93,7 +92,6 @@ public class CassandraStreamReader implements IStreamReader this.inputVersion = streamHeader.version; this.repairedAt = header.repairedAt; this.pendingRepair = header.pendingRepair; - this.format = streamHeader.format; this.sstableLevel = streamHeader.sstableLevel; this.header = streamHeader.serializationHeader; this.fileSeqNum = header.sequenceNumber; @@ -125,7 +123,7 @@ public class CassandraStreamReader implements IStreamReader { TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream); deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata())); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); String sequenceName = writer.getFilename() + '-' + fileSeqNum; long lastBytesRead = 0; while (in.getBytesRead() < totalSize) @@ -157,7 +155,7 @@ public class CassandraStreamReader implements IStreamReader return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader } @SuppressWarnings("resource") - protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat.Type format) throws IOException + protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat format) throws IOException { Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize); if (localDir == null) diff --git a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java index 36eb668e86..5e3cc0c6b6 100644 --- a/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java +++ b/src/java/org/apache/cassandra/db/streaming/ComponentManifest.java @@ -29,6 +29,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.sstable.Component; @@ -115,12 +116,12 @@ public final class ComponentManifest implements Iterable '}'; } - public static final Map> serializers; + public static final Map> serializers; static { - ImmutableMap.Builder> b = ImmutableMap.builder(); - for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) + ImmutableMap.Builder> b = ImmutableMap.builder(); + for (SSTableFormat format : DatabaseDescriptor.getSSTableFormats().values()) { IVersionedSerializer serializer = new IVersionedSerializer() { @@ -143,7 +144,7 @@ public final class ComponentManifest implements Iterable for (int i = 0; i < size; i++) { - Component component = Component.parse(in.readUTF(), formatType); + Component component = Component.parse(in.readUTF(), format); long length = in.readUnsignedVInt(); components.put(component, length); } @@ -163,7 +164,7 @@ public final class ComponentManifest implements Iterable } }; - b.put(formatType, serializer); + b.put(format.name(), serializer); } serializers = b.build(); } diff --git a/src/java/org/apache/cassandra/gms/ApplicationState.java b/src/java/org/apache/cassandra/gms/ApplicationState.java index c45d3c2602..3f44bcc05b 100644 --- a/src/java/org/apache/cassandra/gms/ApplicationState.java +++ b/src/java/org/apache/cassandra/gms/ApplicationState.java @@ -52,7 +52,7 @@ public enum ApplicationState * which new sstables are written), but may contain more on newly upgraded nodes before `upgradesstable` has been * run. * - *

The value (a set of sstable {@link org.apache.cassandra.io.sstable.format.VersionAndType}) is serialized as + *

The value (a set of sstable {@link org.apache.cassandra.io.sstable.format.Version}) is serialized as * a comma-separated list. **/ SSTABLE_VERSIONS, diff --git a/src/java/org/apache/cassandra/gms/VersionedValue.java b/src/java/org/apache/cassandra/gms/VersionedValue.java index d76f3011f4..7c64278721 100644 --- a/src/java/org/apache/cassandra/gms/VersionedValue.java +++ b/src/java/org/apache/cassandra/gms/VersionedValue.java @@ -17,30 +17,32 @@ */ package org.apache.cassandra.gms; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; import java.net.InetAddress; import java.util.Collection; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static java.nio.charset.StandardCharsets.ISO_8859_1; - import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import org.apache.commons.lang3.StringUtils; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.FBUtilities; -import org.apache.commons.lang3.StringUtils; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; /** @@ -324,10 +326,10 @@ public class VersionedValue implements Comparable return new VersionedValue(String.valueOf(value)); } - public VersionedValue sstableVersions(Set versions) + public VersionedValue sstableVersions(Set versions) { return new VersionedValue(versions.stream() - .map(VersionAndType::toString) + .map(Version::toFormatAndVersionString) .collect(Collectors.joining(","))); } } diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index 0288ebc1d5..429995cc48 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.SerializationHeader; @@ -45,7 +46,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable protected final File directory; protected final TableMetadataRef metadata; protected final RegularAndStaticColumns columns; - protected SSTableFormat.Type formatType = SSTableFormat.Type.current(); + protected SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); protected static final AtomicReference id = new AtomicReference<>(SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get()); protected boolean makeRangeAware = false; @@ -56,9 +57,9 @@ abstract class AbstractSSTableSimpleWriter implements Closeable this.columns = columns; } - protected void setSSTableFormatType(SSTableFormat.Type type) + protected void setSSTableFormatType(SSTableFormat type) { - this.formatType = type; + this.format = type; } protected void setRangeAwareWriting(boolean makeRangeAware) @@ -71,10 +72,10 @@ abstract class AbstractSSTableSimpleWriter implements Closeable SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS); if (makeRangeAware) - return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, formatType, 0, header); + return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, 0, header); return SSTableTxnWriter.create(metadata, - createDescriptor(directory, metadata.keyspace, metadata.name, formatType), + createDescriptor(directory, metadata.keyspace, metadata.name, format), 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, @@ -85,7 +86,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable owner); } - private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) throws IOException + private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat fmt) throws IOException { SSTableId nextGen = getNextId(directory, columnFamily); return new Descriptor(directory, keyspace, columnFamily, nextGen, fmt); diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 604a4ee361..d0bcafd21c 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -34,8 +34,6 @@ import com.google.common.collect.Sets; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; -import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; @@ -44,6 +42,8 @@ import org.apache.cassandra.cql3.functions.UDHelper; import org.apache.cassandra.cql3.functions.types.TypeCodec; import org.apache.cassandra.cql3.functions.types.UserType; import org.apache.cassandra.cql3.statements.ModificationStatement; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.Slice; import org.apache.cassandra.db.Slices; @@ -54,7 +54,17 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.util.File; -import org.apache.cassandra.schema.*; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaTransformations; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.schema.Types; +import org.apache.cassandra.schema.UserFunctions; +import org.apache.cassandra.schema.Views; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; @@ -366,7 +376,7 @@ public class CQLSSTableWriter implements Closeable { private File directory; - protected SSTableFormat.Type formatType = null; + protected SSTableFormat format = null; private CreateTableStatement.Raw schemaStatement; private final List typeStatements; @@ -587,8 +597,8 @@ public class CQLSSTableWriter implements Closeable ? new SSTableSimpleWriter(directory, ref, preparedModificationStatement.updatedColumns()) : new SSTableSimpleUnsortedWriter(directory, ref, preparedModificationStatement.updatedColumns(), bufferSizeInMiB); - if (formatType != null) - writer.setSSTableFormatType(formatType); + if (format != null) + writer.setSSTableFormatType(format); return new CQLSSTableWriter(writer, preparedModificationStatement, preparedModificationStatement.getBindVariables()); } diff --git a/src/java/org/apache/cassandra/io/sstable/Component.java b/src/java/org/apache/cassandra/io/sstable/Component.java index 13c63859df..f2eea992a3 100644 --- a/src/java/org/apache/cassandra/io/sstable/Component.java +++ b/src/java/org/apache/cassandra/io/sstable/Component.java @@ -114,19 +114,19 @@ public class Component } @VisibleForTesting - public static Type fromRepresentation(String repr, SSTableFormat.Type formatType) + public static Type fromRepresentation(String repr, SSTableFormat format) { for (Type type : Type.all) { - if (type.repr != null && Pattern.matches(type.repr, repr) && type.formatClass.isAssignableFrom(formatType.info.getClass())) + if (type.repr != null && Pattern.matches(type.repr, repr) && type.formatClass.isAssignableFrom(format.getClass())) return type; } return Types.CUSTOM; } - public static Component createComponent(String repr, SSTableFormat.Type formatType) + public static Component createComponent(String repr, SSTableFormat format) { - Type type = fromRepresentation(repr, formatType); + Type type = fromRepresentation(repr, format); if (type.singleton != null) return type.singleton; else @@ -199,9 +199,9 @@ public class Component * @return the component corresponding to {@code name}. Note that this always return a component as an unrecognized * name is parsed into a CUSTOM component. */ - public static Component parse(String name, SSTableFormat.Type formatType) + public static Component parse(String name, SSTableFormat format) { - return Type.createComponent(name, formatType); + return Type.createComponent(name, format); } public static Iterable getSingletonsFor(SSTableFormat format) @@ -216,7 +216,7 @@ public class Component public boolean isValidFor(Descriptor descriptor) { - return type.formatClass.isAssignableFrom(descriptor.formatType.info.getClass()); + return type.formatClass.isAssignableFrom(descriptor.version.format.getClass()); } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java index 527a7140e3..d77d1f1d6c 100644 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@ -31,6 +31,7 @@ import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Directories; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.Version; @@ -39,7 +40,6 @@ import org.apache.cassandra.io.sstable.metadata.MetadataSerializer; import org.apache.cassandra.io.util.File; import org.apache.cassandra.utils.Pair; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.cassandra.io.sstable.Component.separator; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -87,7 +87,6 @@ public class Descriptor public final String ksname; public final String cfname; public final SSTableId id; - public final SSTableFormat.Type formatType; private final int hashCode; private final String prefix; private final File baseFile; @@ -98,38 +97,35 @@ public class Descriptor @VisibleForTesting public Descriptor(File directory, String ksname, String cfname, SSTableId id) { - this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, id, SSTableFormat.Type.current()); + this(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion(), directory, ksname, cfname, id); } /** * Constructor for sstable writers only. */ - public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType) + public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat format) { - this(formatType.info.getLatestVersion(), directory, ksname, cfname, id, formatType); + this(format.getLatestVersion(), directory, ksname, cfname, id); } @VisibleForTesting - public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType) + public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat format) { - this(formatType.info.getVersion(version), directory, ksname, cfname, id, formatType); + this(format.getVersion(version), directory, ksname, cfname, id); } - public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType) + public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id) { checkNotNull(version); checkNotNull(directory); checkNotNull(ksname); checkNotNull(cfname); - checkNotNull(formatType); - checkArgument(version.getSSTableFormat() == formatType.info); this.version = version; this.directory = directory.toCanonical(); this.ksname = ksname; this.cfname = cfname; this.id = id; - this.formatType = formatType; StringBuilder buf = new StringBuilder(); appendFileName(buf); @@ -137,7 +133,7 @@ public class Descriptor this.baseFile = new File(directory.toPath().resolve(prefix)); // directory is unnecessary for hashCode, and for simulator consistency we do not include it - hashCode = Objects.hashCode(version, id, ksname, cfname, formatType); + hashCode = Objects.hashCode(version, id, ksname, cfname); } private String tmpFilenameFor(Component component) @@ -184,7 +180,7 @@ public class Descriptor { buff.append(version).append(separator); buff.append(id.toString()); - buff.append(separator).append(formatType.name); + buff.append(separator).append(version.format.name()); } public String relativeFilenameFor(Component component) @@ -202,7 +198,7 @@ public class Descriptor public SSTableFormat getFormat() { - return formatType.info; + return version.format; } /** Return any temporary files found in the directory */ @@ -270,17 +266,13 @@ public class Descriptor return Component.parse(tokens.get(3), formatFromName(name, tokens)); } - private static SSTableFormat.Type formatFromName(String fileName, List tokens) + private static SSTableFormat formatFromName(String fileName, List tokens) { - String format = tokens.get(2); - try - { - return SSTableFormat.Type.getByName(format); - } - catch (RuntimeException e) - { - throw invalidSSTable(fileName, "unknown 'format' part (%s)", format); - } + String formatString = tokens.get(2); + SSTableFormat format = DatabaseDescriptor.getSSTableFormats().get(formatString); + if (format == null) + throw invalidSSTable(fileName, "unknown 'format' part (%s)", formatString); + return format; } /** @@ -336,7 +328,7 @@ public class Descriptor throw invalidSSTable(name, String.format("cannot extract keyspace and table name from %s; make sure the sstable is in the proper sub-directories", file)); } - return Pair.create(new Descriptor(info.version, parentOf(name, file), keyspaceName, tableName, info.id, info.format), info.component); + return Pair.create(new Descriptor(info.version, parentOf(name, file), keyspaceName, tableName, info.id), info.component); } /** @@ -360,7 +352,7 @@ public class Descriptor } SSTableInfo info = validateAndExtractInfo(file); - return Pair.create(new Descriptor(info.version, parentOf(file.name(), file), keyspace, table, info.id, info.format), info.component); + return Pair.create(new Descriptor(info.version, parentOf(file.name(), file), keyspace, table, info.id), info.component); } private static List filenameTokens(String name) @@ -401,28 +393,26 @@ public class Descriptor throw invalidSSTable(name, "the 'id' part (%s) of the name doesn't parse as a valid unique identifier", tokens.get(1)); } - SSTableFormat.Type format = formatFromName(name, tokens); + SSTableFormat format = formatFromName(name, tokens); Component component = Component.parse(tokens.get(3), format); - Version version = format.info.getVersion(versionString); + Version version = format.getVersion(versionString); if (!version.isCompatible()) throw invalidSSTable(name, "incompatible sstable version (%s); you should have run upgradesstables before upgrading", versionString); - return new SSTableInfo(version, id, format, component); + return new SSTableInfo(version, id, component); } private static class SSTableInfo { final Version version; final SSTableId id; - final SSTableFormat.Type format; final Component component; - SSTableInfo(Version version, SSTableId id, SSTableFormat.Type format, Component component) + SSTableInfo(Version version, SSTableId id, Component component) { this.version = version; this.id = id; - this.format = format; this.component = component; } } @@ -456,7 +446,7 @@ public class Descriptor public Set discoverComponents() { Set components = Sets.newHashSetWithExpectedSize(Component.Type.all.size()); - for (Component component : Component.getSingletonsFor(formatType.info)) + for (Component component : Component.getSingletonsFor(version.format)) { if (fileFor(component).exists()) components.add(component); @@ -484,8 +474,7 @@ public class Descriptor && that.id.equals(this.id) && that.ksname.equals(this.ksname) && that.cfname.equals(this.cfname) - && that.version.equals(this.version) - && that.formatType == this.formatType; + && that.version.equals(this.version); } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java index 6c75d38c74..9caef0bf9d 100644 --- a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java @@ -45,7 +45,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final long repairedAt; private final TimeUUID pendingRepair; private final boolean isTransient; - private final SSTableFormat.Type format; + private final SSTableFormat format; private final SerializationHeader header; private final LifecycleNewTracker lifecycleNewTracker; private int currentIndex = -1; @@ -54,7 +54,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final List finishedReaders = new ArrayList<>(); private SSTableMultiWriter currentWriter = null; - public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat.Type format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException + public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat format, int sstableLevel, long totalSize, LifecycleNewTracker lifecycleNewTracker, SerializationHeader header) throws IOException { DiskBoundaries db = cfs.getDiskBoundaries(); directories = db.directories; diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index dd296e3fab..5156adc4e1 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -290,7 +290,7 @@ public abstract class SSTable @Override public String toString() { - return String.format("%s:%s(path='%s')", getClass().getSimpleName(), descriptor.formatType.name, getFilename()); + return String.format("%s:%s(path='%s')", getClass().getSimpleName(), descriptor.version.format.name(), getFilename()); } public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRepair, boolean isTransient) @@ -349,7 +349,7 @@ public abstract class SSTable { if (components != null) { - components.forEach(c -> Preconditions.checkState(c.isValidFor(descriptor), "Invalid component type for sstable format " + descriptor.formatType.name)); + components.forEach(c -> Preconditions.checkState(c.isValidFor(descriptor), "Invalid component type for sstable format " + descriptor.version.format.name())); this.components = ImmutableSet.copyOf(components); } else diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java index fb1fad7aa8..08515747d0 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java @@ -83,7 +83,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter { // todo: inefficient - we create and serialize a PU just to get its size, then recreate it // todo: either allow PartitionUpdateBuilder to have .build() called several times or pre-calculate the size - currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), formatType.info.getLatestVersion().correspondingMessagingVersion()); + currentSize += PartitionUpdate.serializer.serializedSize(createPartitionUpdateBuilder(key).build(), format.getLatestVersion().correspondingMessagingVersion()); previous = createPartitionUpdateBuilder(key); buffer.put(key, previous); } @@ -97,7 +97,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter // improve that. In particular, what we count is closer to the serialized value, but it's debatable that it's the right thing // to count since it will take a lot more space in memory and the bufferSize if first and foremost used to avoid OOM when // using this writer. - currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, formatType.info.getLatestVersion().correspondingMessagingVersion()); + currentSize += UnfilteredSerializer.serializer.serializedSize(row, helper, 0, format.getLatestVersion().correspondingMessagingVersion()); } private void maybeSync() throws SyncException diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java index 1544157379..fad7d54424 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java @@ -112,7 +112,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long repairedAt, TimeUUID pendingRepair, boolean isTransient, - SSTableFormat.Type type, + SSTableFormat type, int sstableLevel, SerializationHeader header) { diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable_API.md b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md index aa637da6a1..a72a53f1ca 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable_API.md +++ b/src/java/org/apache/cassandra/io/sstable/SSTable_API.md @@ -3,67 +3,62 @@ [CEP-17](https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-17%3A+SSTable+format+API) / [CASSANDRA-17056](https://issues.apache.org/jira/browse/CASSANDRA-17056) +## SSTable format + +SSTable format is an implementation of the `SSTableFormat` interface. It is responsible for creating readers, writers, +scrubbers, verifiers, and other components for processing the sstables. An SSTable format implementation comes with +a factory class implementing the `SSTableFormat.Factory` interface. The factory is required to provide a unique name +of the implementation and a method for creating the format instance. + ## Configuration specification -SSTable formats and options are specified under the `sstable_formats` key in the _cassandra.yaml_ configuration file. -By default, the first sstable format implementation specified is the default one, which means that newly produced -sstables will use that implementation. Other implementations are used to read the sstables. +SSTable format factories are discovered using +[Java Service Loader](https://docs.oracle.com/javase/8/docs/api/java/util/ServiceLoader.html) mechanism. The loaded +format implementations can be used to read the existing sstables. The write format is chosen based on the configuration. +If it is not specified, `BigFormat` implementation is assumed. + +Optional SSTable formats configuration can be supplied in the _cassandra.yaml_ file under the `sstable` key. ```yaml -sstable_formats: - - class_name: 〈class name of the default SSTableFormat implementation〉 - parameters: - id: 〈unique integer identifier of the implementation〉 - name: 〈unique string identifier of the implementation〉 +sstable: + selected_format: 〈name of the default SSTableFormat implementation〉 + format: + 〈format1 name〉: param1: 〈format specific parameter 1〉 param2: 〈format specific parameter 2〉 # ... - - class_name: 〈class name of additional SSTableFormat implementation〉 - parameters: - id: 〈unique integer identifier of the implementation〉 - name: 〈unique string identifier of the implementation〉 + 〈format2 name〉: param1: 〈format specific parameter 1〉 param2: 〈format specific parameter 2〉 # ... - # ... + # ... ``` -Each sstable format definition includes the class name of the class implementing [`SSTableFormat`](format/SSTableFormat.java) -interface and a map of parameters. Two parameters are mandatory - _id_ and _name_, and they have to be unique across -all the defined formats. Additionally, they need to obey the following rules: -- _id_ must be an integer between 0 and 127 -- _name_ must contain only lowercase ASCII letters +Each implementation must have a unique name that is used to unanonimously identify the format. The name must be +consistently returned by `name()` methods in the `SSTableFormat` and `SSTableFormat.Factory` implementations. It must +include only lowercase ASCII letters. -The _name_ parameter is the string put in the sstable file name. Therefore, it is the way Cassandra chooses the sstable -format implementation when loading an sstable. The _id_ parameter is used in saved cache files. Both parameters should -remain the same for the same format when upgrading. In particular, previous Cassandra versions have only -[_big_ format](format/big/BigFormat.java) implementation, thus when upgrading from the old version, the configuration -should keep _id_ set to _0_ and _name_ set to _big_ for that implementation (this is the default configuration), and any -new implementations should use different values for those parameters. +Parameters specified under the key named after the format name are passed to the factory method of the correspondin +implementation. All of those parameters are optional and depend on the implementation. -The remaining parameters are optional and passed as a map to the `setup` method of a class implementing `SSTableFormat`. - -Default configuration: +The assumed default configuration - which is equivalent to empty configuration: ```yaml -sstable_formats: - - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat - parameters: - id: 0 - name: big +sstable: + selected_format: big ``` -Example configuration which uses `BtiFormat` as the default one keeping the right parameters for the `BigFormat` -implementation. +Example configuration which uses `bti` as the default: ```yaml -sstable_formats: - - class_name: org.apache.cassandra.io.sstable.format.bti.BtiFormat - parameters: - id: 1 - name: bti - - class_name: org.apache.cassandra.io.sstable.format.big.BigFormat - parameters: - id: 0 - name: big +sstable: + selected_format: bti + format: + big: + param1: value1 + param2: value2 + bti: + param1: value1 + param2: value2 + ``` ## Components diff --git a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java index 5085cbc28b..851e4735e9 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/AbstractSSTableFormat.java @@ -19,46 +19,31 @@ package org.apache.cassandra.io.sstable.format; import java.util.Map; - -import com.google.common.base.Preconditions; +import java.util.Objects; public abstract class AbstractSSTableFormat implements SSTableFormat { - private String name; - private int ordinal; + public final String name; + protected final Map options; - @Override - public final synchronized void setup(int ordinal, String name, Map options) - { - if (this.name != null) - throw new IllegalStateException("Attempted to set up already configured sstable format"); - this.name = name; - this.ordinal = ordinal; - setup(options); - } - - protected void setup(Map options) + protected AbstractSSTableFormat(String name, Map options) { + this.name = Objects.requireNonNull(name); + this.options = options; } @Override - public final Type getType() + public final boolean equals(Object o) { - Preconditions.checkState(name != null, "Not configured"); - Type type = Type.getByOrdinal(ordinal); - assert type != null; - return type; + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AbstractSSTableFormat that = (AbstractSSTableFormat) o; + return Objects.equals(name, that.name); } @Override - public int ordinal() + public final int hashCode() { - return ordinal; - } - - @Override - public String name() - { - return name; + return Objects.hash(name); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java index 0f0e55efe0..1cf22c6ed7 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java @@ -18,25 +18,15 @@ package org.apache.cassandra.io.sstable.format; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Set; -import java.util.function.Supplier; +import javax.annotation.Nonnull; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.CharMatcher; -import com.google.common.collect.ImmutableList; - -import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -54,7 +44,6 @@ import org.apache.cassandra.utils.Pair; */ public interface SSTableFormat { - int ordinal(); String name(); Version getLatestVersion(); @@ -113,10 +102,6 @@ public interface SSTableFormat void deleteOrphanedComponents(Descriptor descriptor, Set components); - void setup(int id, String name, Map options); - - Type getType(); - /** * Deletes the existing components of the sstables represented by the provided descriptor. * The method is also responsible for cleaning up the in-memory resources occupied by the stuff related to that @@ -124,91 +109,6 @@ public interface SSTableFormat */ void delete(Descriptor descriptor); - class Type - { - private final static ImmutableList types; - private final static Type[] typesById; - - static - { - Pair, Type[]> factories = readFactories(DatabaseDescriptor.getSSTableFormatFactories()); - types = ImmutableList.copyOf(factories.left); - typesById = factories.right; - } - - @VisibleForTesting - public static Pair, Type[]> readFactories(Map>> factories) - { - List typesList = new ArrayList<>(factories.size()); - factories.forEach((key, factory) -> { - SSTableFormat format = factory.get(); - typesList.add(new Type(format.ordinal(), format.name(), format)); - }); - List types = ImmutableList.copyOf(typesList); - int maxId = typesList.stream().mapToInt(t -> t.ordinal).max().getAsInt(); - Type[] typesById = new Type[maxId + 1]; - typesList.forEach(t -> typesById[t.ordinal] = t); - return Pair.create(types, typesById); - } - - public final int ordinal; - public final SSTableFormat info; - public final String name; - - private static Type currentType; - - public static Type current() - { - if (currentType != null) - return currentType; - - String name = CassandraRelevantProperties.SSTABLE_FORMAT_DEFAULT.getString(); - if (name == null) - return types.get(0); - - try - { - Type type = getByName(name); - currentType = type; - return type; - } - catch (RuntimeException ex) - { - throw new ConfigurationException("SSTable format " + name + " is not registered. Registered formats are: " + types); - } - } - - private Type(int ordinal, String name, SSTableFormat info) - { - //Since format comes right after generation - //we disallow formats with numeric names - assert !CharMatcher.digit().matchesAllOf(name); - this.ordinal = ordinal; - this.name = name; - this.info = info; - } - - public static Type getByName(String name) - { - for (int i = 0; i < types.size(); i++) - if (types.get(i).name.equals(name)) - return types.get(i); - throw new NoSuchElementException(name); - } - - public static Type getByOrdinal(int ordinal) - { - if (ordinal < 0 || ordinal >= typesById.length || typesById[ordinal] == null) - throw new NoSuchElementException(String.valueOf(ordinal)); - return typesById[ordinal]; - } - - public static Iterable values() - { - return types; - } - } - interface SSTableReaderFactory> { /** @@ -298,4 +198,21 @@ public interface SSTableFormat void serialize(T entry, DataOutputPlus output) throws IOException; } + interface Factory + { + /** + * Returns a name of the format. Format name must not be empty, must be unique and must consist only of lowercase letters. + */ + String name(); + + /** + * Returns an instance of the sstable format configured with the provided options. + *

+ * The method is expected to validate the options, and throw + * {@link org.apache.cassandra.exceptions.ConfigurationException} if the validation fails. + * + * @param options overrides for the default options, can be empty, cannot be null + */ + SSTableFormat getInstance(@Nonnull Map options); + } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java index 677d809877..2915d362a7 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java +++ b/src/java/org/apache/cassandra/io/sstable/format/TOCComponent.java @@ -68,7 +68,7 @@ public class TOCComponent Set components = Sets.newHashSetWithExpectedSize(componentNames.size()); for (String componentName : componentNames) { - Component component = Component.parse(componentName, descriptor.formatType); + Component component = Component.parse(componentName, descriptor.version.format); if (skipMissing && !descriptor.fileFor(component).exists()) logger.error("Missing component: {}", descriptor.fileFor(component)); else diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index ae5feedd1a..137979ae15 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -35,8 +35,8 @@ public abstract class Version { private static final Pattern VALIDATION = Pattern.compile("[a-z]+"); - protected final String version; - protected final SSTableFormat format; + public final String version; + public final SSTableFormat format; protected Version(SSTableFormat format, String version) { @@ -88,16 +88,6 @@ public abstract class Version public abstract boolean hasKeyRange(); - public String getVersion() - { - return version; - } - - public SSTableFormat getSSTableFormat() - { - return format; - } - /** * @param ver SSTable version * @return True if the given version string matches the format. @@ -118,6 +108,11 @@ public abstract class Version return version; } + public String toFormatAndVersionString() + { + return format.name() + '-' + version; + } + @Override public boolean equals(Object other) { @@ -125,12 +120,12 @@ public abstract class Version if (other == null || getClass() != other.getClass()) return false; Version otherVersion = (Version) other; - return Objects.equals(version, otherVersion.version); + return Objects.equals(version, otherVersion.version) && Objects.equals(format.name(), otherVersion.format.name()); } @Override public int hashCode() { - return version != null ? version.hashCode() : 0; + return Objects.hash(version, format.name()); } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java b/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java deleted file mode 100644 index 25dc1c5c60..0000000000 --- a/src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.io.sstable.format; - -import java.util.List; -import java.util.Objects; - -import com.google.common.base.Splitter; - -import org.apache.cassandra.io.sstable.Descriptor; - -/** - * Groups a sstable {@link Version} with a {@link SSTableFormat.Type}. - * - *

Note that both information are currently necessary to identify the exact "format" of an sstable (without having - * its {@link Descriptor}). In particular, while {@link Version} contains its {{@link SSTableFormat}}, you cannot get - * the {{@link SSTableFormat.Type}} from that. - * @deprecated TODO remove this class - Version contains SSTableFormat and you _can_ get its type from it - */ -public final class VersionAndType -{ - private static final Splitter splitOnDash = Splitter.on('-').omitEmptyStrings().trimResults(); - - private final Version version; - private final SSTableFormat.Type formatType; - - public VersionAndType(Version version, SSTableFormat.Type formatType) - { - this.version = version; - this.formatType = formatType; - } - - public Version version() - { - return version; - } - - public SSTableFormat.Type formatType() - { - return formatType; - } - - @Override - public boolean equals(Object o) - { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - VersionAndType that = (VersionAndType) o; - return Objects.equals(version, that.version) && - formatType == that.formatType; - } - - @Override - public int hashCode() - { - return Objects.hash(version, formatType); - } - - public static VersionAndType fromString(String versionAndType) - { - List components = splitOnDash.splitToList(versionAndType); - if (components.size() != 2) - throw new IllegalArgumentException("Invalid VersionAndType string: " + versionAndType + " (should be of the form 'big-bc')"); - - SSTableFormat.Type formatType = SSTableFormat.Type.getByName(components.get(0)); - Version version = formatType.info.getVersion(components.get(1)); - return new VersionAndType(version, formatType); - } - - @Override - public String toString() - { - return formatType.name + '-' + version; - } -} diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index 64fb1b0353..0fd272c505 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -20,6 +20,8 @@ package org.apache.cassandra.io.sstable.format.big; import java.io.IOException; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import com.google.common.base.Preconditions; @@ -31,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; @@ -69,13 +72,13 @@ public class BigFormat extends AbstractSSTableFormat options) { + super(NAME, options); + } + @Override + public String name() + { + return NAME; + } + + public static boolean is(SSTableFormat format) + { + return format.name().equals(NAME); } public static BigFormat getInstance() { - return instance; + return (BigFormat) Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(NAME), "Unknown SSTable format: " + NAME); } - public static boolean isDefault() + public static boolean isDefault() // TODO rename to isSelected { - return getInstance().getType() == Type.current(); + return DatabaseDescriptor.getSelectedSSTableFormat().getClass().equals(BigFormat.class); } @Override @@ -539,4 +553,20 @@ public class BigFormat extends AbstractSSTableFormat getInstance(Map options) + { + return new BigFormat(options); + } + } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java index d8989dbb27..242ba7eb4a 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java @@ -137,6 +137,7 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; */ public class RowIndexEntry extends AbstractRowIndexEntry { + private static final BigFormat FORMAT = BigFormat.getInstance(); private static final long EMPTY_SIZE = ObjectSizes.measure(new RowIndexEntry(0)); // constants for type of row-index-entry as serialized for saved-cache @@ -192,7 +193,7 @@ public class RowIndexEntry extends AbstractRowIndexEntry @Override public BigFormat getSSTableFormat() { - return BigFormat.getInstance(); + return FORMAT; } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java index b4cba50cb5..ffad83c9ed 100644 --- a/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java +++ b/src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummarySupport.java @@ -30,8 +30,8 @@ public interface IndexSummarySupport format) { - return IndexSummarySupport.class.isAssignableFrom(formatType.info.getReaderFactory().getReaderClass()); + return IndexSummarySupport.class.isAssignableFrom(format.getReaderFactory().getReaderClass()); } } diff --git a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java index f9c34d5d09..9d5db7ff49 100644 --- a/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java +++ b/src/java/org/apache/cassandra/io/sstable/keycache/KeyCacheSupport.java @@ -69,7 +69,7 @@ public interface KeyCacheSupport> { KeyCache keyCache = getKeyCache(); AbstractRowIndexEntry cachedEntry = keyCache.get(key, updateStats); - assert cachedEntry == null || cachedEntry.getSSTableFormat() == ((T) this).descriptor.formatType.info; + assert cachedEntry == null || cachedEntry.getSSTableFormat() == ((T) this).descriptor.version.format; return cachedEntry; } @@ -80,7 +80,7 @@ public interface KeyCacheSupport> default void cacheKey(@Nonnull DecoratedKey key, @Nonnull AbstractRowIndexEntry info) { T reader = (T) this; - assert info.getSSTableFormat() == reader.descriptor.formatType.info; + assert info.getSSTableFormat() == reader.descriptor.version.format; KeyCacheKey cacheKey = getCacheKey(key); getKeyCache().put(cacheKey, info); @@ -89,9 +89,9 @@ public interface KeyCacheSupport> @Nonnull AbstractRowIndexEntry deserializeKeyCacheValue(@Nonnull DataInputPlus input) throws IOException; - static boolean isSupportedBy(SSTableFormat.Type formatType) + static boolean isSupportedBy(SSTableFormat format) { - return KeyCacheSupport.class.isAssignableFrom(formatType.info.getReaderFactory().getReaderClass()); + return KeyCacheSupport.class.isAssignableFrom(format.getReaderFactory().getReaderClass()); } } diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 63481fcd7f..707cca25e5 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -28,6 +28,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Timer; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.sstable.GaugeProvider; @@ -172,7 +173,7 @@ public class KeyspaceMetrics public final Meter rowIndexSizeAborts; public final Histogram rowIndexSize; - public final ImmutableMap>> formatSpecificGauges; + public final ImmutableMap, ImmutableMap>> formatSpecificGauges; public final MetricNameFactory factory; private final Keyspace keyspace; @@ -288,20 +289,20 @@ public class KeyspaceMetrics } } - private ImmutableMap>> createFormatSpecificGauges(Keyspace keyspace) + private ImmutableMap, ImmutableMap>> createFormatSpecificGauges(Keyspace keyspace) { - ImmutableMap.Builder>> builder = ImmutableMap.builder(); - for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) + ImmutableMap.Builder, ImmutableMap>> builder = ImmutableMap.builder(); + for (SSTableFormat format : DatabaseDescriptor.getSSTableFormats().values()) { ImmutableMap.Builder> gauges = ImmutableMap.builder(); - for (GaugeProvider gaugeProvider : formatType.info.getFormatSpecificMetricsProviders().getGaugeProviders()) + for (GaugeProvider gaugeProvider : format.getFormatSpecificMetricsProviders().getGaugeProviders()) { String finalName = gaugeProvider.name; allMetrics.add(() -> releaseMetric(finalName)); Gauge gauge = Metrics.register(factory.createMetricName(finalName), gaugeProvider.getKeyspaceGauge(keyspace)); gauges.put(gaugeProvider.name, gauge); } - builder.put(formatType, gauges.build()); + builder.put(format, gauges.build()); } return builder.build(); } diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index dc90318e19..f4b9221aa5 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -42,6 +42,7 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.Timer; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.lifecycle.SSTableSet; @@ -278,7 +279,7 @@ public class TableMetrics public final TableMeter rowIndexSizeAborts; public final TableHistogram rowIndexSize; - public final ImmutableMap>> formatSpecificGauges; + public final ImmutableMap, ImmutableMap>> formatSpecificGauges; private static Pair totalNonSystemTablesSize(Predicate predicate) { @@ -891,18 +892,18 @@ public class TableMetrics } } - private ImmutableMap>> createFormatSpecificGauges(ColumnFamilyStore cfs) + private ImmutableMap, ImmutableMap>> createFormatSpecificGauges(ColumnFamilyStore cfs) { - ImmutableMap.Builder>> builder = ImmutableMap.builder(); - for (SSTableFormat.Type formatType : SSTableFormat.Type.values()) + ImmutableMap.Builder, ImmutableMap>> builder = ImmutableMap.builder(); + for (SSTableFormat format : DatabaseDescriptor.getSSTableFormats().values()) { ImmutableMap.Builder> gauges = ImmutableMap.builder(); - for (GaugeProvider gaugeProvider : formatType.info.getFormatSpecificMetricsProviders().getGaugeProviders()) + for (GaugeProvider gaugeProvider : format.getFormatSpecificMetricsProviders().getGaugeProviders()) { Gauge gauge = createTableGauge(gaugeProvider.name, gaugeProvider.getTableGauge(cfs), gaugeProvider.getGlobalGauge()); gauges.put(gaugeProvider.name, gauge); } - builder.put(formatType, gauges.build()); + builder.put(format, gauges.build()); } return builder.build(); } diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java index 70ce53db54..0dc31105ab 100644 --- a/src/java/org/apache/cassandra/service/CacheService.java +++ b/src/java/org/apache/cassandra/service/CacheService.java @@ -22,12 +22,14 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.ExecutionException; +import org.apache.commons.lang3.tuple.ImmutableTriple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,19 +51,19 @@ import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.filter.DataLimits; -import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.partitions.CachedBTreePartition; import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableIdFactory; -import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteArrayUtil; import org.apache.cassandra.utils.ByteBufferUtil; @@ -358,84 +360,126 @@ public class CacheService implements CacheServiceMBean logger.debug("cache saves completed"); } - public static class CounterCacheSerializer implements CacheSerializer + public static class CounterCacheSerializer extends CacheSerializer { public void serialize(CounterCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException { assert(cfs.metadata().isCounter()); - TableMetadata tableMetadata = cfs.metadata(); - tableMetadata.id.serialize(out); - out.writeUTF(tableMetadata.indexName().orElse("")); + writeCFS(out, cfs); key.write(out); } - public Future> deserialize(DataInputPlus in, final ColumnFamilyStore cfs) throws IOException + public Future> deserialize(DataInputPlus in) throws IOException { //Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a //parameter so they aren't deserialized here, even though they are serialized by this serializer + ColumnFamilyStore cfs = readCFS(in); if (cfs == null) return null; final CounterCacheKey cacheKey = CounterCacheKey.read(cfs.metadata(), in); if (!cfs.metadata().isCounter() || !cfs.isCounterCacheEnabled()) return null; - return Stage.READ.submit(new Callable>() - { - public Pair call() throws Exception - { - ByteBuffer value = cacheKey.readCounterValue(cfs); - return value == null - ? null - : Pair.create(cacheKey, CounterContext.instance().getLocalClockAndCount(value)); - } + return Stage.READ.submit(() -> { + ByteBuffer value = cacheKey.readCounterValue(cfs); + return value == null + ? null + : Pair.create(cacheKey, CounterContext.instance().getLocalClockAndCount(value)); }); } } - public static class RowCacheSerializer implements CacheSerializer + public static class RowCacheSerializer extends CacheSerializer { public void serialize(RowCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException { assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes - TableMetadata tableMetadata = cfs.metadata(); - tableMetadata.id.serialize(out); - out.writeUTF(tableMetadata.indexName().orElse("")); + writeCFS(out, cfs); ByteArrayUtil.writeWithLength(key.key, out); } - public Future> deserialize(DataInputPlus in, final ColumnFamilyStore cfs) throws IOException + public Future> deserialize(DataInputPlus in) throws IOException { //Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a //parameter so they aren't deserialized here, even though they are serialized by this serializer + ColumnFamilyStore cfs = readCFS(in); final ByteBuffer buffer = ByteBufferUtil.readWithLength(in); if (cfs == null || !cfs.isRowCacheEnabled()) return null; final int rowsToCache = cfs.metadata().params.caching.rowsPerPartitionToCache(); assert(!cfs.isIndex());//Shouldn't have row cache entries for indexes - return Stage.READ.submit(new Callable>() - { - public Pair call() throws Exception + return Stage.READ.submit(() -> { + DecoratedKey key = cfs.decorateKey(buffer); + int nowInSec = FBUtilities.nowInSeconds(); + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata(), nowInSec, key); + try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller)) { - DecoratedKey key = cfs.decorateKey(buffer); - int nowInSec = FBUtilities.nowInSeconds(); - SinglePartitionReadCommand cmd = SinglePartitionReadCommand.fullPartitionRead(cfs.metadata(), nowInSec, key); - try (ReadExecutionController controller = cmd.executionController(); UnfilteredRowIterator iter = cmd.queryMemtableAndDisk(cfs, controller)) - { - CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec); - return Pair.create(new RowCacheKey(cfs.metadata(), key), toCache); - } + CachedPartition toCache = CachedBTreePartition.create(DataLimits.cqlLimits(rowsToCache).filter(iter, nowInSec, true), nowInSec); + return Pair.create(new RowCacheKey(cfs.metadata(), key), toCache); } }); } } - public static class KeyCacheSerializer implements CacheSerializer + public static class KeyCacheSerializer extends CacheSerializer { - // For column families with many SSTables the linear nature of getSSTables slowed down KeyCache loading - // by orders of magnitude. So we cache the sstables once and rely on cleanupAfterDeserialize to cleanup any - // cached state we may have accumulated during the load. - Map, Map> cachedSSTableReaders = new ConcurrentHashMap<>(); + private final ArrayList, SSTableFormat>> readers = new ArrayList<>(); + private final LinkedHashMap> readerOrdinals = new LinkedHashMap<>(); + + @Override + public void serializeMetadata(DataOutputPlus out) throws IOException + { + super.serializeMetadata(out); + out.writeUnsignedVInt32(readerOrdinals.size()); + Descriptor desc; + for (Map.Entry> table : readerOrdinals.entrySet()) + { + desc = table.getKey(); + ColumnFamilyStore cfs = table.getValue().right; + super.writeCFS(out, cfs); + out.writeUTF(desc.version.format.name()); + out.writeUTF(desc.version.toString()); + ByteBufferUtil.writeWithShortLength(desc.id.asBytes(), out); + } + } + + @Override + public void deserializeMetadata(DataInputPlus in) throws IOException + { + super.deserializeMetadata(in); + Map>, SSTableReader>> tmpReaders = new HashMap<>(); + int sstablesNum = in.readUnsignedVInt32(); + readers.clear(); + readers.ensureCapacity(sstablesNum); + for (int i = 0; i < sstablesNum; i++) + { + ColumnFamilyStore cfs = readCFS(in); + String formatName = in.readUTF(); + SSTableFormat format = Objects.requireNonNull(DatabaseDescriptor.getSSTableFormats().get(formatName), "Unknown SSTable format: " + formatName); + String version = in.readUTF(); + SSTableId id = SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(in)); + + SSTableReader reader = null; + if (cfs != null) + { + Map>, SSTableReader> readersMap = tmpReaders.get(cfs); + if (readersMap == null) + { + Set liveReaders = cfs.getLiveSSTables(); + readersMap = new HashMap<>(liveReaders.size()); + for (SSTableReader r : liveReaders) + readersMap.put(ImmutableTriple.of(r.descriptor.id, r.descriptor.version.toString(), r.descriptor.version.format), r); + tmpReaders.put(cfs, readersMap); + } + reader = readersMap.get(ImmutableTriple.of(id, version, format)); + } + if (reader instanceof KeyCacheSupport) + readers.add(Pair.create((KeyCacheSupport) reader, format)); + else + readers.add(Pair.create(null, format)); + } + } public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException { @@ -443,93 +487,74 @@ public class CacheService implements CacheServiceMBean if (entry == null) return; - TableMetadata tableMetadata = cfs.metadata(); - tableMetadata.id.serialize(out); - out.writeUTF(tableMetadata.indexName().orElse("")); - ByteArrayUtil.writeWithLength(key.key, out); - if (key.desc.id instanceof SequenceBasedSSTableId) - { - out.writeInt(((SequenceBasedSSTableId) key.desc.id).generation); - } - else - { - out.writeInt(Integer.MIN_VALUE); // backwards compatibility for "int based generation only" - ByteBufferUtil.writeWithShortLength(key.desc.id.asBytes(), out); - } - // Format type id is stored so that in case there is no sstable for the key we can still figure out which - // serializer (of which sstable format) was used and thus, we can use the right implemnentation to skip - // the unmatched entry - out.writeByte(key.desc.formatType.ordinal); + writeSSTable(cfs, key.desc, out); + out.writeInt(key.key.length); + out.write(key.key); entry.serializeForCache(out); - } - public Future> deserialize(DataInputPlus input, ColumnFamilyStore cfs) throws IOException + public Future> deserialize(DataInputPlus input) throws IOException { - boolean skipEntry = cfs == null || !cfs.isKeyCacheEnabled(); + Pair, SSTableFormat> reader = readSSTable(input); + boolean skipEntry = reader.left == null || !reader.left.getKeyCache().isEnabled(); - //Keyspace and CF name are deserialized by AutoSaving cache and used to fetch the CFS provided as a - //parameter so they aren't deserialized here, even though they are serialized by this serializer int keyLength = input.readInt(); if (keyLength > FBUtilities.MAX_UNSIGNED_SHORT) - { throw new IOException(String.format("Corrupted key cache. Key length of %d is longer than maximum of %d", keyLength, FBUtilities.MAX_UNSIGNED_SHORT)); - } ByteBuffer key = ByteBufferUtil.read(input, keyLength); - int generation = input.readInt(); - SSTableId generationId = generation == Integer.MIN_VALUE - ? SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(input)) - : new SequenceBasedSSTableId(generation); // Backwards compatibility for "int based generation sstables" - int typeOrdinal = input.readByte(); - SSTableFormat.Type type; - try - { - type = SSTableFormat.Type.getByOrdinal(typeOrdinal); - } - catch (RuntimeException ex) - { - throw new IOException("Failed to deserialize key of key cache - invalid type ordinal " + typeOrdinal, ex); - } - SSTableFormat.KeyCacheValueSerializer serializer = type.info.getKeyCacheValueSerializer(); - SSTableReader reader = null; - if (!skipEntry) - { - Pair qualifiedName = Pair.create(cfs.metadata.keyspace, cfs.metadata.name); - Map generationToSSTableReader = cachedSSTableReaders.get(qualifiedName); - if (generationToSSTableReader == null) - { - generationToSSTableReader = new HashMap<>(cfs.getLiveSSTables().size()); - for (SSTableReader ssTableReader : cfs.getSSTables(SSTableSet.CANONICAL)) - { - generationToSSTableReader.put(ssTableReader.descriptor.id, ssTableReader); - } - - cachedSSTableReaders.putIfAbsent(qualifiedName, generationToSSTableReader); - } - reader = generationToSSTableReader.get(generationId); - } - - if (skipEntry || reader == null) + if (skipEntry) { // The sstable doesn't exist anymore, so we can't be sure of the exact version and assume its the current version. The only case where we'll be // wrong is during upgrade, in which case we fail at deserialization. This is not a huge deal however since 1) this is unlikely enough that // this won't affect many users (if any) and only once, 2) this doesn't prevent the node from starting and 3) CASSANDRA-10219 shows that this // part of the code has been broken for a while without anyone noticing (it is, btw, still broken until CASSANDRA-10219 is fixed). + SSTableFormat.KeyCacheValueSerializer serializer = reader.right.getKeyCacheValueSerializer(); serializer.skip(input); return null; } - KeyCacheSupport keyCacheSupportingReader = (KeyCacheSupport) reader; - AbstractRowIndexEntry cacheValue = keyCacheSupportingReader.deserializeKeyCacheValue(input); - KeyCacheKey cacheKey = keyCacheSupportingReader.getCacheKey(key); + long pos = ((RandomAccessReader) input).getPosition(); + AbstractRowIndexEntry cacheValue; + try + { + cacheValue = reader.left.deserializeKeyCacheValue(input); + } catch (RuntimeException | Error ex) + { + logger.error("Deserializing key cache entry at {} for {}", pos, reader.left); + throw ex; + } + KeyCacheKey cacheKey = reader.left.getCacheKey(key); return ImmediateFuture.success(Pair.create(cacheKey, cacheValue)); } + private void writeSSTable(ColumnFamilyStore cfs, Descriptor desc, DataOutputPlus out) throws IOException + { + getOrCreateCFSOrdinal(cfs); + Pair existing = readerOrdinals.putIfAbsent(desc, Pair.create(readerOrdinals.size(), cfs)); + int ordinal = existing == null ? readerOrdinals.size() - 1 : existing.left; + out.writeUnsignedVInt32(ordinal); + } + + private Pair, SSTableFormat> readSSTable(DataInputPlus input) throws IOException + { + int ordinal = input.readUnsignedVInt32(); + if (ordinal >= readers.size()) + throw new IOException("Corrupted key cache. Failed to deserialize key of key cache - invalid sstable ordinal " + ordinal); + return readers.get(ordinal); + } + public void cleanupAfterDeserialize() { - cachedSSTableReaders.clear(); + super.cleanupAfterDeserialize(); + readers.clear(); + } + + public void cleanupAfterSerialize() + { + super.cleanupAfterSerialize(); + readerOrdinals.clear(); } } } diff --git a/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java b/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java index de78892c9e..6a7baf34c9 100644 --- a/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java +++ b/src/java/org/apache/cassandra/service/SSTablesGlobalTracker.java @@ -36,7 +36,7 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.lifecycle.Tracker; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.notifications.INotification; import org.apache.cassandra.notifications.INotificationConsumer; import org.apache.cassandra.notifications.InitialSSTableAddedNotification; @@ -78,23 +78,23 @@ public class SSTablesGlobalTracker implements INotificationConsumer private final Set allSSTables = ConcurrentHashMap.newKeySet(); - private final VersionAndType currentVersion; + private final Version currentVersion; private int sstablesForCurrentVersion; - private final Map sstablesForOtherVersions = new HashMap<>(); + private final Map sstablesForOtherVersions = new HashMap<>(); - private volatile ImmutableSet versionsInUse = ImmutableSet.of(); + private volatile ImmutableSet versionsInUse = ImmutableSet.of(); private final Set subscribers = new CopyOnWriteArraySet<>(); - public SSTablesGlobalTracker(SSTableFormat.Type currentSSTableFormat) + public SSTablesGlobalTracker(SSTableFormat currentSSTableFormat) { - this.currentVersion = new VersionAndType(currentSSTableFormat.info.getLatestVersion(), currentSSTableFormat); + this.currentVersion = currentSSTableFormat.getLatestVersion(); } /** * The set of all sstable versions currently in use on this node. */ - public Set versionsInUse() + public Set versionsInUse() { return versionsInUse; } @@ -151,7 +151,7 @@ public class SSTablesGlobalTracker implements INotificationConsumer synchronized block. */ int currentDelta = 0; - Map othersDelta = null; + Map othersDelta = null; /* Note: we deal with removes first as if a notification both removes and adds, it's a compaction and while it should never remove and add the same descriptor in practice, doing the remove first is more logical. @@ -161,7 +161,7 @@ public class SSTablesGlobalTracker implements INotificationConsumer if (!allSSTables.remove(desc)) continue; - VersionAndType version = version(desc); + Version version = desc.version; if (currentVersion.equals(version)) --currentDelta; else @@ -172,7 +172,7 @@ public class SSTablesGlobalTracker implements INotificationConsumer if (!allSSTables.add(desc)) continue; - VersionAndType version = version(desc); + Version version = desc.version; if (currentVersion.equals(version)) ++currentDelta; else @@ -196,9 +196,9 @@ public class SSTablesGlobalTracker implements INotificationConsumer if (othersDelta != null) { - for (Map.Entry entry : othersDelta.entrySet()) + for (Map.Entry entry : othersDelta.entrySet()) { - VersionAndType version = entry.getKey(); + Version version = entry.getKey(); int delta = entry.getValue(); /* Updates the count, removing the version if it reaches 0 (note: we could use Map#compute for this, @@ -221,16 +221,16 @@ public class SSTablesGlobalTracker implements INotificationConsumer return triggerUpdate; } - private static ImmutableSet computeVersionsInUse(int sstablesForCurrentVersion, VersionAndType currentVersion, Map sstablesForOtherVersions) + private static ImmutableSet computeVersionsInUse(int sstablesForCurrentVersion, Version currentVersion, Map sstablesForOtherVersions) { - ImmutableSet.Builder builder = ImmutableSet.builder(); + ImmutableSet.Builder builder = ImmutableSet.builder(); if (sstablesForCurrentVersion > 0) builder.add(currentVersion); builder.addAll(sstablesForOtherVersions.keySet()); return builder.build(); } - private static int sanitizeSSTablesCount(int sstableCount, VersionAndType version) + private static int sanitizeSSTablesCount(int sstableCount, Version version) { if (sstableCount >= 0) return sstableCount; @@ -241,7 +241,7 @@ public class SSTablesGlobalTracker implements INotificationConsumer */ noSpamLogger.error("Invalid state while handling sstables change notification: the number of sstables for " + "version {} was computed to {}. This indicate a bug and please report it, but it should " + - "not have adverse consequences.", version, sstableCount, new RuntimeException()); + "not have adverse consequences.", version.toFormatAndVersionString(), sstableCount, new RuntimeException()); return 0; } @@ -267,18 +267,13 @@ public class SSTablesGlobalTracker implements INotificationConsumer return Collections.emptyList(); } - private static Map update(Map counts, - VersionAndType toUpdate, + private static Map update(Map counts, + Version toUpdate, int delta) { - Map m = counts == null ? new HashMap<>() : counts; + Map m = counts == null ? new HashMap<>() : counts; m.merge(toUpdate, delta, (a, b) -> (a + b == 0) ? null : (a + b)); return m; } - @VisibleForTesting - static VersionAndType version(Descriptor sstable) - { - return new VersionAndType(sstable.version, sstable.formatType); - } } diff --git a/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java b/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java index 352bec7ba8..08f67a4ec8 100644 --- a/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java +++ b/src/java/org/apache/cassandra/service/SSTablesVersionsInUseChangeNotification.java @@ -18,9 +18,11 @@ package org.apache.cassandra.service; +import java.util.stream.Collectors; + import com.google.common.collect.ImmutableSet; -import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.notifications.INotification; /** @@ -35,9 +37,9 @@ public class SSTablesVersionsInUseChangeNotification implements INotification /** * The set of all sstable versions in use on this node at the time of this notification. */ - public final ImmutableSet versionsInUse; + public final ImmutableSet versionsInUse; - SSTablesVersionsInUseChangeNotification(ImmutableSet versionsInUse) + SSTablesVersionsInUseChangeNotification(ImmutableSet versionsInUse) { this.versionsInUse = versionsInUse; } @@ -45,6 +47,6 @@ public class SSTablesVersionsInUseChangeNotification implements INotification @Override public String toString() { - return String.format("SSTablesInUseChangeNotification(%s)", versionsInUse); + return String.format("SSTablesInUseChangeNotification(%s)", versionsInUse.stream().map(Version::toFormatAndVersionString).collect(Collectors.toList())); } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 0178871cf1..172c444072 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -149,8 +149,7 @@ import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.io.sstable.IScrubber; import org.apache.cassandra.io.sstable.IVerifier; import org.apache.cassandra.io.sstable.SSTableLoader; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.PathUtils; @@ -481,7 +480,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE jmxObjectName = "org.apache.cassandra.db:type=StorageService"; - sstablesTracker = new SSTablesGlobalTracker(SSTableFormat.Type.current()); + sstablesTracker = new SSTablesGlobalTracker(DatabaseDescriptor.getSelectedSSTableFormat()); } private void registerMBeans() @@ -1148,7 +1147,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (!(notification instanceof SSTablesVersionsInUseChangeNotification)) return; - Set versions = ((SSTablesVersionsInUseChangeNotification)notification).versionsInUse; + Set versions = ((SSTablesVersionsInUseChangeNotification)notification).versionsInUse; logger.debug("Updating local sstables version in Gossip to {}", versions); Gossiper.instance.addLocalApplicationState(ApplicationState.SSTABLE_VERSIONS, diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java index 16e8326242..11e9d53137 100644 --- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java +++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java @@ -327,7 +327,7 @@ public class SSTableMetadataViewer } field("SSTable", descriptor); - if (scan && descriptor.version.getVersion().compareTo("ma") >= 0) + if (scan && descriptor.version.version.compareTo("ma") >= 0) { printScannedOverview(descriptor, stats); } diff --git a/src/java/org/apache/cassandra/tools/SSTablePartitions.java b/src/java/org/apache/cassandra/tools/SSTablePartitions.java index fec5ef94be..c513853e18 100644 --- a/src/java/org/apache/cassandra/tools/SSTablePartitions.java +++ b/src/java/org/apache/cassandra/tools/SSTablePartitions.java @@ -734,8 +734,8 @@ public class SSTablePartitions notNull(desc.snapshot), notNull(desc.backup), desc.descriptor.id, - desc.descriptor.formatType.name, - desc.descriptor.version.getVersion()); + desc.descriptor.version.format.name(), + desc.descriptor.version.version); } } @@ -779,9 +779,9 @@ public class SSTablePartitions return sb.append(" #") .append(descriptor.id) .append(" (") - .append(descriptor.formatType.name) + .append(descriptor.version.format.name()) .append('-') - .append(descriptor.version.getVersion()) + .append(descriptor.version.version) .append(')') .toString(); } diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java index 8458009422..1009c3fc41 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java +++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java @@ -39,7 +39,6 @@ import org.apache.cassandra.db.compaction.Upgrader; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -95,7 +94,7 @@ public class StandaloneUpgrader try { SSTableReader sstable = SSTableReader.openNoValidation(entry.getKey(), components, cfs); - if (sstable.descriptor.version.equals(SSTableFormat.Type.current().info.getLatestVersion())) + if (sstable.descriptor.version.equals(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion())) { sstable.selfRef().release(); continue; diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java index 6147b99648..8a254e25aa 100644 --- a/src/java/org/apache/cassandra/tools/Util.java +++ b/src/java/org/apache/cassandra/tools/Util.java @@ -309,7 +309,7 @@ public final class Util public static TableMetadata metadataFromSSTable(Descriptor desc) throws IOException { if (!desc.version.isCompatible()) - throw new IOException("Unsupported SSTable version " + desc.getFormat().getType().name + "/" + desc.version); + throw new IOException("Unsupported SSTable version " + desc.getFormat().name() + "/" + desc.version); StatsComponent statsComponent = StatsComponent.load(desc, MetadataType.STATS, MetadataType.HEADER); SerializationHeader.Component header = statsComponent.serializationHeader(); diff --git a/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory b/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory new file mode 100644 index 0000000000..b9bdb4b831 --- /dev/null +++ b/src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory @@ -0,0 +1 @@ +org.apache.cassandra.io.sstable.format.big.BigFormat$BigFormatFactory \ No newline at end of file diff --git a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java index d67e68314b..afafad67ed 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CacheLoaderBench.java @@ -25,13 +25,19 @@ import java.util.concurrent.TimeUnit; import org.junit.Assert; +import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.Util; import org.apache.cassandra.cache.AutoSavingCache; import org.apache.cassandra.cache.KeyCacheKey; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -45,6 +51,7 @@ import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -55,24 +62,40 @@ import org.openjdk.jmh.annotations.Warmup; @SuppressWarnings("unused") @BenchmarkMode(Mode.SampleTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) -@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 2, time = 2, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 1, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 1, time = 10, timeUnit = TimeUnit.SECONDS) @Fork(value = 1) @Threads(1) @State(Scope.Benchmark) -public class CacheLoaderBench extends CQLTester +public class CacheLoaderBench { - private static final int numSSTables = 1000; + private static final int numSSTables = 100; + private static final int numKeysPerTable = 10000; + private final Random random = new Random(); + @Param({ "true", "false" }) + boolean useUUIDGenerationIdentifiers; + @Setup(Level.Trial) public void setup() throws Throwable { - CQLTester.prepareServer(); - String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); - String table1 = createTable(keyspace, "CREATE TABLE %s (key text PRIMARY KEY, val text)"); - String table2 = createTable(keyspace, "CREATE TABLE %s (key text PRIMARY KEY, val text)"); + DatabaseDescriptor.daemonInitialization(() -> { + Config config = DatabaseDescriptor.loadConfig(); + config.key_cache_size = new DataStorageSpec.LongMebibytesBound(256); + config.uuid_sstable_identifiers_enabled = useUUIDGenerationIdentifiers; + config.dump_heap_on_uncaught_exception = false; + return config; + }); + ServerTestUtils.prepareServer(); + String keyspace = "ks"; + String table1 = "tab1"; + String table2 = "tab2"; + + QueryProcessor.executeInternal(String.format("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false", keyspace)); + QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (key text PRIMARY KEY, val text)", keyspace, table1)); + QueryProcessor.executeInternal(String.format("CREATE TABLE %s.%s (key text PRIMARY KEY, val text)", keyspace, table2)); Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction)); @@ -87,16 +110,18 @@ public class CacheLoaderBench extends CQLTester columnFamilyStores.add(cfs1); columnFamilyStores.add(cfs2); - logger.info("Creating {} sstables", numSSTables); for (ColumnFamilyStore cfs: columnFamilyStores) { cfs.truncateBlocking(); for (int i = 0; i < numSSTables ; i++) { ColumnMetadata colDef = ColumnMetadata.regularColumn(cfs.metadata(), ByteBufferUtil.bytes("val"), AsciiType.instance); - RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() + random.nextInt(), "key"); - rowBuilder.add(colDef, "val1"); - rowBuilder.build().apply(); + for (int k = 0; k < numKeysPerTable; k++) + { + RowUpdateBuilder rowBuilder = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis() + random.nextInt(), "key" + k); + rowBuilder.add(colDef, "val1"); + rowBuilder.build().apply(); + } cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); } @@ -104,7 +129,12 @@ public class CacheLoaderBench extends CQLTester // preheat key cache for (SSTableReader sstable : cfs.getLiveSSTables()) - sstable.getPosition(Util.dk("key"), SSTableReader.Operator.EQ); + { + for (int k = 0; k < numKeysPerTable; k++) + { + sstable.getPosition(Util.dk("key" + k), SSTableReader.Operator.EQ); + } + } } AutoSavingCache keyCache = CacheService.instance.keyCache; @@ -123,8 +153,9 @@ public class CacheLoaderBench extends CQLTester @TearDown(Level.Trial) public void teardown() { - CQLTester.cleanup(); CQLTester.tearDownClass(); + CommitLog.instance.stopUnsafe(true); + CQLTester.cleanup(); } @Benchmark diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java index 64389221a1..59acfc6ae6 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java +++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBenchmark.java @@ -54,7 +54,6 @@ import org.apache.cassandra.net.AsyncStreamingInputPlus; import org.apache.cassandra.net.AsyncStreamingOutputPlus; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionInfo; import org.apache.cassandra.streaming.StreamCoordinator; @@ -63,6 +62,7 @@ import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -134,7 +134,6 @@ public class ZeroCopyStreamingBenchmark CassandraStreamHeader entireSSTableStreamHeader = CassandraStreamHeader.builder() - .withSSTableFormat(sstable.descriptor.formatType) .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(sstable.estimatedKeys()) @@ -154,7 +153,6 @@ public class ZeroCopyStreamingBenchmark List> requestedRanges = Arrays.asList(new Range<>(sstable.first.minValue().getToken(), sstable.last.getToken())); CassandraStreamHeader partialSSTableStreamHeader = CassandraStreamHeader.builder() - .withSSTableFormat(sstable.descriptor.formatType) .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(sstable.estimatedKeys()) diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index 1285ad9291..3e36692197 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -115,7 +115,6 @@ import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SSTableLoader; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; import org.apache.cassandra.io.util.DataInputPlus; @@ -1260,6 +1259,6 @@ public class Util public static RuntimeException testMustBeImplementedForSSTableFormat() { - return new UnsupportedOperationException("Test must be implemented for sstable format " + SSTableFormat.Type.current().info.getClass().getName()); + return new UnsupportedOperationException("Test must be implemented for sstable format " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName()); } } diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java index 570f8c311d..50145e10b3 100644 --- a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java +++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java @@ -31,7 +31,6 @@ import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.schema.ColumnMetadata; @@ -51,7 +50,7 @@ public class AutoSavingCacheTest public static void defineSchema() throws ConfigurationException { DatabaseDescriptor.daemonInitialization(); - Assume.assumeTrue(KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current())); + Assume.assumeTrue(KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat())); SchemaLoader.prepareServer(); SchemaLoader.createKeyspace(KEYSPACE1, KeyspaceParams.simple(1), diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java index b07bfa384f..2aeeeefb0b 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorRefTest.java @@ -92,6 +92,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.config.Config$PaxosStatePurging", "org.apache.cassandra.config.Config$PaxosVariant", "org.apache.cassandra.config.Config$RepairCommandPoolFullStrategy", + "org.apache.cassandra.config.Config$SSTableConfig", "org.apache.cassandra.config.Config$UserFunctionTimeoutPolicy", "org.apache.cassandra.config.ConfigBeanInfo", "org.apache.cassandra.config.ConfigCustomizer", @@ -203,6 +204,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.io.sstable.format.SSTableFormat", "org.apache.cassandra.io.sstable.format.SSTableFormat$Components", "org.apache.cassandra.io.sstable.format.SSTableFormat$Components$Types", + "org.apache.cassandra.io.sstable.format.SSTableFormat$Factory", "org.apache.cassandra.io.sstable.format.SSTableFormat$KeyCacheValueSerializer", "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableReaderFactory", "org.apache.cassandra.io.sstable.format.SSTableFormat$SSTableWriterFactory", @@ -218,6 +220,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.io.sstable.format.SortedTableWriter$Builder", "org.apache.cassandra.io.sstable.format.Version", "org.apache.cassandra.io.sstable.format.big.BigFormat", + "org.apache.cassandra.io.sstable.format.big.BigFormat$BigFormatFactory", "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableReaderFactory", "org.apache.cassandra.io.sstable.format.big.BigFormat$BigTableWriterFactory", "org.apache.cassandra.io.sstable.format.big.BigFormat$BigVersion", @@ -265,7 +268,7 @@ public class DatabaseDescriptorRefTest "org.apache.cassandra.utils.concurrent.RefCounted", "org.apache.cassandra.utils.concurrent.SelfRefCounted", "org.apache.cassandra.utils.concurrent.Transactional", - "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException", + "org.apache.cassandra.utils.concurrent.UncheckedInterruptedException" }; static final Set checkedClasses = new HashSet<>(Arrays.asList(validClasses)); diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java index 779b39eea3..9c7dac159c 100644 --- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java +++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java @@ -34,7 +34,6 @@ import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.keycache.KeyCacheSupport; import org.apache.cassandra.metrics.CacheMetrics; import org.apache.cassandra.metrics.CassandraMetricsRegistry; @@ -102,7 +101,7 @@ public class KeyCacheCqlTest extends CQLTester { CachingParams.DEFAULT = CachingParams.CACHE_NOTHING; CQLTester.setUpClass(); - sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current()); + sstableImplCachesKeys = KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat()); } /** @@ -619,7 +618,7 @@ public class KeyCacheCqlTest extends CQLTester private long recentBloomFilterFalsePositives() { - return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.formatSpecificGauges.get(SSTableFormat.Type.current()) + return getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).metric.formatSpecificGauges.get(DatabaseDescriptor.getSelectedSSTableFormat()) .get(BloomFilterMetrics.instance.recentBloomFilterFalsePositives.name) .getValue() .longValue(); diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index e678413f20..8855ceb0d3 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -349,7 +349,7 @@ public class ColumnFamilyStoreTest KEYSPACE2, CF_STANDARD1, liveSSTable.descriptor.id, - liveSSTable.descriptor.formatType); + liveSSTable.descriptor.version.format); for (Component c : liveSSTable.getComponents()) assertTrue("Cannot find backed-up file:" + desc.fileFor(c), desc.fileFor(c).exists()); } diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index 39bf4b36c3..6954fe248a 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -82,7 +82,6 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; @@ -253,7 +252,7 @@ public class DirectoriesTest File snapshotDir = new File(tableDir, Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + tag); snapshotDir.tryCreateDirectories(); - Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), SSTableFormat.Type.current()); + Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat()); createFakeSSTable(sstableDesc); SnapshotManifest manifest = null; @@ -273,14 +272,14 @@ public class DirectoriesTest private List createFakeSSTable(File dir, String cf, int gen) { - Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), SSTableFormat.Type.current()); + Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), DatabaseDescriptor.getSelectedSSTableFormat()); return createFakeSSTable(desc); } private List createFakeSSTable(Descriptor desc) { List components = new ArrayList<>(3); - for (Component c : SSTableFormat.Type.current().info.uploadComponents()) + for (Component c : DatabaseDescriptor.getSelectedSSTableFormat().uploadComponents()) { File f = desc.fileFor(c); f.createFileIfNotExists(); @@ -315,7 +314,7 @@ public class DirectoriesTest Directories directories = new Directories(cfm, toDataDirectories(tempDataDir)); assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables()); - Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.current()); + Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat()); File snapshotDir = new File(cfDir(cfm), File.pathSeparator() + Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + LEGACY_SNAPSHOT_NAME); assertEquals(snapshotDir.toCanonical(), Directories.getSnapshotDirectory(desc, LEGACY_SNAPSHOT_NAME)); @@ -391,7 +390,7 @@ public class DirectoriesTest { String tag = "test"; Directories directories = new Directories(cfm, toDataDirectories(tempDataDir)); - Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), SSTableFormat.Type.current()); + Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, tag); List files = new LinkedList<>(); @@ -438,8 +437,8 @@ public class DirectoriesTest { assertEquals(cfDir(INDEX_CFM), dir); } - Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.current()); - Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.current()); + Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); + Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); // snapshot dir should be created under its parent's File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, "test"); @@ -452,9 +451,9 @@ public class DirectoriesTest assertTrue(indexDirectories.snapshotExists("test")); // check true snapshot size - Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.current()); + Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); createFile(parentSnapshot.fileFor(Components.DATA), 30); - Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.current()); + Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); createFile(indexSnapshot.fileFor(Components.DATA), 40); assertEquals(30, parentDirectories.trueSnapshotsSize()); @@ -599,7 +598,7 @@ public class DirectoriesTest final String n = Long.toString(nanoTime()); Callable directoryGetter = () -> { - Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.current()); + Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), DatabaseDescriptor.getSelectedSSTableFormat()); return Directories.getSnapshotDirectory(desc, n); }; List> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter)); diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java index 1f0ef8bc5c..d77e1fefc0 100644 --- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java @@ -66,7 +66,7 @@ public class SerializationHeaderTest @Test public void testWrittenAsDifferentKind() throws Exception { - SSTableFormat format = SSTableFormat.Type.current().info; + SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); final String tableName = "testWrittenAsDifferentKind"; ColumnIdentifier v = ColumnIdentifier.getInterned("v", false); TableMetadata schemaWithStatic = TableMetadata.builder(KEYSPACE, tableName) @@ -91,7 +91,7 @@ public class SerializationHeaderTest try { BiFunction>, Callable> writer = (schema, clusteringFunction) -> () -> { - Descriptor descriptor = new Descriptor(format.getLatestVersion(), dir, schema.keyspace, schema.name, id.get(), format.getType()); + Descriptor descriptor = new Descriptor(format.getLatestVersion(), dir, schema.keyspace, schema.name, id.get()); SerializationHeader header = SerializationHeader.makeWithoutStats(schema); try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index eed4da8cd1..4386c21938 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -41,6 +41,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; @@ -49,7 +50,6 @@ import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; @@ -1262,7 +1262,7 @@ public class LogTransactionTest extends AbstractTransactionalTest { if (BigFormat.isDefault()) { - Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), SSTableFormat.Type.current()); + Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), DatabaseDescriptor.getSelectedSSTableFormat()); Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); for (Component component : components) { @@ -1297,7 +1297,7 @@ public class LogTransactionTest extends AbstractTransactionalTest } else { - throw new UnsupportedOperationException("Please implement this method for sstable format: " + SSTableFormat.Type.current().info.getClass().getName()); + throw new UnsupportedOperationException("Please implement this method for sstable format: " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName()); } } diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java index d0a81028cc..898da7c782 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java @@ -24,8 +24,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Queue; -import org.apache.cassandra.Util; -import org.apache.cassandra.io.sstable.Descriptor; import org.junit.BeforeClass; import org.junit.Test; @@ -37,19 +35,20 @@ import io.netty.channel.ChannelPromise; import io.netty.channel.DefaultFileRegion; import io.netty.channel.embedded.EmbeddedChannel; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.net.SharedDefaultFileRegion; import org.apache.cassandra.net.AsyncStreamingOutputPlus; +import org.apache.cassandra.net.SharedDefaultFileRegion; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.KeyspaceParams; -import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionInfo; import org.apache.cassandra.streaming.StreamCoordinator; @@ -58,6 +57,7 @@ import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.StreamSummary; +import org.apache.cassandra.streaming.async.NettyStreamingConnectionFactory; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -149,7 +149,6 @@ public class CassandraEntireSSTableStreamWriterTest CassandraStreamHeader header = CassandraStreamHeader.builder() - .withSSTableFormat(sstable.descriptor.formatType) .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(sstable.estimatedKeys()) diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java index 1506c952d1..a0e4e51068 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraStreamHeaderTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; @@ -39,7 +40,6 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.Component; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; @@ -146,7 +146,6 @@ public class CassandraStreamHeaderTest DecoratedKey firstKey = entireSSTable ? sstable.first : null; return CassandraStreamHeader.builder() - .withSSTableFormat(sstable.descriptor.getFormat().getType()) .withSSTableVersion(sstable.descriptor.version) .withSSTableLevel(0) .withEstimatedKeys(10) @@ -167,8 +166,7 @@ public class CassandraStreamHeaderTest TableMetadata metadata = CreateTableStatement.parse(ddl, "ks").build(); CassandraStreamHeader header = CassandraStreamHeader.builder() - .withSSTableFormat(SSTableFormat.Type.current()) - .withSSTableVersion(SSTableFormat.Type.current().info.getLatestVersion()) + .withSSTableVersion(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()) .withSSTableLevel(0) .withEstimatedKeys(0) .withSections(Collections.emptyList()) @@ -189,8 +187,7 @@ public class CassandraStreamHeaderTest CassandraStreamHeader header = CassandraStreamHeader.builder() - .withSSTableFormat(SSTableFormat.Type.current()) - .withSSTableVersion(SSTableFormat.Type.current().info.getLatestVersion()) + .withSSTableVersion(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()) .withSSTableLevel(0) .withEstimatedKeys(0) .withSections(Collections.emptyList()) diff --git a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java index 9e7c9d8fb6..aaf7e54c08 100644 --- a/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/ComponentManifestTest.java @@ -41,7 +41,7 @@ public class ComponentManifestTest public void testSerialization() { ComponentManifest expected = new ComponentManifest(new LinkedHashMap() {{ put(Components.DATA, 100L); }}); - SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializers.get(BigFormat.getInstance().getType())); + SerializationUtils.assertSerializationCycle(expected, ComponentManifest.serializers.get(BigFormat.getInstance().name())); } // Propose removing this test which now fails on VIntOutOfRange diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index c97a367d37..3c33327c97 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -226,7 +226,7 @@ public class SASIIndexTest sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.id, - sstable.descriptor.formatType); + sstable.descriptor.version.format); Set components = snapshotSSTables.get(snapshotSSTable); @@ -236,7 +236,7 @@ public class SASIIndexTest for (Component c : components) { long componentSize = snapshotSSTable.fileFor(c).length(); - if (Component.Type.fromRepresentation(c.name, sstable.descriptor.formatType) == Components.Types.SECONDARY_INDEX) + if (Component.Type.fromRepresentation(c.name, sstable.descriptor.version.format) == Components.Types.SECONDARY_INDEX) indexSize += componentSize; else tableSize += componentSize; diff --git a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java index af56ff703b..5ecba47eb3 100644 --- a/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java +++ b/test/unit/org/apache/cassandra/io/DiskSpaceMetricsTest.java @@ -31,13 +31,13 @@ import org.junit.Assume; import org.junit.Test; import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.compaction.CompactionInterruptedException; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.lifecycle.SSTableSet; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryManager; import org.apache.cassandra.io.sstable.indexsummary.IndexSummaryRedistribution; @@ -75,7 +75,7 @@ public class DiskSpaceMetricsTest extends CQLTester @Test public void summaryRedistribution() throws Throwable { - Assume.assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); + Assume.assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat())); createTable("CREATE TABLE %s (pk bigint, PRIMARY KEY (pk)) WITH min_index_interval=1"); ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); diff --git a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java index 847f3ac753..a9b76303f2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/ComponentTest.java @@ -21,44 +21,32 @@ package org.apache.cassandra.io.sstable; import java.util.HashSet; import java.util.function.Function; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.junit.Test; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.ParameterizedClass; import org.apache.cassandra.io.sstable.Component.Type; -import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat; +import org.apache.cassandra.io.sstable.SSTableFormatTest.Format1; +import org.apache.cassandra.io.sstable.SSTableFormatTest.Format2; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.mockito.Mockito; +import static org.apache.cassandra.io.sstable.SSTableFormatTest.factory; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; public class ComponentTest { - public static abstract class Format1 extends AbstractSSTableFormat - { - public final static Format1 instance = Mockito.spy(Format1.class); - } - - public static abstract class Format2 extends AbstractSSTableFormat - { - public final static Format2 instance = Mockito.spy(Format2.class); - } + private static final String SECOND = "second"; + private static final String FIRST = "first"; static { DatabaseDescriptor.daemonInitialization(() -> { Config config = DatabaseDescriptor.loadConfig(); - config.sstable_formats = Lists.newArrayList(config.sstable_formats); - config.sstable_formats.add(new ParameterizedClass(Format1.class.getName(), ImmutableMap.of(Config.SSTABLE_FORMAT_ID, "11", Config.SSTABLE_FORMAT_NAME, "first"))); - config.sstable_formats.add(new ParameterizedClass(Format2.class.getName(), ImmutableMap.of(Config.SSTABLE_FORMAT_ID, "12", Config.SSTABLE_FORMAT_NAME, "second"))); + SSTableFormatTest.configure(new Config.SSTableConfig(), new BigFormat.BigFormatFactory(), factory("first", Format1.class), factory("second", Format2.class)); return config; }); } @@ -82,11 +70,11 @@ public class ComponentTest assertThatExceptionOfType(NullPointerException.class).isThrownBy(() -> Type.createSingleton(null, "-Three.db", Format1.class)); - assertThat(Type.fromRepresentation("should be custom", BigFormat.getInstance().getType())).isSameAs(Components.Types.CUSTOM); - assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BigFormat.getInstance().getType())).isSameAs(Components.Types.TOC); - assertThat(Type.fromRepresentation(t1.repr, Format1.instance.getType())).isSameAs(t1); - assertThat(Type.fromRepresentation(t2f1.repr, Format1.instance.getType())).isSameAs(t2f1); - assertThat(Type.fromRepresentation(t2f2.repr, Format2.instance.getType())).isSameAs(t2f2); + assertThat(Type.fromRepresentation("should be custom", BigFormat.getInstance())).isSameAs(Components.Types.CUSTOM); + assertThat(Type.fromRepresentation(Components.Types.TOC.repr, BigFormat.getInstance())).isSameAs(Components.Types.TOC); + assertThat(Type.fromRepresentation(t1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t1); + assertThat(Type.fromRepresentation(t2f1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(t2f1); + assertThat(Type.fromRepresentation(t2f2.repr, DatabaseDescriptor.getSSTableFormats().get(SECOND))).isSameAs(t2f2); } @Test @@ -105,14 +93,14 @@ public class ComponentTest assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f1.getSingleton()); assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> t4f2.getSingleton()); - assertThat(Component.parse(t3f1.repr, Format1.instance.getType())).isSameAs(c1); - assertThat(Component.parse(t3f2.repr, Format2.instance.getType())).isSameAs(c2); + assertThat(Component.parse(t3f1.repr, DatabaseDescriptor.getSSTableFormats().get(FIRST))).isSameAs(c1); + assertThat(Component.parse(t3f2.repr, DatabaseDescriptor.getSSTableFormats().get("second"))).isSameAs(c2); assertThat(c1).isNotEqualTo(c2); assertThat(c1.type).isSameAs(t3f1); assertThat(c2.type).isSameAs(t3f2); - Component c3 = Component.parse("abc-Four.db", Format1.instance.getType()); - Component c4 = Component.parse("abc-Four.db", Format2.instance.getType()); + Component c3 = Component.parse("abc-Four.db", DatabaseDescriptor.getSSTableFormats().get(FIRST)); + Component c4 = Component.parse("abc-Four.db", DatabaseDescriptor.getSSTableFormats().get("second")); assertThat(c3.type).isSameAs(t4f1); assertThat(c4.type).isSameAs(t4f2); assertThat(c3.name).isEqualTo("abc-Four.db"); @@ -121,11 +109,11 @@ public class ComponentTest assertThat(c3).isNotEqualTo(c1); assertThat(c4).isNotEqualTo(c2); - Component c5 = Component.parse("abc-Five.db", Format1.instance.getType()); + Component c5 = Component.parse("abc-Five.db", DatabaseDescriptor.getSSTableFormats().get(FIRST)); assertThat(c5.type).isSameAs(Components.Types.CUSTOM); assertThat(c5.name).isEqualTo("abc-Five.db"); - Component c6 = Component.parse("Data.db", Format2.instance.getType()); + Component c6 = Component.parse("Data.db", DatabaseDescriptor.getSSTableFormats().get("second")); assertThat(c6.type).isSameAs(Components.Types.DATA); assertThat(c6).isSameAs(Components.DATA); @@ -136,7 +124,7 @@ public class ComponentTest assertThat(s1).doesNotContain(c2); assertThat(s2).doesNotContain(c1); - assertThat(Sets.newHashSet(Component.getSingletonsFor(Format1.instance))).isEqualTo(s1); - assertThat(Sets.newHashSet(Component.getSingletonsFor(Format2.instance))).isEqualTo(s2); + assertThat(Sets.newHashSet(Component.getSingletonsFor(DatabaseDescriptor.getSSTableFormats().get(FIRST)))).isEqualTo(s1); + assertThat(Sets.newHashSet(Component.getSingletonsFor(DatabaseDescriptor.getSSTableFormats().get("second")))).isEqualTo(s2); } } diff --git a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java index 9d2243b3c9..92b94b56c7 100644 --- a/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java @@ -27,7 +27,6 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Directories; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; @@ -86,12 +85,12 @@ public class DescriptorTest private void testFromFilenameFor(File dir) { - checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), SSTableFormat.Type.current())); + checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat())); // secondary index String idxName = "myidx"; File idxDir = new File(dir.absolutePath() + File.pathSeparator() + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName); - checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), SSTableFormat.Type.current())); + checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), DatabaseDescriptor.getSelectedSSTableFormat())); } private void checkFromFilename(Descriptor original) @@ -114,8 +113,8 @@ public class DescriptorTest { // Descriptor should be equal when parent directory points to the same directory File dir = new File("."); - Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.current()); - Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.current()); + Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat()); + Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), DatabaseDescriptor.getSelectedSSTableFormat()); assertEquals(desc1, desc2); assertEquals(desc1.hashCode(), desc2.hashCode()); } @@ -123,7 +122,7 @@ public class DescriptorTest @Test public void validateNames() { - String name = SSTableFormat.Type.current().name; + String name = DatabaseDescriptor.getSelectedSSTableFormat().name(); String[] fileNames = { "ma-1-" + name + "-Data.db", // 2ndary index ".idx1" + File.pathSeparator() + "ma-1-" + name + "-Data.db", diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 2cef163c7c..b3c31ec44e 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -592,7 +592,7 @@ public class LegacySSTableTest /** * Generates sstables for 8 CQL tables (see {@link #createTables(String)}) in current * sstable format (version) into {@code test/data/legacy-sstables/VERSION}, where - * {@code VERSION} matches {@link Version#getVersion() BigFormat.latestVersion.getVersion()}. + * {@code VERSION} matches {@link Version#version BigFormat.latestVersion.getVersion()}. *

* Run this test alone (e.g. from your IDE) when a new version is introduced or format changed * during development. I.e. remove the {@code @Ignore} annotation temporarily. @@ -602,7 +602,7 @@ public class LegacySSTableTest @Test public void testGenerateSstables() throws Throwable { - SSTableFormat format = SSTableFormat.Type.current().info; + SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); Random rand = new Random(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < 128; i++) diff --git a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java index 57e85927bb..2b258bad79 100644 --- a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java @@ -32,7 +32,6 @@ import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; @@ -79,7 +78,7 @@ public class RangeAwareSSTableWriterTest 0, null, false, - SSTableFormat.Type.current(), + DatabaseDescriptor.getSelectedSSTableFormat(), 0, 0, txn, diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java index 69e8670a99..58ca05ba0f 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java @@ -94,13 +94,12 @@ public class SSTableFlushObserverTest if (!directory.exists() && !directory.tryCreateDirectories()) throw new FSWriteError(new IOException("failed to create tmp directory"), directory.absolutePath()); - SSTableFormat.Type sstableFormat = SSTableFormat.Type.current(); - Descriptor descriptor = new Descriptor(sstableFormat.info.getLatestVersion(), + SSTableFormat sstableFormat = DatabaseDescriptor.getSelectedSSTableFormat(); + Descriptor descriptor = new Descriptor(sstableFormat.getLatestVersion(), directory, cfm.keyspace, cfm.name, - new SequenceBasedSSTableId(0), - sstableFormat); + new SequenceBasedSSTableId(0)); SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor) .setKeyCount(10) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java new file mode 100644 index 0000000000..30a68dd212 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.Map; +import java.util.function.BiFunction; + +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.Config.SSTableConfig; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.YamlConfigurationLoader; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.util.FileUtils; +import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.Mockito.when; + +public class SSTableFormatTest +{ + public static abstract class AbstractFormat implements SSTableFormat + { + public Map options; + public String name; + public String latestVersion; + + public AbstractFormat(String latestVersion) + { + this.latestVersion = latestVersion; + } + + @Override + public Version getVersion(String version) + { + Version v = Mockito.mock(Version.class); + when(v.toString()).thenReturn(version); + when(v.isCompatible()).thenReturn(version.charAt(0) == latestVersion.charAt(0)); + return v; + } + + @Override + public Version getLatestVersion() + { + return getVersion(latestVersion); + } + + @Override + public String name() + { + return name; + } + + static class Factory implements SSTableFormat.Factory + { + public String name; + public BiFunction, String, SSTableFormat> provider; + + public Factory(String name, BiFunction, String, SSTableFormat> provider) + { + this.name = name; + this.provider = provider; + } + + @Override + public String name() + { + return name; + } + + @Override + public SSTableFormat getInstance(Map options) + { + return provider.apply(options, name); + } + } + } + + public static AbstractFormat.Factory factory(String name, Class clazz) + { + return new AbstractFormat.Factory(name, (options, version) -> { + AbstractFormat format = Mockito.spy(clazz); + format.name = name; + format.options = options; + return format; + }); + } + + public static abstract class Format1 extends AbstractFormat + { + public Format1() + { + super("xx"); + } + } + + public static abstract class Format2 extends AbstractFormat + { + public Format2() + { + super("yy"); + } + } + + public static abstract class Format3 extends AbstractFormat + { + public Format3() + { + super("zz"); + } + } + + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.clientInitialization(); + } + + private static final String yamlContent0 = ""; + private static final SSTableConfig expected0 = new Config.SSTableConfig(); + + private static final String yamlContent1 = "sstable:\n" + + " selected_format: aaa\n"; + private static final SSTableConfig expected1 = new Config.SSTableConfig() + { + { + selected_format = "aaa"; + } + }; + + private static final String yamlContent2 = "sstable:\n" + + " selected_format: aaa\n" + + " format:\n" + + " aaa:\n" + + " param1: value1\n" + + " param2: value2\n" + + " bbb:\n" + + " param3: value3\n" + + " param4: value4\n"; + + private static final Config.SSTableConfig expected2 = new SSTableConfig() + { + { + selected_format = "aaa"; + format = ImmutableMap.of("aaa", ImmutableMap.of("param1", "value1", "param2", "value2"), + "bbb", ImmutableMap.of("param3", "value3", "param4", "value4")); + } + }; + + private static final SSTableConfig unexpected = new Config.SSTableConfig() + { + { + selected_format = "aaa"; + } + }; + + + @Test + public void testParsingYamlConfig() throws IOException + { + YamlConfigurationLoader loader = new YamlConfigurationLoader(); + File f = FileUtils.createTempFile("sstable_format_test_config", ".yaml"); + URL url = f.toPath().toUri().toURL(); + + ImmutableMap.of(yamlContent0, expected0, yamlContent1, expected1, yamlContent2, expected2).forEach((yamlContent, expected) -> { + try (FileOutputStreamPlus out = f.newOutputStream(File.WriteMode.OVERWRITE)) + { + out.write(yamlContent.getBytes()); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + Config config = loader.loadConfig(url); + assertThat(config.sstable).describedAs("Yaml: \n%s\n", yamlContent).isEqualToComparingFieldByField(expected); + }); + } + + @Before + public void before() + { + } + + public static void configure(SSTableConfig config, SSTableFormat.Factory... factories) + { + DatabaseDescriptor.resetSSTableFormats(Arrays.asList(factories), config); + } + + private void verifyFormat(String name, Map options) + { + AbstractFormat format = (AbstractFormat) DatabaseDescriptor.getSSTableFormats().get(name); + assertThat(format.name).isEqualTo(name); + assertThat(format.options).isEqualTo(options); + } + + private void verifySelectedFormat(String name) + { + assertThat(DatabaseDescriptor.getSelectedSSTableFormat().name()).isEqualTo(name); + } + + @Test + public void testValidConfig() + { + configure(expected1, factory("aaa", Format1.class)); + assertThat(DatabaseDescriptor.getSSTableFormats()).hasSize(1); + verifyFormat("aaa", ImmutableMap.of()); + verifySelectedFormat("aaa"); + + configure(expected2, factory("aaa", Format1.class), factory("bbb", Format2.class), factory("ccc", Format3.class)); + assertThat(DatabaseDescriptor.getSSTableFormats()).hasSize(3); + verifyFormat("aaa", ImmutableMap.of("param1", "value1", "param2", "value2")); + verifyFormat("bbb", ImmutableMap.of("param3", "value3", "param4", "value4")); + verifyFormat("ccc", ImmutableMap.of()); + verifySelectedFormat("aaa"); + } + + @Test + public void testConfigValidation() + { + // invalid name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("Aa", Format1.class))) + .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("a-a", Format1.class))) + .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("a1", Format1.class))) + .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string"); + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("", Format1.class))) + .withMessageContainingAll("SSTable format name", "must be non-empty, lower-case letters only string"); + + // duplicate name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("aaa", Format1.class), factory("aaa", Format2.class))) + .withMessageContainingAll("Multiple sstable format implementations with the same name", "aaa"); + + // missing name + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory(null, Format1.class))) + .withMessageContainingAll("SSTable format name", "cannot be null"); + + // Configuration contains options of unknown sstable formats + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected2, factory("aaa", Format1.class))) + .withMessageContainingAll("Configuration contains options of unknown sstable formats", "bbb"); + + // Selected sstable format '%s' is not available + assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> configure(expected1, factory("bbb", Format1.class))) + .withMessageContainingAll("Selected sstable format", "aaa", "is not available"); + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java deleted file mode 100644 index 167c02ec25..0000000000 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.io.sstable; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import com.google.common.collect.ImmutableMap; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.ParameterizedClass; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.utils.Pair; -import org.mockito.Mockito; - -import static org.apache.cassandra.config.Config.SSTABLE_FORMAT_ID; -import static org.apache.cassandra.config.Config.SSTABLE_FORMAT_NAME; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; - -public class SSTableFormatTypeTest -{ - public static abstract class Format1 extends AbstractSSTableFormat - { - public static Format1 instance = null; - } - - public static abstract class Format2 extends AbstractSSTableFormat - { - public static Format2 instance = null; - } - - public static abstract class NonSSTableFormat - { - public static NonSSTableFormat instance = null; - } - - public abstract static class BrokenSSTableFormat extends AbstractSSTableFormat - { - public BrokenSSTableFormat() - { - // initialization will fail because it is abstract class - } - } - - private Map>> factories; - Pair, SSTableFormat.Type[]> formats; - - @BeforeClass - public static void beforeClass() - { - DatabaseDescriptor.clientInitialization(); - } - - @Before - public void before() - { - Format1.instance = Mockito.spy(Format1.class); - Format2.instance = Mockito.spy(Format2.class); - NonSSTableFormat.instance = Mockito.spy(NonSSTableFormat.class); - } - - @Test - public void testValidConfig() - { - init(format(Format1.class, "0", "aaa"), format(Format2.class, "1", "bbb")); - verifyFormat(Format1.class, 0, "aaa"); - verifyFormat(Format2.class, 1, "bbb"); - assertThat(formats.left.get(0).name).isEqualTo("aaa"); - } - - @Test - public void testDefaultSelectedByOrder() - { - init(format(Format1.class, "1", "aaa"), format(Format2.class, "0", "bbb")); - verifyFormat(Format1.class, 1, "aaa"); - verifyFormat(Format2.class, 0, "bbb"); - assertThat(formats.left.get(0).name).isEqualTo("aaa"); - } - - @Test - public void testNonConsecutiveOrdinals() - { - init(format(Format1.class, "5", "aaa"), format(Format2.class, "10", "bbb")); - verifyFormat(Format1.class, 5, "aaa"); - verifyFormat(Format2.class, 10, "bbb"); - assertThat(formats.left.get(0).name).isEqualTo("aaa"); - } - - @Test - public void testConfigValidation() - { - // invalid name - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "Aa"))) - .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "a-a"))) - .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "a1"))) - .withMessageContainingAll("'name'", "Format1", "must be non-empty, lower-case letters only string"); - - // invalid id - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "-1", "aaa"))) - .withMessageContainingAll("'id'", "Format1", "must be within bounds [0..127] range"); - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "128", "aaa"))) - .withMessageContainingAll("'id'", "Format1", "must be within bounds [0..127] range"); - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "x", "aaa"))) - .withMessageContainingAll("'id'", "Format1", "must be an integer"); - - // duplicate name - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "aaa"), format(Format2.class, "1", "aaa"))) - .withMessageContainingAll("Name 'aaa' of sstable format", "Format2", "is already defined"); - - // duplicate id - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class, "0", "aaa"), format(Format2.class, "0", "bbb"))) - .withMessageContainingAll("ID '0' of sstable format", "Format2", "is already defined"); - - // missing name - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class.getName(), ImmutableMap.of(SSTABLE_FORMAT_ID, "0")))) - .withMessageContainingAll("Missing 'name' parameter", "Format1"); - - // missing id - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> load(format(Format1.class.getName(), ImmutableMap.of(SSTABLE_FORMAT_NAME, "aaa")))) - .withMessageContainingAll("Missing 'id' parameter", "Format1"); - - // not an sstable format class - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format(NonSSTableFormat.class, "0", "aaa"))) - .withMessageContainingAll("NonSSTableFormat", "format aaa", "does not implement org.apache.cassandra.io.sstable.format.SSTableFormat"); - - // missing class - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format("org.apache.cassandra.io.sstable.format.dummy.DummyFormat", ImmutableMap.of(SSTABLE_FORMAT_ID, "0", SSTABLE_FORMAT_NAME, "aaa")))) - .withMessageContainingAll("Unable to find sstable format class", "org.apache.cassandra.io.sstable.format.dummy.DummyFormat"); - - // failed initialization - assertThatExceptionOfType(ConfigurationException.class).isThrownBy(() -> init(format(BrokenSSTableFormat.class, "0", "aaa"))) - .withMessageContainingAll(BrokenSSTableFormat.class.getName(), "sstable format"); - } - - private void verifyFormat(Class expectedClass, int expectedOrdinal, String expectedName) - { - SSTableFormat.Type formatType = formats.right[expectedOrdinal]; - assertThat(formatType).isNotNull(); - assertThat(formatType.name).isEqualTo(expectedName); - assertThat(formatType.ordinal).isEqualTo(expectedOrdinal); - assertThat(formatType.info).isInstanceOf(expectedClass); - assertThat(formats.left.stream().filter(f -> f.name.equals(expectedName)).findFirst().get()).isSameAs(formatType); - } - - private ParameterizedClass format(Class clazz, String id, String name, String... otherParams) - { - ImmutableMap.Builder b = ImmutableMap.builder(); - b.put(SSTABLE_FORMAT_ID, id); - b.put(SSTABLE_FORMAT_NAME, name); - for (int i = 0; i < otherParams.length; i += 2) - b.put(otherParams[i], otherParams[i + 1]); - return format(clazz.getName(), b.build()); - } - - private ParameterizedClass format(String className, Map params) - { - return new ParameterizedClass(className, params); - } - - private Map>> load(ParameterizedClass... formats) - { - return DatabaseDescriptor.loadSSTableFormatFactories(Arrays.asList(formats)); - } - - private void init(ParameterizedClass... formats) - { - this.factories = load(formats); - this.formats = SSTableFormat.Type.readFactories(factories); - } -} diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java index ed28538b4c..8991ed8108 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java @@ -830,7 +830,7 @@ public class SSTableHeaderFixTest try { - Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation), BigFormat.getInstance().getType()); + Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation)); // Just create the component files - we don't really need those. for (Component component : requiredComponents) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index e2b46e7803..47d35a01db 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -67,7 +67,6 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.sstable.format.CompressionInfoComponent; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; import org.apache.cassandra.io.sstable.format.big.BigFormat; @@ -362,7 +361,7 @@ public class SSTableReaderTest @Test public void testGetPositionsKeyCacheStats() { - Assume.assumeTrue(KeyCacheSupport.isSupportedBy(SSTableFormat.Type.current())); + Assume.assumeTrue(KeyCacheSupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat())); ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD2); partitioner = store.getPartitioner(); CacheService.instance.keyCache.setCapacity(1000); @@ -713,7 +712,7 @@ public class SSTableReaderTest @Test public void testIndexSummaryReplacement() throws IOException, ExecutionException, InterruptedException { - assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); + assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat())); ColumnFamilyStore store = discardSSTables(KEYSPACE1, CF_STANDARD_LOW_INDEX_INTERVAL); // index interval of 8, no key caching final int NUM_PARTITIONS = 512; @@ -775,7 +774,7 @@ public class SSTableReaderTest @Test public void testIndexSummaryUpsampleAndReload() throws Exception { - assumeTrue(IndexSummarySupport.isSupportedBy(SSTableFormat.Type.current())); + assumeTrue(IndexSummarySupport.isSupportedBy(DatabaseDescriptor.getSelectedSSTableFormat())); int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE; MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this will generate lots of segments diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index 2ebbe3ffdd..05964a8000 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -93,7 +93,7 @@ public class SSTableUtils File cfDir = new File(tempdir, keyspaceName + File.pathSeparator() + cfname); cfDir.tryCreateDirectories(); cfDir.deleteOnExit(); - File datafile = new Descriptor(cfDir, keyspaceName, cfname, id, BigFormat.getInstance().getType()).fileFor(Components.DATA); + File datafile = new Descriptor(cfDir, keyspaceName, cfname, id, BigFormat.getInstance()).fileFor(Components.DATA); if (!datafile.createFileIfNotExists()) throw new IOException("unable to create file " + datafile); datafile.deleteOnExit(); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java index e65c410ae7..b439b9c980 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java @@ -96,21 +96,21 @@ public class SSTableWriterTransactionTest extends AbstractTransactionalTest protected void assertInProgress() throws Exception { - assertExists(descriptor.formatType.info.primaryComponents()); - assertNotExists(descriptor.formatType.info.generatedOnLoadComponents()); + assertExists(descriptor.version.format.primaryComponents()); + assertNotExists(descriptor.version.format.generatedOnLoadComponents()); Assert.assertTrue(file.length() > 0); } protected void assertPrepared() throws Exception { - assertExists(descriptor.formatType.info.primaryComponents()); - assertExists(descriptor.formatType.info.generatedOnLoadComponents()); + assertExists(descriptor.version.format.primaryComponents()); + assertExists(descriptor.version.format.generatedOnLoadComponents()); } protected void assertAborted() throws Exception { - assertNotExists(descriptor.formatType.info.primaryComponents()); - assertNotExists(descriptor.formatType.info.generatedOnLoadComponents()); + assertNotExists(descriptor.version.format.primaryComponents()); + assertNotExists(descriptor.version.format.generatedOnLoadComponents()); Assert.assertFalse(file.exists()); } diff --git a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java index 307e09c9a3..eb8d8b3cc7 100644 --- a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java @@ -57,7 +57,6 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.io.FSWriteError; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReaderWithFilter; import org.apache.cassandra.io.sstable.format.SortedTableVerifier.RangeOwnHelper; @@ -559,7 +558,7 @@ public class VerifyTest @Test public void testVerifyBf() throws IOException { - Assume.assumeTrue(SSTableReaderWithFilter.class.isAssignableFrom(SSTableFormat.Type.current().info.getReaderFactory().getReaderClass())); + Assume.assumeTrue(SSTableReaderWithFilter.class.isAssignableFrom(DatabaseDescriptor.getSelectedSSTableFormat().getReaderFactory().getReaderClass())); testBrokenComponentHelper(Components.FILTER); } diff --git a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java b/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java deleted file mode 100644 index 8dc7425779..0000000000 --- a/test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.io.sstable.format; - -import java.util.NoSuchElementException; - -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.config.DatabaseDescriptor; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.Assert.assertEquals; - -public class VersionAndTypeTest -{ - @BeforeClass - public static void beforeClass() - { - DatabaseDescriptor.clientInitialization(); - } - - @Test - public void testValidInput() - { - assertEquals(VersionAndType.fromString("big-bc").toString(), "big-bc"); - } - - @Test - public void testInvalidInputs() - { - assertThatThrownBy(() -> VersionAndType.fromString(" ")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("should be of the form 'big-bc'"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc-")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("should be of the form 'big-bc'"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc-d")).isInstanceOf(NoSuchElementException.class) - .hasMessageContaining("mcc"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc-d-")).isInstanceOf(NoSuchElementException.class) - .hasMessageContaining("mcc"); - assertThatThrownBy(() -> VersionAndType.fromString("mcc")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("should be of the form 'big-bc'"); - assertThatThrownBy(() -> VersionAndType.fromString("-")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("should be of the form 'big-bc'"); - assertThatThrownBy(() -> VersionAndType.fromString("--")).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("should be of the form 'big-bc'"); - } -} diff --git a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java index 984e9f2475..40d04a4840 100644 --- a/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java @@ -59,7 +59,6 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.schema.CachingParams; @@ -98,7 +97,7 @@ public class IndexSummaryManagerTest keyCacheSerializer.deserialize(answer.getArgument(0), - answer.getArgument(1)) )) - .when(keyCacheSerializerSpy).deserialize(any(DataInputPlus.class), any(ColumnFamilyStore.class)); + doAnswer(new AnswersWithDelay(delayMillis, InvocationOnMock::callRealMethod)).when(keyCacheSerializerSpy) + .deserialize(any(DataInputPlus.class)); long maxExpectedKeyCache = Math.min(numberOfRows, 1 + TimeUnit.SECONDS.toMillis(DatabaseDescriptor.getCacheLoadTimeout()) / delayMillis); diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 64a1a2ce14..ef94253492 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -70,7 +70,7 @@ public class MetadataSerializerTest public static void initDD() { DatabaseDescriptor.daemonInitialization(); - format = SSTableFormat.Type.current().info; + format = DatabaseDescriptor.getSelectedSSTableFormat(); } @Test @@ -79,9 +79,9 @@ public class MetadataSerializerTest Map originalMetadata = constructMetadata(false); MetadataSerializer serializer = new MetadataSerializer(); - File statsFile = serialize(originalMetadata, serializer, SSTableFormat.Type.current().info.getLatestVersion()); + File statsFile = serialize(originalMetadata, serializer, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()); - Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.current()); + Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); try (RandomAccessReader in = RandomAccessReader.open(statsFile)) { Map deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class)); @@ -108,7 +108,7 @@ public class MetadataSerializerTest // Serialize w/ overflowed histograms: MetadataSerializer serializer = new MetadataSerializer(); File statsFile = serialize(originalMetadata, serializer, format.getLatestVersion()); - Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), format.getType()); + Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), format); try (RandomAccessReader in = RandomAccessReader.open(statsFile)) { @@ -187,7 +187,7 @@ public class MetadataSerializerTest for (char minor = 'a'; minor <= 'z'; minor++){ Version version = format.getVersion(String.format("%s%s", major, minor)); if (version.isCompatible()) - supportedVersions.computeIfAbsent(major, ignored -> new ArrayList<>()).add(version.getVersion()); + supportedVersions.computeIfAbsent(major, ignored -> new ArrayList<>()).add(version.version); } } @@ -204,7 +204,7 @@ public class MetadataSerializerTest File statsFileLb = serialize(originalMetadata, serializer, format.getVersion(newV)); File statsFileLa = serialize(originalMetadata, serializer, format.getVersion(oldV)); // Reading both as earlier version should yield identical results. - Descriptor desc = new Descriptor(format.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0), format.getType()); + Descriptor desc = new Descriptor(format.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0)); try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb); RandomAccessReader inLa = RandomAccessReader.open(statsFileLa)) { @@ -224,7 +224,7 @@ public class MetadataSerializerTest @Test public void pendingRepairCompatibility() { - if (format == BigFormat.instance) + if (BigFormat.is(format)) { Arrays.asList("ma", "mb", "mc", "md", "me").forEach(v -> assertFalse(format.getVersion(v).hasPendingRepair())); Arrays.asList("na", "nb", "nc").forEach(v -> assertTrue(format.getVersion(v).hasPendingRepair())); @@ -238,7 +238,7 @@ public class MetadataSerializerTest @Test public void originatingHostCompatibility() { - if (format == BigFormat.instance) + if (BigFormat.is(format)) { Arrays.asList("ma", "mb", "mc", "md", "na").forEach(v -> assertFalse(format.getVersion(v).hasOriginatingHostId())); Arrays.asList("me", "nb").forEach(v -> assertTrue(format.getVersion(v).hasOriginatingHostId())); @@ -252,10 +252,10 @@ public class MetadataSerializerTest @Test public void improvedMinMaxCompatibility() { - if (format == BigFormat.instance) + if (BigFormat.is(format)) { - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasImprovedMinMax())); - Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.getInstance().getVersion(v).hasImprovedMinMax())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.getInstance().getVersion(v).hasImprovedMinMax())); } else { @@ -266,10 +266,10 @@ public class MetadataSerializerTest @Test public void legacyMinMaxCompatiblity() { - if (format == BigFormat.instance) + if (BigFormat.is(format)) { - Arrays.asList("oa").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasLegacyMinMax())); - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + Arrays.asList("oa").forEach(v -> assertFalse(BigFormat.getInstance().getVersion(v).hasLegacyMinMax())); + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb", "nc").forEach(v -> assertTrue(BigFormat.getInstance().getVersion(v).hasLegacyMinMax())); } else { @@ -280,10 +280,10 @@ public class MetadataSerializerTest @Test public void partitionLevelDeletionPresenceMarkerCompatibility() { - if (format == BigFormat.instance) + if (BigFormat.is(format)) { - Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); - Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.getInstance().getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.getInstance().getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); } else { diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 0da6e1f092..4b986d9dbd 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -163,14 +163,14 @@ public class MockSchema public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime, long timestamp) { - SSTableFormat format = SSTableFormat.Type.current().info; + SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getTableName(), sstableId(generation), - format.getType()); + format); - if (format == BigFormat.getInstance()) + if (BigFormat.is(format)) { Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); for (Component component : components) diff --git a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java index c4fe58a542..6bd5fd280c 100644 --- a/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java +++ b/test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java @@ -33,7 +33,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.VersionAndType; +import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.File; import org.assertj.core.util.Files; import org.quicktheories.core.Gen; @@ -68,15 +68,15 @@ public class SSTablesGlobalTrackerTest { qt().forAll(lists().of(updates()).ofSizeBetween(0, MAX_UPDATES_PER_GEN), sstableFormatTypes()) - .checkAssert((updates, formatType) -> { - SSTablesGlobalTracker tracker = new SSTablesGlobalTracker(formatType); + .checkAssert((updates, format) -> { + SSTablesGlobalTracker tracker = new SSTablesGlobalTracker(format); Set all = new HashSet<>(); - Set previous = Collections.emptySet(); + Set previous = Collections.emptySet(); for (Update update : updates) { update.applyTo(all); boolean triggerUpdate = tracker.handleSSTablesChange(update.removed, update.added); - Set expectedInUse = versionAndTypes(all); + Set expectedInUse = versionAndTypes(all); assertEquals(expectedInUse, tracker.versionsInUse()); assertEquals(!expectedInUse.equals(previous), triggerUpdate); previous = expectedInUse; @@ -84,9 +84,9 @@ public class SSTablesGlobalTrackerTest }); } - private Set versionAndTypes(Set descriptors) + private Set versionAndTypes(Set descriptors) { - return descriptors.stream().map(SSTablesGlobalTracker::version).collect(Collectors.toSet()); + return descriptors.stream().map(d -> d.version).collect(Collectors.toSet()); } private Gen keyspaces() @@ -118,16 +118,16 @@ public class SSTablesGlobalTrackerTest return lists().of(descriptors()).ofSizeBetween(minSize, MAX_VERSION_LIST_SIZE); } - private Gen sstableFormatTypes() + private Gen> sstableFormatTypes() { - return Generate.pick(Iterables.toList(SSTableFormat.Type.values())); + return Generate.pick(Iterables.toList(DatabaseDescriptor.getSSTableFormats().values())); } private Gen sstableVersionString() { // We want to somewhat favor the current version, as that is technically more realistic so we generate it 50% // of the time, and generate something random 50% of the time. - return Generate.constant(SSTableFormat.Type.current().info.getLatestVersion().getVersion()) + return Generate.constant(DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion().version) .mix(strings().betweenCodePoints('a', 'z').ofLength(2)); } diff --git a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java index 8aa9867e5f..cdd05ec3e7 100644 --- a/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java +++ b/tools/stress/src/org/apache/cassandra/io/sstable/StressCQLSSTableWriter.java @@ -21,22 +21,22 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.antlr.runtime.RecognitionException; -import org.apache.cassandra.cql3.CQLStatement; -import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; -import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.schema.Schema; -import org.apache.cassandra.schema.SchemaTransformations; import org.apache.cassandra.cql3.CQLFragmentParser; +import org.apache.cassandra.cql3.CQLStatement; import org.apache.cassandra.cql3.ColumnSpecification; import org.apache.cassandra.cql3.CqlParser; import org.apache.cassandra.cql3.QueryOptions; @@ -44,7 +44,12 @@ import org.apache.cassandra.cql3.UpdateParameters; import org.apache.cassandra.cql3.functions.UDHelper; import org.apache.cassandra.cql3.functions.types.TypeCodec; import org.apache.cassandra.cql3.statements.UpdateStatement; -import org.apache.cassandra.db.*; +import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; +import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -53,6 +58,10 @@ import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.SchemaTransformations; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.schema.Types; import org.apache.cassandra.service.ClientState; @@ -345,7 +354,7 @@ public class StressCQLSSTableWriter implements Closeable private final List directoryList; private ColumnFamilyStore cfs; - protected SSTableFormat.Type formatType = null; + protected SSTableFormat format = null; private Boolean makeRangeAware = false; @@ -581,8 +590,8 @@ public class StressCQLSSTableWriter implements Closeable ? new SSTableSimpleWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns()) : new SSTableSimpleUnsortedWriter(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.metadata, preparedInsert.updatedColumns(), bufferSizeInMiB); - if (formatType != null) - writer.setSSTableFormatType(formatType); + if (format != null) + writer.setSSTableFormatType(format); writer.setRangeAwareWriting(makeRangeAware);