natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement sstable generation identifier as uuid 

Patch by Jacek Lewandowski; reviewed by Andrés de la Peña, Benjamin Lerer and Dan Jatnieks for CASSANDRA-17048
This commit is contained in:
Jacek Lewandowski 2022-01-24 11:51:13 +01:00 committed by Andrés de la Peña
parent 30ad754d7e
commit 0040fea379
70 changed files with 106639 additions and 642 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.build/build-rat.xml
View File

@ -68,6 +68,7 @@
<exclude name="**/test/data/jmxdump/cassandra-3.0-jmx.yaml"/>
<exclude name="**/test/data/jmxdump/cassandra-3.11-jmx.yaml"/>
<exclude name="**/test/data/jmxdump/cassandra-4.0-jmx.yaml"/>
<exclude name="**/test/data/jmxdump/cassandra-4.1-jmx.yaml"/>
<exclude name="**/test/resources/data/config/YamlConfigurationLoaderTest/shared_client_error_reporting_exclusions.yaml"/>
<exclude name="**/tools/cqlstress-counter-example.yaml"/>
<exclude name="**/tools/cqlstress-example.yaml"/>

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.1
* Add support for UUID based sstable generation identifiers (CASSANDRA-17048)
* Log largest memtable flush at info instead of debug (CASSANDRA-17472)
* Add native transport rate limiter options to example cassandra.yaml, and expose metric for dispatch rate (CASSANDRA-17423)
* Add diagnostic events for guardrails (CASSANDRA-17197)

6
NEWS.txt
View File

@ -116,6 +116,12 @@ New features
table.
- Added ability to invalidate auth caches through corresponding `nodetool` commands and virtual tables.
- DCL statements in audit logs will now obscure only the password if they don't fail to parse.
- Starting from 4.1 sstables support UUID based generation identifiers. They are globally unique and thus they let
the node to create sstables without any prior knowledge about the existing sstables in the data directory.
The feature is disabled by default in cassandra.yaml because once enabled, there is no easy way to downgrade.
When the node is restarted with UUID based generation identifiers enabled, each newly created sstable will have
a UUID based generation identifier and such files are not readable by previous Cassandra versions. In the future
those new identifiers will become enabled by default.
Upgrading
---------

10
conf/cassandra.yaml
View File

@ -947,6 +947,13 @@ compaction_throughput: 64MiB/s
# between the sstables, reducing page cache churn and keeping hot rows hot
sstable_preemptive_open_interval: 50MiB
# Starting from 4.1 sstables support UUID based generation identifiers. They are disabled by default
# because once enabled, there is no easy way to downgrade. When the node is restarted with this option
# set to true, each newly created sstable will have a UUID based generation identifier and such files are
# not readable by previous Cassandra versions. At some point, this option will become true by default
# and eventually get removed from the configuration.
enable_uuid_sstable_identifiers: false
# When enabled, permits Cassandra to zero-copy stream entire eligible
# SSTables between nodes, including every component.
# This speeds up the network transfer significantly subject to
@ -1628,7 +1635,7 @@ drop_compact_storage_enabled: false
# removing list elements by either index or value. Defaults to true.
# read_before_write_list_operations_enabled: true
# Guardrail to warn or fail when querying with an IN restriction selecting more partition keys than threshold.
# The two thresholds default to -1 to disable.
# The two thresholds default to -1 to disable.
# partition_keys_in_select_warn_threshold: -1
# partition_keys_in_select_fail_threshold: -1
# Guardrail to warn or fail when an IN query creates a cartesian product with a size exceeding threshold,
@ -1672,3 +1679,4 @@ drop_compact_storage_enabled: false
# enabled: true # (overriden by cassandra.ignore_dc system property)
# rack:
# enabled: true # (overriden by cassandra.ignore_rack system property)

3
src/java/org/apache/cassandra/config/Config.java
View File

@ -748,6 +748,9 @@ public class Config
public volatile boolean auto_optimise_full_repair_streams = false;
public volatile boolean auto_optimise_preview_repair_streams = false;
// see CASSANDRA-17048 and the comment in cassandra.yaml
public boolean enable_uuid_sstable_identifiers = false;
/**
* Client mode means that the process is a pure client, that uses C* code base but does
* not read or write local C* database files.

48
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -18,9 +18,22 @@
package org.apache.cassandra.config;
import java.io.IOException;
import java.net.*;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.nio.file.FileStore;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Supplier;
@ -31,18 +44,12 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.config.Config.PaxosStatePurging;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.audit.AuditLogOptions;
import org.apache.cassandra.fql.FullQueryLoggerOptions;
import org.apache.cassandra.auth.AllowAllInternodeAuthenticator;
import org.apache.cassandra.auth.AuthConfig;
import org.apache.cassandra.auth.IAuthenticator;
@ -51,14 +58,20 @@ import org.apache.cassandra.auth.IInternodeAuthenticator;
import org.apache.cassandra.auth.INetworkAuthorizer;
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.Config.CommitLogSync;
import org.apache.cassandra.config.Config.PaxosOnLinearizabilityViolation;
import org.apache.cassandra.config.Config.PaxosStatePurging;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerCDC;
import org.apache.cassandra.db.commitlog.CommitLogSegmentManagerStandard;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.fql.FullQueryLoggerOptions;
import org.apache.cassandra.gms.IFailureDetector;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.DiskOptimizationStrategy;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.io.util.SpinningDiskOptimizationStrategy;
@ -72,22 +85,14 @@ import org.apache.cassandra.locator.SeedProvider;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.security.SSLFactory;
import org.apache.cassandra.service.CacheService.CacheType;
import org.apache.cassandra.service.FileSystemOwnershipCheck;
import org.apache.cassandra.service.StartupChecks;
import org.apache.cassandra.service.paxos.Paxos;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.OS_ARCH;
import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_ARCH_DATA_MODEL;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL;
import static org.apache.cassandra.io.util.FileUtils.ONE_GIB;
import static org.apache.cassandra.io.util.FileUtils.ONE_MIB;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_JVM_DTEST_DISABLE_SSL;
import static org.apache.cassandra.utils.Clock.Global.logInitializationOutcome;
public class DatabaseDescriptor
@ -4086,4 +4091,9 @@ public class DatabaseDescriptor
conf.streaming_state_size = duration;
}
}
public static boolean isUUIDSSTableIdentifiersEnabled()
{
return conf.enable_uuid_sstable_identifiers;
}
}

160
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -24,53 +24,84 @@ import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import javax.management.*;
import javax.management.openmbean.*;
import java.util.stream.Collectors;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.CompositeDataSupport;
import javax.management.openmbean.CompositeType;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.OpenType;
import javax.management.openmbean.SimpleType;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.*;
import com.google.common.collect.*;
import com.google.common.util.concurrent.*;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.utils.concurrent.Future;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.cache.*;
import org.apache.cassandra.concurrent.*;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cache.CounterCacheKey;
import org.apache.cassandra.cache.IRowCacheEntry;
import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.cache.RowCacheSentinel;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.concurrent.FutureTask;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.*;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.CompactionStrategyManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.compaction.Verifier;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.streaming.CassandraStreamManager;
import org.apache.cassandra.db.repair.CassandraTableRepairManager;
import org.apache.cassandra.db.view.TableViews;
import org.apache.cassandra.db.lifecycle.*;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.Tracker;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.repair.CassandraTableRepairManager;
import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.db.streaming.CassandraStreamManager;
import org.apache.cassandra.db.view.TableViews;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.index.SecondaryIndexManager;
@ -80,9 +111,15 @@ import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.format.*;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.Sampler;
import org.apache.cassandra.metrics.Sampler.Sample;
@ -91,23 +128,38 @@ import org.apache.cassandra.metrics.TableMetrics;
import org.apache.cassandra.repair.TableRepairManager;
import org.apache.cassandra.repair.consistent.admin.CleanupSummary;
import org.apache.cassandra.repair.consistent.admin.PendingStat;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.TablePaxosRepairHistory;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.TableStreamManager;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.TablePaxosRepairHistory;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.utils.DefaultValue;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MBeanWrapper;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.concurrent.Promise;
import org.apache.cassandra.utils.concurrent.Refs;
@ -116,10 +168,8 @@ import org.apache.cassandra.utils.memory.MemtableAllocator;
import static com.google.common.base.Throwables.propagate;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.DatabaseDescriptor.getFlushWriters;
import static org.apache.cassandra.db.commitlog.CommitLog.instance;
import static org.apache.cassandra.db.commitlog.CommitLogPosition.NONE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@ -204,7 +254,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public final OpOrder readOrdering = new OpOrder();
/* This is used to generate the next index for a SSTable */
private final AtomicInteger fileIndexGenerator = new AtomicInteger(0);
private final Supplier<? extends SSTableId> sstableIdGenerator;
public final SecondaryIndexManager indexManager;
public final TableViews viewManager;
@ -406,7 +456,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
@VisibleForTesting
public ColumnFamilyStore(Keyspace keyspace,
String columnFamilyName,
int generation,
Supplier<? extends SSTableId> sstableIdGenerator,
TableMetadataRef metadata,
Directories directories,
boolean loadSSTables,
@ -424,7 +474,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
maxCompactionThreshold = new DefaultValue<>(metadata.get().params.compaction.maxCompactionThreshold());
crcCheckChance = new DefaultValue<>(metadata.get().params.crcCheckChance);
viewManager = keyspace.viewManager.forTable(metadata.id);
fileIndexGenerator.set(generation);
this.sstableIdGenerator = sstableIdGenerator;
sampleReadLatencyNanos = DatabaseDescriptor.getReadRpcTimeout(NANOSECONDS) / 2;
additionalWriteLatencyNanos = DatabaseDescriptor.getWriteRpcTimeout(NANOSECONDS) / 2;
@ -661,21 +711,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
boolean registerBookkeeping,
boolean offline)
{
// get the max generation number, to prevent generation conflicts
Directories.SSTableLister lister = directories.sstableLister(Directories.OnTxnErr.IGNORE).includeBackups(true);
List<Integer> generations = new ArrayList<>();
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
{
Descriptor desc = entry.getKey();
generations.add(desc.generation);
if (!desc.isCompatible())
throw new RuntimeException(String.format("Incompatible SSTable found. Current version %s is unable to read file: %s. Please run upgradesstables.",
desc.getFormat().getLatestVersion(), desc));
}
Collections.sort(generations);
int value = (generations.size() > 0) ? (generations.get(generations.size() - 1)) : 0;
return new ColumnFamilyStore(keyspace, columnFamily, value, metadata, directories, loadSSTables, registerBookkeeping, offline);
return new ColumnFamilyStore(keyspace, columnFamily,
directories.getUIDGenerator(SSTableIdFactory.instance.defaultBuilder()),
metadata, directories, loadSSTables, registerBookkeeping, offline);
}
/**
@ -808,10 +846,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
descriptor.cfname,
// Increment the generation until we find a filename that doesn't exist. This is needed because the new
// SSTables that are being loaded might already use these generation numbers.
fileIndexGenerator.incrementAndGet(),
sstableIdGenerator.get(),
descriptor.formatType);
}
while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
while (newDescriptor.fileFor(Component.DATA).exists());
return newDescriptor;
}
@ -865,12 +903,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Descriptor newSSTableDescriptor(File directory, Version version, SSTableFormat.Type format)
{
return new Descriptor(version,
directory,
keyspace.getName(),
name,
fileIndexGenerator.incrementAndGet(),
format);
Descriptor newDescriptor = new Descriptor(version,
directory,
keyspace.getName(),
name,
sstableIdGenerator.get(),
format);
assert !newDescriptor.fileFor(Component.DATA).exists();
return newDescriptor;
}
/**
@ -2072,9 +2112,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public Refs<SSTableReader> getSnapshotSSTableReaders(String tag) throws IOException
{
Map<Integer, SSTableReader> active = new HashMap<>();
Map<SSTableId, SSTableReader> active = new HashMap<>();
for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
active.put(sstable.descriptor.generation, sstable);
active.put(sstable.descriptor.id, sstable);
Map<Descriptor, Set<Component>> snapshots = getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list();
Refs<SSTableReader> refs = new Refs<>();
try
@ -2083,7 +2123,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
// Try acquire reference to an active sstable instead of snapshot if it exists,
// to avoid opening new sstables. If it fails, use the snapshot reference instead.
SSTableReader sstable = active.get(entries.getKey().generation);
SSTableReader sstable = active.get(entries.getKey().id);
if (sstable == null || !refs.tryRef(sstable))
{
if (logger.isTraceEnabled())

24
src/java/org/apache/cassandra/db/Directories.java
View File

@ -21,6 +21,7 @@ import java.time.Instant;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.BiPredicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
@ -30,9 +31,8 @@ import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@ -1198,6 +1198,24 @@ public class Directories
return result;
}
/**
* Initializes the sstable unique identifier generator using a provided builder for this instance of directories.
* If the id builder needs that, sstables in these directories are listed to provide the existing identifiers to
* the builder. The listing is done lazily so if the builder does not require that, listing is skipped.
*/
public <T extends SSTableId> Supplier<T> getUIDGenerator(SSTableId.Builder<T> builder)
{
// this stream is evaluated lazily - if the generator does not need the existing ids, we do not even call #sstableLister
Stream<SSTableId> curIds = StreamSupport.stream(() -> sstableLister(Directories.OnTxnErr.IGNORE)
.includeBackups(true)
.list()
.keySet()
.spliterator(), Spliterator.DISTINCT, false)
.map(d -> d.id);
return builder.generator(curIds);
}
private static File getOrCreate(File base, String... subdirs)
{
File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));

2
src/java/org/apache/cassandra/db/SerializationHeader.java
View File

@ -105,7 +105,7 @@ public class SerializationHeader
return sstables;
List<SSTableReader> readers = new ArrayList<>(sstables);
readers.sort(SSTableReader.generationReverseComparator);
readers.sort(SSTableReader.idReverseComparator);
return readers;
}

145
src/java/org/apache/cassandra/db/SystemKeyspace.java
View File

@ -23,7 +23,17 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@ -38,55 +48,87 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.cql3.functions.*;
import org.apache.cassandra.cql3.functions.AggregateFcts;
import org.apache.cassandra.cql3.functions.BytesConversionFcts;
import org.apache.cassandra.cql3.functions.CastFcts;
import org.apache.cassandra.cql3.functions.OperationFcts;
import org.apache.cassandra.cql3.functions.TimeFcts;
import org.apache.cassandra.cql3.functions.UuidFcts;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.compaction.CompactionHistoryTabularData;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.*;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.RebufferingInputStream;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.Functions;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.schema.Views;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.service.paxos.*;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.Commit;
import org.apache.cassandra.service.paxos.Commit.Accepted;
import org.apache.cassandra.service.paxos.Commit.AcceptedWithTTL;
import org.apache.cassandra.service.paxos.Commit.Committed;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex;
import org.apache.cassandra.utils.*;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MD5Digest;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Future;
import static java.lang.String.format;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.apache.cassandra.config.Config.PaxosStatePurging.*;
import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternalWithNowInSec;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
import static org.apache.cassandra.service.paxos.Commit.latest;
import static org.apache.cassandra.utils.CassandraVersion.NULL_VERSION;
import static org.apache.cassandra.utils.CassandraVersion.UNREADABLE_VERSION;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeAsUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
public final class SystemKeyspace
@ -97,15 +139,6 @@ public final class SystemKeyspace
private static final Logger logger = LoggerFactory.getLogger(SystemKeyspace.class);
// Used to indicate that there was a previous version written to the legacy (pre 1.2)
// system.Versions table, but that we cannot read it. Suffice to say, any upgrade should
// proceed through 1.2.x before upgrading to the current version.
public static final CassandraVersion UNREADABLE_VERSION = new CassandraVersion("0.0.0-unknown");
// Used to indicate that no previous version information was found. When encountered, we assume that
// Cassandra was not previously installed and we're in the process of starting a fresh node.
public static final CassandraVersion NULL_VERSION = new CassandraVersion("0.0.0-absent");
public static final CassandraVersion CURRENT_VERSION = new CassandraVersion(FBUtilities.getReleaseVersionString());
public static final String BATCHES = "batches";
@ -116,7 +149,7 @@ public final class SystemKeyspace
public static final String PEERS_V2 = "peers_v2";
public static final String PEER_EVENTS_V2 = "peer_events_v2";
public static final String COMPACTION_HISTORY = "compaction_history";
public static final String SSTABLE_ACTIVITY = "sstable_activity";
public static final String SSTABLE_ACTIVITY_V2 = "sstable_activity_v2"; // v2 has modified generation column type (v1 - int, v2 - blob), see CASSANDRA-17048
public static final String TABLE_ESTIMATES = "table_estimates";
public static final String TABLE_ESTIMATES_TYPE_PRIMARY = "primary";
public static final String TABLE_ESTIMATES_TYPE_LOCAL_PRIMARY = "local_primary";
@ -143,6 +176,7 @@ public final class SystemKeyspace
@Deprecated public static final String LEGACY_TRANSFERRED_RANGES = "transferred_ranges";
@Deprecated public static final String LEGACY_AVAILABLE_RANGES = "available_ranges";
@Deprecated public static final String LEGACY_SIZE_ESTIMATES = "size_estimates";
@Deprecated public static final String LEGACY_SSTABLE_ACTIVITY = "sstable_activity";
public static final TableMetadata Batches =
@ -269,8 +303,8 @@ public final class SystemKeyspace
.defaultTimeToLive((int) TimeUnit.DAYS.toSeconds(7))
.build();
private static final TableMetadata SSTableActivity =
parse(SSTABLE_ACTIVITY,
private static final TableMetadata LegacySSTableActivity =
parse(LEGACY_SSTABLE_ACTIVITY,
"historic sstable read rates",
"CREATE TABLE %s ("
+ "keyspace_name text,"
@ -281,6 +315,18 @@ public final class SystemKeyspace
+ "PRIMARY KEY ((keyspace_name, columnfamily_name, generation)))")
.build();
private static final TableMetadata SSTableActivity =
parse(SSTABLE_ACTIVITY_V2,
"historic sstable read rates",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "table_name text,"
+ "id blob,"
+ "rate_120m double,"
+ "rate_15m double,"
+ "PRIMARY KEY ((keyspace_name, table_name, id)))")
.build();
@Deprecated
private static final TableMetadata LegacySizeEstimates =
parse(LEGACY_SIZE_ESTIMATES,
@ -456,6 +502,7 @@ public final class SystemKeyspace
PeerEventsV2,
LegacyPeerEvents,
CompactionHistory,
LegacySSTableActivity,
SSTableActivity,
LegacySizeEstimates,
TableEstimates,
@ -1391,12 +1438,12 @@ public final class SystemKeyspace
* from values in system.sstable_activity if present.
* @param keyspace the keyspace the sstable belongs to
* @param table the table the sstable belongs to
* @param generation the generation number for the sstable
* @param id the generation id for the sstable
*/
public static RestorableMeter getSSTableReadMeter(String keyspace, String table, int generation)
public static RestorableMeter getSSTableReadMeter(String keyspace, String table, SSTableId id)
{
String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and columnfamily_name=? and generation=?";
UntypedResultSet results = executeInternal(format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
String cql = "SELECT * FROM system.%s WHERE keyspace_name=? and table_name=? and id=?";
UntypedResultSet results = executeInternal(format(cql, SSTABLE_ACTIVITY_V2), keyspace, table, id.asBytes());
if (results.isEmpty())
return new RestorableMeter();
@ -1410,25 +1457,45 @@ public final class SystemKeyspace
/**
* Writes the current read rates for a given SSTable to system.sstable_activity
*/
public static void persistSSTableReadMeter(String keyspace, String table, int generation, RestorableMeter meter)
public static void persistSSTableReadMeter(String keyspace, String table, SSTableId id, RestorableMeter meter)
{
// Store values with a one-day TTL to handle corner cases where cleanup might not occur
String cql = "INSERT INTO system.%s (keyspace_name, columnfamily_name, generation, rate_15m, rate_120m) VALUES (?, ?, ?, ?, ?) USING TTL 864000";
executeInternal(format(cql, SSTABLE_ACTIVITY),
String cql = "INSERT INTO system.%s (keyspace_name, table_name, id, rate_15m, rate_120m) VALUES (?, ?, ?, ?, ?) USING TTL 864000";
executeInternal(format(cql, SSTABLE_ACTIVITY_V2),
keyspace,
table,
generation,
id.asBytes(),
meter.fifteenMinuteRate(),
meter.twoHourRate());
if (!DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled() && id instanceof SequenceBasedSSTableId)
{
// we do this in order to make it possible to downgrade until we switch in cassandra.yaml to UUID based ids
// see the discussion on CASSANDRA-17048
cql = "INSERT INTO system.%s (keyspace_name, columnfamily_name, generation, rate_15m, rate_120m) VALUES (?, ?, ?, ?, ?) USING TTL 864000";
executeInternal(format(cql, LEGACY_SSTABLE_ACTIVITY),
keyspace,
table,
((SequenceBasedSSTableId) id).generation,
meter.fifteenMinuteRate(),
meter.twoHourRate());
}
}
/**
* Clears persisted read rates from system.sstable_activity for SSTables that have been deleted.
*/
public static void clearSSTableReadMeter(String keyspace, String table, int generation)
public static void clearSSTableReadMeter(String keyspace, String table, SSTableId id)
{
String cql = "DELETE FROM system.%s WHERE keyspace_name=? AND columnfamily_name=? and generation=?";
executeInternal(format(cql, SSTABLE_ACTIVITY), keyspace, table, generation);
String cql = "DELETE FROM system.%s WHERE keyspace_name=? AND table_name=? and id=?";
executeInternal(format(cql, SSTABLE_ACTIVITY_V2), keyspace, table, id.asBytes());
if (!DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled() && id instanceof SequenceBasedSSTableId)
{
// we do this in order to make it possible to downgrade until we switch in cassandra.yaml to UUID based ids
// see the discussion on CASSANDRA-17048
cql = "DELETE FROM system.%s WHERE keyspace_name=? AND columnfamily_name=? and generation=?";
executeInternal(format(cql, LEGACY_SSTABLE_ACTIVITY), keyspace, table, ((SequenceBasedSSTableId) id).generation);
}
}
/**

230
src/java/org/apache/cassandra/db/SystemKeyspaceMigrator40.java
View File

@ -1,230 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
/**
* Migrate 3.0 versions of some tables to 4.0. In this case it's just extra columns and some keys
* that are changed.
*
* Can't just add the additional columns because they are primary key columns and C* doesn't support changing
* key columns even if it's just clustering columns.
*/
public class SystemKeyspaceMigrator40
{
static final String legacyPeersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEERS);
static final String peersName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEERS_V2);
static final String legacyPeerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEER_EVENTS);
static final String peerEventsName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEER_EVENTS_V2);
static final String legacyTransferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_TRANSFERRED_RANGES);
static final String transferredRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.TRANSFERRED_RANGES_V2);
static final String legacyAvailableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_AVAILABLE_RANGES);
static final String availableRangesName = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.AVAILABLE_RANGES_V2);
private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator40.class);
private SystemKeyspaceMigrator40() {}
public static void migrate()
{
migratePeers();
migratePeerEvents();
migrateTransferredRanges();
migrateAvailableRanges();
}
private static void migratePeers()
{
ColumnFamilyStore newPeers = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEERS_V2);
if (!newPeers.isEmpty())
return;
logger.info("{} table was empty, migrating legacy {}, if this fails you should fix the issue and then truncate {} to have it try again.",
peersName, legacyPeersName, peersName);
String query = String.format("SELECT * FROM %s",
legacyPeersName);
String insert = String.format("INSERT INTO %s ( "
+ "peer, "
+ "peer_port, "
+ "data_center, "
+ "host_id, "
+ "preferred_ip, "
+ "preferred_port, "
+ "rack, "
+ "release_version, "
+ "native_address, "
+ "native_port, "
+ "schema_version, "
+ "tokens) "
+ " values ( ?, ?, ? , ? , ?, ?, ?, ?, ?, ?, ?, ?)",
peersName);
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
int transferred = 0;
logger.info("Migrating rows from legacy {} to {}", legacyPeersName, peersName);
for (UntypedResultSet.Row row : rows)
{
logger.debug("Transferring row {}", transferred);
QueryProcessor.executeInternal(insert,
row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("data_center") ? row.getString("data_center") : null,
row.has("host_id") ? row.getUUID("host_id") : null,
row.has("preferred_ip") ? row.getInetAddress("preferred_ip") : null,
DatabaseDescriptor.getStoragePort(),
row.has("rack") ? row.getString("rack") : null,
row.has("release_version") ? row.getString("release_version") : null,
row.has("rpc_address") ? row.getInetAddress("rpc_address") : null,
DatabaseDescriptor.getNativeTransportPort(),
row.has("schema_version") ? row.getUUID("schema_version") : null,
row.has("tokens") ? row.getSet("tokens", UTF8Type.instance) : null);
transferred++;
}
logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeersName, peersName);
}
private static void migratePeerEvents()
{
ColumnFamilyStore newPeerEvents = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.PEER_EVENTS_V2);
if (!newPeerEvents.isEmpty())
return;
logger.info("{} table was empty, migrating legacy {} to {}", peerEventsName, legacyPeerEventsName, peerEventsName);
String query = String.format("SELECT * FROM %s",
legacyPeerEventsName);
String insert = String.format("INSERT INTO %s ( "
+ "peer, "
+ "peer_port, "
+ "hints_dropped) "
+ " values ( ?, ?, ? )",
peerEventsName);
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
int transferred = 0;
for (UntypedResultSet.Row row : rows)
{
logger.debug("Transferring row {}", transferred);
QueryProcessor.executeInternal(insert,
row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("hints_dropped") ? row.getMap("hints_dropped", TimeUUIDType.instance, Int32Type.instance) : null);
transferred++;
}
logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyPeerEventsName, peerEventsName);
}
static void migrateTransferredRanges()
{
ColumnFamilyStore newTransferredRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.TRANSFERRED_RANGES_V2);
if (!newTransferredRanges.isEmpty())
return;
logger.info("{} table was empty, migrating legacy {} to {}", transferredRangesName, legacyTransferredRangesName, transferredRangesName);
String query = String.format("SELECT * FROM %s",
legacyTransferredRangesName);
String insert = String.format("INSERT INTO %s ("
+ "operation, "
+ "peer, "
+ "peer_port, "
+ "keyspace_name, "
+ "ranges) "
+ " values ( ?, ?, ? , ?, ?)",
transferredRangesName);
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
int transferred = 0;
for (UntypedResultSet.Row row : rows)
{
logger.debug("Transferring row {}", transferred);
QueryProcessor.executeInternal(insert,
row.has("operation") ? row.getString("operation") : null,
row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("keyspace_name") ? row.getString("keyspace_name") : null,
row.has("ranges") ? row.getSet("ranges", BytesType.instance) : null);
transferred++;
}
logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyTransferredRangesName, transferredRangesName);
}
static void migrateAvailableRanges()
{
ColumnFamilyStore newAvailableRanges = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.AVAILABLE_RANGES_V2);
if (!newAvailableRanges.isEmpty())
return;
logger.info("{} table was empty, migrating legacy {} to {}", availableRangesName, legacyAvailableRangesName, availableRangesName);
String query = String.format("SELECT * FROM %s",
legacyAvailableRangesName);
String insert = String.format("INSERT INTO %s ("
+ "keyspace_name, "
+ "full_ranges, "
+ "transient_ranges) "
+ " values ( ?, ?, ? )",
availableRangesName);
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, 1000);
int transferred = 0;
for (UntypedResultSet.Row row : rows)
{
logger.debug("Transferring row {}", transferred);
String keyspace = row.getString("keyspace_name");
Set<ByteBuffer> ranges = Optional.ofNullable(row.getSet("ranges", BytesType.instance)).orElse(Collections.emptySet());
QueryProcessor.executeInternal(insert,
keyspace,
ranges,
Collections.emptySet());
transferred++;
}
logger.info("Migrated {} rows from legacy {} to {}", transferred, legacyAvailableRangesName, availableRangesName);
}
}

205
src/java/org/apache/cassandra/db/SystemKeyspaceMigrator41.java Normal file
View File

@ -0,0 +1,205 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
/**
* Migrate 3.0 versions of some tables to 4.1. In this case it's just extra columns and some keys
* that are changed.
* <p>
* Can't just add the additional columns because they are primary key columns and C* doesn't support changing
* key columns even if it's just clustering columns.
*/
public class SystemKeyspaceMigrator41
{
private static final Logger logger = LoggerFactory.getLogger(SystemKeyspaceMigrator41.class);
private SystemKeyspaceMigrator41()
{
}
public static void migrate()
{
migratePeers();
migratePeerEvents();
migrateTransferredRanges();
migrateAvailableRanges();
migrateSSTableActivity();
}
@VisibleForTesting
static void migratePeers()
{
migrateTable(false,
SystemKeyspace.LEGACY_PEERS,
SystemKeyspace.PEERS_V2,
new String[]{ "peer",
"peer_port",
"data_center",
"host_id",
"preferred_ip",
"preferred_port",
"rack",
"release_version",
"native_address",
"native_port",
"schema_version",
"tokens" },
row -> Collections.singletonList(new Object[]{ row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("data_center") ? row.getString("data_center") : null,
row.has("host_id") ? row.getUUID("host_id") : null,
row.has("preferred_ip") ? row.getInetAddress("preferred_ip") : null,
DatabaseDescriptor.getStoragePort(),
row.has("rack") ? row.getString("rack") : null,
row.has("release_version") ? row.getString("release_version") : null,
row.has("rpc_address") ? row.getInetAddress("rpc_address") : null,
DatabaseDescriptor.getNativeTransportPort(),
row.has("schema_version") ? row.getUUID("schema_version") : null,
row.has("tokens") ? row.getSet("tokens", UTF8Type.instance) : null }));
}
@VisibleForTesting
static void migratePeerEvents()
{
migrateTable(false,
SystemKeyspace.LEGACY_PEER_EVENTS,
SystemKeyspace.PEER_EVENTS_V2,
new String[]{ "peer",
"peer_port",
"hints_dropped" },
row -> Collections.singletonList(
new Object[]{ row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("hints_dropped") ? row.getMap("hints_dropped", TimeUUIDType.instance, Int32Type.instance) : null }
));
}
@VisibleForTesting
static void migrateTransferredRanges()
{
migrateTable(false,
SystemKeyspace.LEGACY_TRANSFERRED_RANGES,
SystemKeyspace.TRANSFERRED_RANGES_V2,
new String[]{ "operation", "peer", "peer_port", "keyspace_name", "ranges" },
row -> Collections.singletonList(new Object[]{ row.has("operation") ? row.getString("operation") : null,
row.has("peer") ? row.getInetAddress("peer") : null,
DatabaseDescriptor.getStoragePort(),
row.has("keyspace_name") ? row.getString("keyspace_name") : null,
row.has("ranges") ? row.getSet("ranges", BytesType.instance) : null }));
}
@VisibleForTesting
static void migrateAvailableRanges()
{
migrateTable(false,
SystemKeyspace.LEGACY_AVAILABLE_RANGES,
SystemKeyspace.AVAILABLE_RANGES_V2,
new String[]{ "keyspace_name", "full_ranges", "transient_ranges" },
row -> Collections.singletonList(new Object[]{ row.getString("keyspace_name"),
Optional.ofNullable(row.getSet("ranges", BytesType.instance)).orElse(Collections.emptySet()),
Collections.emptySet() }));
}
@VisibleForTesting
static void migrateSSTableActivity()
{
String prevVersionString = FBUtilities.getPreviousReleaseVersionString();
CassandraVersion prevVersion = prevVersionString != null ? new CassandraVersion(prevVersionString) : CassandraVersion.NULL_VERSION;
// if we are upgrading from pre 4.1, we want to force repopulate the table; this is for the case when we
// upgraded from pre 4.1, then downgraded to pre 4.1 and then upgraded again
migrateTable(CassandraVersion.CASSANDRA_4_1.compareTo(prevVersion) > 0,
SystemKeyspace.LEGACY_SSTABLE_ACTIVITY,
SystemKeyspace.SSTABLE_ACTIVITY_V2,
new String[]{ "keyspace_name", "table_name", "id", "rate_120m", "rate_15m" },
row ->
Collections.singletonList(new Object[]{ row.getString("keyspace_name"),
row.getString("columnfamily_name"),
new SequenceBasedSSTableId(row.getInt("generation")).asBytes(),
row.has("rate_120m") ? row.getDouble("rate_120m") : null,
row.has("rate_15m") ? row.getDouble("rate_15m") : null
})
);
}
/**
* Perform table migration by reading data from the old table, converting it, and adding to the new table.
*
* @param truncateIfExists truncate the existing table if it exists before migration; if it is disabled
* and the new table is not empty, no migration is performed
* @param oldName old table name
* @param newName new table name
* @param columns columns to fill in the new table in the same order as returned by the transformation
* @param transformation transformation function which gets the row from the old table and returns a row for the new table
*/
@VisibleForTesting
static void migrateTable(boolean truncateIfExists, String oldName, String newName, String[] columns, Function<UntypedResultSet.Row, Collection<Object[]>> transformation)
{
ColumnFamilyStore newTable = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(newName);
if (!newTable.isEmpty() && !truncateIfExists)
return;
if (truncateIfExists)
newTable.truncateBlockingWithoutSnapshot();
logger.info("{} table was empty, migrating legacy {}, if this fails you should fix the issue and then truncate {} to have it try again.",
newName, oldName, newName);
String query = String.format("SELECT * FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, oldName);
String insert = String.format("INSERT INTO %s.%s (%s) VALUES (%s)", SchemaConstants.SYSTEM_KEYSPACE_NAME, newName,
StringUtils.join(columns, ", "), StringUtils.repeat("?", ", ", columns.length));
UntypedResultSet rows = QueryProcessor.executeInternal(query);
int transferred = 0;
logger.info("Migrating rows from legacy {} to {}", oldName, newName);
for (UntypedResultSet.Row row : rows)
{
logger.debug("Transferring row {}", transferred);
for (Object[] newRow : transformation.apply(row))
QueryProcessor.executeInternal(insert, newRow);
transferred++;
}
logger.info("Migrated {} rows from legacy {} to {}", transferred, oldName, newName);
}
}

2
src/java/org/apache/cassandra/db/compaction/CompactionLogger.java
View File

@ -171,7 +171,7 @@ public class CompactionLogger
private JsonNode formatSSTable(AbstractCompactionStrategy strategy, SSTableReader sstable)
{
ObjectNode node = json.objectNode();
node.put("generation", sstable.descriptor.generation);
node.put("generation", sstable.descriptor.id.asString());
node.put("version", sstable.descriptor.version.getVersion());
node.put("size", sstable.onDiskLength());
JsonNode logResult = strategy.strategyLogger().sstable(sstable);

4
src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java
View File

@ -33,12 +33,12 @@ import java.util.concurrent.TimeUnit;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import com.google.common.primitives.Ints;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.FBUtilities;
@ -77,7 +77,7 @@ class LeveledGenerations
private static final Comparator<SSTableReader> nonL0Comparator = (o1, o2) -> {
int cmp = SSTableReader.sstableComparator.compare(o1, o2);
if (cmp == 0)
cmp = Ints.compare(o1.descriptor.generation, o2.descriptor.generation);
cmp = SSTableIdFactory.COMPARATOR.compare(o1.descriptor.id, o2.descriptor.id);
return cmp;
};

2
src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
View File

@ -161,7 +161,7 @@ public class LeveledManifest
{
builder.append(sstable.descriptor.cfname)
.append('-')
.append(sstable.descriptor.generation)
.append(sstable.descriptor.id)
.append("(L")
.append(sstable.getSSTableLevel())
.append("), ");

2
src/java/org/apache/cassandra/db/lifecycle/LogTransaction.java
View File

@ -380,7 +380,7 @@ class LogTransaction extends Transactional.AbstractTransactional implements Tran
// While this may be a dummy tracker w/out information in the metrics table, we attempt to delete regardless
// and allow the delete to silently fail if this is an invalid ks + cf combination at time of tidy run.
if (DatabaseDescriptor.isDaemonInitialized())
SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.id);
synchronized (lock)
{

4
src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriter.java
View File

@ -84,7 +84,7 @@ public class CassandraEntireSSTableStreamWriter
logger.debug("[Stream #{}] Streaming {}.{} gen {} component {} size {}", session.planId(),
sstable.getKeyspaceName(),
sstable.getColumnFamilyName(),
sstable.descriptor.generation,
sstable.descriptor.id,
component,
prettyPrintMemory(length));
@ -99,7 +99,7 @@ public class CassandraEntireSSTableStreamWriter
session.planId(),
sstable.getKeyspaceName(),
sstable.getColumnFamilyName(),
sstable.descriptor.generation,
sstable.descriptor.id,
component,
session.peer,
prettyPrintMemory(bytesWritten),

3
src/java/org/apache/cassandra/gms/Gossiper.java
View File

@ -41,7 +41,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Uninterruptibles;
@ -2359,7 +2358,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
public boolean isUpgradingFromVersionLowerThan(CassandraVersion referenceVersion)
{
CassandraVersion v = upgradeFromVersionMemoized.get();
if (SystemKeyspace.NULL_VERSION.equals(v) && scheduledGossipTask == null)
if (CassandraVersion.NULL_VERSION.equals(v) && scheduledGossipTask == null)
return false;
return v != null && v.compareTo(referenceVersion) < 0;

7
src/java/org/apache/cassandra/index/sasi/SASIIndex.java
View File

@ -72,9 +72,7 @@ public class SASIIndex implements Index, INotificationConsumer
Set<Index> indexes,
Collection<SSTableReader> sstablesToRebuild)
{
NavigableMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables = new TreeMap<>((a, b) -> {
return Integer.compare(a.descriptor.generation, b.descriptor.generation);
});
NavigableMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables = new TreeMap<>(SSTableReader.idComparator);
indexes.stream()
.filter((i) -> i instanceof SASIIndex)
@ -113,8 +111,7 @@ public class SASIIndex implements Index, INotificationConsumer
Tracker tracker = baseCfs.getTracker();
tracker.subscribe(this);
SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> toRebuild = new TreeMap<>((a, b)
-> Integer.compare(a.descriptor.generation, b.descriptor.generation));
SortedMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> toRebuild = new TreeMap<>(SSTableReader.idComparator);
for (SSTableReader sstable : index.init(tracker.getView().liveSSTables()))
{

47
src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
View File

@ -21,10 +21,11 @@ package org.apache.cassandra.io.sstable;
import java.io.IOException;
import java.io.Closeable;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
@ -43,7 +44,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
protected final TableMetadataRef metadata;
protected final RegularAndStaticColumns columns;
protected SSTableFormat.Type formatType = SSTableFormat.Type.current();
protected static AtomicInteger generation = new AtomicInteger(0);
protected static final AtomicReference<SSTableId> id = new AtomicReference<>(SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
protected boolean makeRangeAware = false;
protected AbstractSSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns)
@ -63,8 +64,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
this.makeRangeAware = makeRangeAware;
}
protected SSTableTxnWriter createWriter()
protected SSTableTxnWriter createWriter() throws IOException
{
SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
@ -82,34 +82,29 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
Collections.emptySet());
}
private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt)
private static Descriptor createDescriptor(File directory, final String keyspace, final String columnFamily, final SSTableFormat.Type fmt) throws IOException
{
int maxGen = getNextGeneration(directory, columnFamily);
return new Descriptor(directory, keyspace, columnFamily, maxGen + 1, fmt);
SSTableId nextGen = getNextId(directory, columnFamily);
return new Descriptor(directory, keyspace, columnFamily, nextGen, fmt);
}
private static int getNextGeneration(File directory, final String columnFamily)
private static SSTableId getNextId(File directory, final String columnFamily) throws IOException
{
final Set<Descriptor> existing = new HashSet<>();
directory.tryList(file -> {
Descriptor desc = SSTable.tryDescriptorFromFilename(file);
if (desc == null)
return false;
if (desc.cfname.equals(columnFamily))
existing.add(desc);
return false;
});
int maxGen = generation.getAndIncrement();
for (Descriptor desc : existing)
while (true)
{
while (desc.generation > maxGen)
try (Stream<Path> existingPaths = Files.list(directory.toPath()))
{
maxGen = generation.getAndIncrement();
Stream<SSTableId> existingIds = existingPaths.map(File::new)
.map(SSTable::tryDescriptorFromFilename)
.filter(d -> d != null && d.cfname.equals(columnFamily))
.map(d -> d.id);
SSTableId lastId = id.get();
SSTableId newId = SSTableIdFactory.instance.defaultBuilder().generator(Stream.concat(existingIds, Stream.of(lastId))).get();
if (id.compareAndSet(lastId, newId))
return newId;
}
}
return maxGen;
}
PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException

57
src/java/org/apache/cassandra/io/sstable/Descriptor.java
View File

@ -37,7 +37,7 @@ import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
/**
* A SSTable is described by the keyspace and column family it contains data
* for, a generation (where higher generations contain more recent data) and
* for, an id (generation - where higher generations contain more recent data) and
* an alphabetic version string.
*
* A descriptor can be marked as temporary, which influences generated filenames.
@ -49,7 +49,9 @@ public class Descriptor
public static String TMP_EXT = ".tmp";
private static final Splitter filenameSplitter = Splitter.on('-');
public static final char FILENAME_SEPARATOR = '-';
private static final Splitter filenameSplitter = Splitter.on(FILENAME_SEPARATOR);
/** canonicalized path to the directory where SSTable resides */
public final File directory;
@ -57,7 +59,7 @@ public class Descriptor
public final Version version;
public final String ksname;
public final String cfname;
public final int generation;
public final SSTableId id;
public final SSTableFormat.Type formatType;
private final int hashCode;
@ -65,47 +67,37 @@ public class Descriptor
* A descriptor that assumes CURRENT_VERSION.
*/
@VisibleForTesting
public Descriptor(File directory, String ksname, String cfname, int generation)
public Descriptor(File directory, String ksname, String cfname, SSTableId id)
{
this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current());
this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, id, SSTableFormat.Type.current());
}
/**
* Constructor for sstable writers only.
*/
public Descriptor(File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
public Descriptor(File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
{
this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType);
this(formatType.info.getLatestVersion(), directory, ksname, cfname, id, formatType);
}
@VisibleForTesting
public Descriptor(String version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
public Descriptor(String version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
{
this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType);
this(formatType.info.getVersion(version), directory, ksname, cfname, id, formatType);
}
public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
public Descriptor(Version version, File directory, String ksname, String cfname, SSTableId id, SSTableFormat.Type formatType)
{
assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
this.version = version;
this.directory = directory.toCanonical();
this.ksname = ksname;
this.cfname = cfname;
this.generation = generation;
this.id = id;
this.formatType = formatType;
// directory is unnecessary for hashCode, and for simulator consistency we do not include it
hashCode = Objects.hashCode(version, generation, ksname, cfname, formatType);
}
public Descriptor withGeneration(int newGeneration)
{
return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType);
}
public Descriptor withFormatType(SSTableFormat.Type newType)
{
return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType);
hashCode = Objects.hashCode(version, id, ksname, cfname, formatType);
}
public String tmpFilenameFor(Component component)
@ -128,6 +120,11 @@ public class Descriptor
return baseFilename() + separator + component.name();
}
public File fileFor(Component component)
{
return new File(filenameFor(component));
}
public String baseFilename()
{
StringBuilder buff = new StringBuilder();
@ -139,7 +136,7 @@ public class Descriptor
private void appendFileName(StringBuilder buff)
{
buff.append(version).append(separator);
buff.append(generation);
buff.append(id.asString());
buff.append(separator).append(formatType.name);
}
@ -258,14 +255,14 @@ public class Descriptor
if (!Version.validate(versionString))
throw invalidSSTable(name, "invalid version %s", versionString);
int generation;
SSTableId id;
try
{
generation = Integer.parseInt(tokens.get(1));
id = SSTableIdFactory.instance.fromString(tokens.get(1));
}
catch (NumberFormatException e)
catch (RuntimeException e)
{
throw invalidSSTable(name, "the 'generation' part of the name doesn't parse as a number");
throw invalidSSTable(name, "the 'id' part (%s) of the name doesn't parse as a valid unique identifier", tokens.get(1));
}
String formatString = tokens.get(2);
@ -274,7 +271,7 @@ public class Descriptor
{
format = SSTableFormat.Type.validate(formatString);
}
catch (IllegalArgumentException e)
catch (RuntimeException e)
{
throw invalidSSTable(name, "unknown 'format' part (%s)", formatString);
}
@ -305,7 +302,7 @@ public class Descriptor
String table = tableDir.name().split("-")[0] + indexName;
String keyspace = parentOf(name, tableDir).name();
return Pair.create(new Descriptor(version, directory, keyspace, table, generation, format), component);
return Pair.create(new Descriptor(version, directory, keyspace, table, id, format), component);
}
private static File parentOf(String name, File file)
@ -349,7 +346,7 @@ public class Descriptor
return false;
Descriptor that = (Descriptor)o;
return that.directory.equals(this.directory)
&& that.generation == this.generation
&& that.id.equals(this.id)
&& that.ksname.equals(this.ksname)
&& that.cfname.equals(this.cfname)
&& that.version.equals(this.version)

8
src/java/org/apache/cassandra/io/sstable/SSTable.java
View File

@ -28,9 +28,11 @@ import java.nio.file.NoSuchFileException;
import java.util.*;
import java.util.concurrent.CopyOnWriteArraySet;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
@ -102,6 +104,12 @@ public abstract class SSTable
this.optimizationStrategy = Objects.requireNonNull(optimizationStrategy);
}
@VisibleForTesting
public Set<Component> getComponents()
{
return ImmutableSet.copyOf(components);
}
/**
* We use a ReferenceQueue to manage deleting files that have been compacted
* and for which no more SSTable references exist. But this is not guaranteed

91
src/java/org/apache/cassandra/io/sstable/SSTableId.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.util.function.Supplier;
import java.util.stream.Stream;
import org.apache.cassandra.io.util.File;
/**
* Represents a unique identifier in the sstable descriptor filename.
* This ensures each sstable file uniqueness for the certain table on a single node. However, new implementations
* should ensure the uniqueness across the entire cluster. The legacy implementation which does not satisfy cluster-wide
* uniqueness will be deprecated and eventually removed.
* <p>
* A new implementation must adhere to the following invariants:
* - Must be locally sortable - that is, the comparison must reflect the comparison of generation times of identifiers
* generated on the same node
* - String representation must *not* include the {@link Descriptor#FILENAME_SEPARATOR} character, see {@link Descriptor#fromFilenameWithComponent(File)}
* - must be case-insensitive because the sstables can be stored on case-insensitive file system
* <p>
*/
public interface SSTableId
{
/**
* Creates a byte format of the identifier that can be parsed by
* {@link Builder#fromBytes(ByteBuffer)}
*/
ByteBuffer asBytes();
/**
* Creates a String format of the identifier that can be parsed by
* {@link Builder#fromString(String)}
* <p>
* Must not contain any {@link Descriptor#FILENAME_SEPARATOR} character as it is used in the Descriptor
* see {@link Descriptor#fromFilenameWithComponent(File)}
*/
String asString();
/**
* Builder that can create instances of certain implementation of {@link SSTableId}.
*/
interface Builder<T extends SSTableId>
{
/**
* Creates a new generator of identifiers. Each supplied value must be different to all the previously generated
* values and different to all the provided existing identifiers.
*/
Supplier<T> generator(Stream<SSTableId> existingIdentifiers);
boolean isUniqueIdentifier(String str);
boolean isUniqueIdentifier(ByteBuffer bytes);
/**
* Creates an identifier instance from its string representation
*
* @param str string representation as returned by {@link #asString()}
* @throws IllegalArgumentException when the provided string is not a valid string representation of the identifier
*/
T fromString(String str) throws IllegalArgumentException;
/**
* Creates an identifier instance from its binary representation
* <p>
* The method expects the identifier is encoded in all remaining bytes of the buffer. The method does not move the
* pointer of the buffer.
*
* @param bytes binary representation as returned by {@link #asBytes()}
* @throws IllegalArgumentException when the provided bytes are not a valid binary representation of the identifier
*/
T fromBytes(ByteBuffer bytes) throws IllegalArgumentException;
}
}

97
src/java/org/apache/cassandra/io/sstable/SSTableIdFactory.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.stream.Stream;
import org.apache.cassandra.config.DatabaseDescriptor;
public class SSTableIdFactory
{
public static final SSTableIdFactory instance = new SSTableIdFactory();
/**
* Constructs the instance of {@link SSTableId} from the given string representation.
* It finds the right builder by verifying whether the given string is the representation of the related identifier
* type using {@link SSTableId.Builder#isUniqueIdentifier(String)} method.
*
* @throws IllegalArgumentException when the provided string representation does not represent id of any type
*/
public SSTableId fromString(String str) throws IllegalArgumentException
{
return Stream.of(UUIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance)
.filter(b -> b.isUniqueIdentifier(str))
.findFirst()
.map(b -> b.fromString(str))
.orElseThrow(() -> new IllegalArgumentException("String '" + str + "' does not match any SSTable identifier format"));
}
/**
* Constructs the instance of {@link SSTableId} from the given bytes.
* It finds the right builder by verifying whether the given buffer is the representation of the related identifier
* type using {@link SSTableId.Builder#isUniqueIdentifier(ByteBuffer)} method.
*
* The method expects the identifier is encoded in all remaining bytes of the buffer. The method does not move the
* pointer of the buffer.
*
* @throws IllegalArgumentException when the provided binary representation does not represent id of any type
*/
public SSTableId fromBytes(ByteBuffer bytes)
{
return Stream.of(UUIDBasedSSTableId.Builder.instance, SequenceBasedSSTableId.Builder.instance)
.filter(b -> b.isUniqueIdentifier(bytes))
.findFirst()
.map(b -> b.fromBytes(bytes))
.orElseThrow(() -> new IllegalArgumentException("Byte buffer of length " + bytes.remaining() + " does not match any SSTable identifier format"));
}
/**
* Returns default identifiers builder.
*/
@SuppressWarnings("unchecked")
public SSTableId.Builder<SSTableId> defaultBuilder()
{
SSTableId.Builder<? extends SSTableId> builder = DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled()
? UUIDBasedSSTableId.Builder.instance
: SequenceBasedSSTableId.Builder.instance;
return (SSTableId.Builder<SSTableId>) builder;
}
/**
* Compare sstable identifiers so that UUID based identifier is always greater than sequence based identifier
*/
public final static Comparator<SSTableId> COMPARATOR = Comparator.nullsFirst((id1, id2) -> {
if (id1 instanceof UUIDBasedSSTableId)
{
UUIDBasedSSTableId uuidId1 = (UUIDBasedSSTableId) id1;
return (id2 instanceof UUIDBasedSSTableId) ? uuidId1.compareTo((UUIDBasedSSTableId) id2) : 1;
}
else if (id1 instanceof SequenceBasedSSTableId)
{
SequenceBasedSSTableId seqId1 = (SequenceBasedSSTableId) id1;
return (id2 instanceof SequenceBasedSSTableId) ? seqId1.compareTo((SequenceBasedSSTableId) id2) : -1;
}
else
{
throw new AssertionError("Unsupported comparison between " + id1.getClass().getName() + " and " + id2.getClass().getName());
}
});
}

2
src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
View File

@ -114,7 +114,7 @@ public class SSTableLoader implements StreamEventHandler
Descriptor newDesc = new Descriptor(desc.directory,
desc.ksname,
Directories.BACKUPS_SUBDIR,
desc.generation,
desc.id,
desc.formatType);
metadata = client.getTableMetadata(newDesc.cfname);
if (metadata != null)

2
src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
View File

@ -48,7 +48,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
super(directory, metadata, columns);
}
private SSTableTxnWriter getOrCreateWriter()
private SSTableTxnWriter getOrCreateWriter() throws IOException
{
if (writer == null)
writer = createWriter();

140
src/java/org/apache/cassandra/io/sstable/SequenceBasedSSTableId.java Normal file
View File

@ -0,0 +1,140 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.google.common.base.Preconditions;
/**
* Generation identifier based on sequence of integers.
* This has been the standard implementation in C* since inception.
*/
public class SequenceBasedSSTableId implements SSTableId, Comparable<SequenceBasedSSTableId>
{
public final int generation;
public SequenceBasedSSTableId(final int generation)
{
assert generation >= 0;
this.generation = generation;
}
@Override
public int compareTo(SequenceBasedSSTableId o)
{
if (o == null)
return 1;
else if (o == this)
return 0;
return Integer.compare(this.generation, o.generation);
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass())
return false;
SequenceBasedSSTableId that = (SequenceBasedSSTableId) o;
return generation == that.generation;
}
@Override
public int hashCode()
{
return Objects.hash(generation);
}
@Override
public ByteBuffer asBytes()
{
ByteBuffer bytes = ByteBuffer.allocate(Integer.BYTES);
bytes.putInt(0, generation);
return bytes;
}
@Override
public String asString()
{
return String.valueOf(generation);
}
@Override
public String toString()
{
return asString();
}
public static class Builder implements SSTableId.Builder<SequenceBasedSSTableId>
{
public final static Builder instance = new Builder();
private final static Pattern PATTERN = Pattern.compile("\\d+");
/**
* Generates a sequential number to represent an sstables identifier. The first generated identifier will be
* greater by one than the largest generation number found across the provided existing identifiers.
*/
@Override
public Supplier<SequenceBasedSSTableId> generator(Stream<SSTableId> existingIdentifiers)
{
int value = existingIdentifiers.filter(SequenceBasedSSTableId.class::isInstance)
.map(SequenceBasedSSTableId.class::cast)
.mapToInt(id -> id.generation)
.max()
.orElse(0);
AtomicInteger fileIndexGenerator = new AtomicInteger(value);
return () -> new SequenceBasedSSTableId(fileIndexGenerator.incrementAndGet());
}
@Override
public boolean isUniqueIdentifier(String str)
{
return str != null && !str.isEmpty() && str.length() <= 10 && PATTERN.matcher(str).matches();
}
@Override
public boolean isUniqueIdentifier(ByteBuffer bytes)
{
return bytes != null && bytes.remaining() == Integer.BYTES && bytes.getInt(0) >= 0;
}
@Override
public SequenceBasedSSTableId fromString(String token) throws IllegalArgumentException
{
return new SequenceBasedSSTableId(Integer.parseInt(token));
}
@Override
public SequenceBasedSSTableId fromBytes(ByteBuffer bytes) throws IllegalArgumentException
{
Preconditions.checkArgument(bytes.remaining() == Integer.BYTES, "Buffer does not have a valid number of bytes remaining. Expecting: %s but was: %s", Integer.BYTES, bytes.remaining());
return new SequenceBasedSSTableId(bytes.getInt(0));
}
}
}

164
src/java/org/apache/cassandra/io/sstable/UUIDBasedSSTableId.java Normal file
View File

@ -0,0 +1,164 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import com.google.common.base.Preconditions;
import org.apache.cassandra.utils.TimeUUID;
/**
* SSTable generation identifiers that can be stored across nodes in one directory/bucket
* <p>
* Uses the UUID v1 identifiers
*/
public final class UUIDBasedSSTableId implements SSTableId, Comparable<UUIDBasedSSTableId>
{
public final static int STRING_LEN = 28;
public final static int BYTES_LEN = 16;
private final TimeUUID uuid;
public UUIDBasedSSTableId(TimeUUID uuid)
{
this.uuid = uuid;
}
@Override
public ByteBuffer asBytes()
{
return ByteBuffer.allocate(16)
.order(ByteOrder.BIG_ENDIAN)
.putLong(0, uuid.uuidTimestamp())
.putLong(Long.BYTES, uuid.lsb());
}
@Override
public String asString()
{
long ts = uuid.uuidTimestamp();
long nanoPart = ts % 10_000_000;
ts = ts / 10_000_000;
long seconds = ts % 86_400;
ts = ts / 86_400;
return String.format("%4s_%4s_%5s%13s",
Long.toString(ts, 36),
Long.toString(seconds, 36),
Long.toString(nanoPart, 36),
Long.toUnsignedString(uuid.lsb(), 36)).replace(' ', '0');
}
@Override
public String toString()
{
return uuid.toString();
}
@Override
public int compareTo(UUIDBasedSSTableId o)
{
if (o == null)
return 1;
else if (o == this)
return 0;
return uuid.compareTo(o.uuid);
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass())
return false;
UUIDBasedSSTableId that = (UUIDBasedSSTableId) o;
return uuid.equals(that.uuid);
}
@Override
public int hashCode()
{
return Objects.hash(uuid);
}
public static class Builder implements SSTableId.Builder<UUIDBasedSSTableId>
{
public static final Builder instance = new Builder();
private final static Pattern PATTERN = Pattern.compile("([0-9a-z]{4})_([0-9a-z]{4})_([0-9a-z]{5})([0-9a-z]{13})", Pattern.CASE_INSENSITIVE);
/**
* Creates a new UUID based identifiers generator.
*
* @param existingIdentifiers not used by UUID based generator
*/
@Override
public Supplier<UUIDBasedSSTableId> generator(Stream<SSTableId> existingIdentifiers)
{
return () -> new UUIDBasedSSTableId(TimeUUID.Generator.nextTimeUUID());
}
@Override
public boolean isUniqueIdentifier(String str)
{
return str != null && str.length() == STRING_LEN && PATTERN.matcher(str).matches();
}
@Override
public boolean isUniqueIdentifier(ByteBuffer bytes)
{
return bytes != null && bytes.remaining() == BYTES_LEN;
}
@Override
public UUIDBasedSSTableId fromString(@Nonnull String s) throws IllegalArgumentException
{
Matcher m = PATTERN.matcher(s);
if (!m.matches())
throw new IllegalArgumentException("String '" + s + "' is not a valid UUID based sstable identifier");
long dayPart = Long.parseLong(m.group(1), 36);
long secondPart = Long.parseLong(m.group(2), 36);
long nanoPart = Long.parseLong(m.group(3), 36);
long ts = (dayPart * 86_400 + secondPart) * 10_000_000 + nanoPart;
long randomPart = Long.parseUnsignedLong(m.group(4), 36);
TimeUUID uuid = new TimeUUID(ts, randomPart);
return new UUIDBasedSSTableId(uuid);
}
@Override
public UUIDBasedSSTableId fromBytes(@Nonnull ByteBuffer bytes) throws IllegalArgumentException
{
Preconditions.checkArgument(bytes.remaining() == UUIDBasedSSTableId.BYTES_LEN, "Buffer does not have a valid number of bytes remaining. Expecting: %s but was: %s", UUIDBasedSSTableId.BYTES_LEN, bytes.remaining());
bytes = bytes.order() == ByteOrder.BIG_ENDIAN ? bytes : bytes.duplicate().order(ByteOrder.BIG_ENDIAN);
TimeUUID uuid = new TimeUUID(bytes.getLong(0), bytes.getLong(Long.BYTES));
return new UUIDBasedSSTableId(uuid);
}
}
}

9
src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
View File

@ -169,7 +169,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
public static final Comparator<SSTableReader> sstableComparator = (o1, o2) -> o1.first.compareTo(o2.first);
public static final Comparator<SSTableReader> generationReverseComparator = (o1, o2) -> -Integer.compare(o1.descriptor.generation, o2.descriptor.generation);
public static final Comparator<SSTableReader> idComparator = Comparator.comparing(t -> t.descriptor.id, SSTableIdFactory.COMPARATOR);
public static final Comparator<SSTableReader> idReverseComparator = idComparator.reversed();
public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
@ -2101,7 +2102,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
@Override
public String toString()
{
return "Tidy " + descriptor.ksname + '.' + descriptor.cfname + '-' + descriptor.generation;
return "Tidy " + descriptor.ksname + '.' + descriptor.cfname + '-' + descriptor.id;
}
});
}
@ -2163,7 +2164,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
return;
}
readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
readMeter = SystemKeyspace.getSSTableReadMeter(desc.ksname, desc.cfname, desc.id);
// sync the average read rate to system.sstable_activity every five minutes, starting one minute from now
readMeterSyncFuture = new WeakReference<>(syncExecutor.scheduleAtFixedRate(new Runnable()
{
@ -2172,7 +2173,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
if (obsoletion == null)
{
meterSyncThrottle.acquire();
SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.generation, readMeter);
SystemKeyspace.persistSSTableReadMeter(desc.ksname, desc.cfname, desc.id, readMeter);
}
}
}, 1, 5, TimeUnit.MINUTES));

12
src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
View File

@ -152,7 +152,7 @@ public class BigTableReader extends SSTableReader
if (!bf.isPresent((DecoratedKey)key))
{
listener.onSSTableSkipped(this, SkippingReason.BLOOM_FILTER);
Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.generation);
Tracing.trace("Bloom filter allows skipping sstable {}", descriptor.id);
bloomFilterTracker.addTrueNegative();
return null;
}
@ -168,7 +168,7 @@ public class BigTableReader extends SSTableReader
// we do not need to track "true positive" for Bloom Filter here because it has been already tracked
// inside getCachedPosition method
listener.onSSTableSelected(this, cachedPosition, SelectionReason.KEY_CACHE_HIT);
Tracing.trace("Key cache hit for sstable {}", descriptor.generation);
Tracing.trace("Key cache hit for sstable {}", descriptor.id);
return cachedPosition;
}
}
@ -197,7 +197,7 @@ public class BigTableReader extends SSTableReader
if (op == Operator.EQ && updateCacheAndStats)
bloomFilterTracker.addFalsePositive();
listener.onSSTableSkipped(this, SkippingReason.MIN_MAX_KEYS);
Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.generation);
Tracing.trace("Check against min and max keys allows skipping sstable {}", descriptor.id);
return null;
}
@ -244,7 +244,7 @@ public class BigTableReader extends SSTableReader
if (op == SSTableReader.Operator.EQ && updateCacheAndStats)
bloomFilterTracker.addFalsePositive();
listener.onSSTableSkipped(this, SkippingReason.PARTITION_INDEX_LOOKUP);
Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.generation);
Tracing.trace("Partition index lookup allows skipping sstable {}", descriptor.id);
return null;
}
}
@ -275,7 +275,7 @@ public class BigTableReader extends SSTableReader
if (op == Operator.EQ && updateCacheAndStats)
bloomFilterTracker.addTruePositive();
listener.onSSTableSelected(this, indexEntry, SelectionReason.INDEX_ENTRY_FOUND);
Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndexCount(), descriptor.generation);
Tracing.trace("Partition index with {} entries found for sstable {}", indexEntry.columnsIndexCount(), descriptor.id);
return indexEntry;
}
@ -291,7 +291,7 @@ public class BigTableReader extends SSTableReader
if (op == SSTableReader.Operator.EQ && updateCacheAndStats)
bloomFilterTracker.addFalsePositive();
listener.onSSTableSkipped(this, SkippingReason.INDEX_ENTRY_NOT_FOUND);
Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.generation);
Tracing.trace("Partition index lookup complete (bloom filter false positive) for sstable {}", descriptor.id);
return null;
}

30
src/java/org/apache/cassandra/service/CacheService.java
View File

@ -19,11 +19,7 @@ package org.apache.cassandra.service;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@ -42,6 +38,9 @@ import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.CachedBTreePartition;
import org.apache.cassandra.db.partitions.CachedPartition;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -418,7 +417,7 @@ public class CacheService implements CacheServiceMBean
// For column families with many SSTables the linear nature of getSSTables slowed down KeyCache loading
// by orders of magnitude. So we cache the sstables once and rely on cleanupAfterDeserialize to cleanup any
// cached state we may have accumulated during the load.
Map<Pair<String, String>, Map<Integer, SSTableReader>> cachedSSTableReaders = new ConcurrentHashMap<>();
Map<Pair<String, String>, Map<SSTableId, SSTableReader>> cachedSSTableReaders = new ConcurrentHashMap<>();
public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
@ -430,7 +429,15 @@ public class CacheService implements CacheServiceMBean
tableMetadata.id.serialize(out);
out.writeUTF(tableMetadata.indexName().orElse(""));
ByteArrayUtil.writeWithLength(key.key, out);
out.writeInt(key.desc.generation);
if (key.desc.id instanceof SequenceBasedSSTableId)
{
out.writeInt(((SequenceBasedSSTableId) key.desc.id).generation);
}
else
{
out.writeInt(Integer.MIN_VALUE); // backwards compatibility for "int based generation only"
ByteBufferUtil.writeWithShortLength(key.desc.id.asBytes(), out);
}
out.writeBoolean(true);
SerializationHeader header = new SerializationHeader(false, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
@ -451,23 +458,26 @@ public class CacheService implements CacheServiceMBean
}
ByteBuffer key = ByteBufferUtil.read(input, keyLength);
int generation = input.readInt();
SSTableId generationId = generation == Integer.MIN_VALUE
? SSTableIdFactory.instance.fromBytes(ByteBufferUtil.readWithShortLength(input))
: new SequenceBasedSSTableId(generation); // Backwards compatibility for "int based generation sstables"
input.readBoolean(); // backwards compatibility for "promoted indexes" boolean
SSTableReader reader = null;
if (!skipEntry)
{
Pair<String, String> qualifiedName = Pair.create(cfs.metadata.keyspace, cfs.metadata.name);
Map<Integer, SSTableReader> generationToSSTableReader = cachedSSTableReaders.get(qualifiedName);
Map<SSTableId, SSTableReader> generationToSSTableReader = cachedSSTableReaders.get(qualifiedName);
if (generationToSSTableReader == null)
{
generationToSSTableReader = new HashMap<>(cfs.getLiveSSTables().size());
for (SSTableReader ssTableReader : cfs.getSSTables(SSTableSet.CANONICAL))
{
generationToSSTableReader.put(ssTableReader.descriptor.generation, ssTableReader);
generationToSSTableReader.put(ssTableReader.descriptor.id, ssTableReader);
}
cachedSSTableReaders.putIfAbsent(qualifiedName, generationToSSTableReader);
}
reader = generationToSSTableReader.get(generation);
reader = generationToSSTableReader.get(generationId);
}
if (skipEntry || reader == null)

7
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -36,8 +36,8 @@ import javax.management.StandardMBean;
import javax.management.remote.JMXConnectorServer;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.io.util.File;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,7 +59,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SizeEstimatesRecorder;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.SystemKeyspaceMigrator40;
import org.apache.cassandra.db.SystemKeyspaceMigrator41;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.virtual.SystemViewsKeyspace;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
@ -68,6 +68,7 @@ import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.io.sstable.SSTableHeaderFix;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
@ -270,7 +271,7 @@ public class CassandraDaemon
Thread.setDefaultUncaughtExceptionHandler(JVMStabilityInspector::uncaughtException);
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migrate();
// Populate token metadata before flushing, for token-aware sstable partitioning (#6696)
StorageService.instance.populateTokenMetadata();

58
src/java/org/apache/cassandra/service/StartupChecks.java
View File

@ -21,9 +21,23 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.nio.file.*;
import java.nio.file.FileStore;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
@ -32,34 +46,32 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.apache.commons.lang3.StringUtils;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.StartupChecksOptions;
import org.apache.cassandra.io.util.File;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.jpountz.lz4.LZ4Factory;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.config.StartupChecksOptions;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JavaUtils;
import org.apache.cassandra.utils.NativeLibrary;
import org.apache.cassandra.utils.SigarLibrary;
import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
@ -514,6 +526,7 @@ public class StartupChecks
return;
final Set<String> invalid = new HashSet<>();
final Set<String> nonSSTablePaths = new HashSet<>();
final List<String> withIllegalGenId = new ArrayList<>();
nonSSTablePaths.add(FileUtils.getCanonicalPath(DatabaseDescriptor.getCommitLogLocation()));
nonSSTablePaths.add(FileUtils.getCanonicalPath(DatabaseDescriptor.getSavedCachesLocation()));
nonSSTablePaths.add(FileUtils.getCanonicalPath(DatabaseDescriptor.getHintsDirectory()));
@ -528,8 +541,12 @@ public class StartupChecks
try
{
if (!Descriptor.fromFilename(file).isCompatible())
Descriptor desc = Descriptor.fromFilename(file);
if (!desc.isCompatible())
invalid.add(file.toString());
if (!DatabaseDescriptor.isUUIDSSTableIdentifiersEnabled() && desc.id instanceof UUIDBasedSSTableId)
withIllegalGenId.add(file.toString());
}
catch (Exception e)
{
@ -569,6 +586,15 @@ public class StartupChecks
"upgradesstables",
Joiner.on(",").join(invalid)));
if (!withIllegalGenId.isEmpty())
throw new StartupException(StartupException.ERR_WRONG_CONFIG,
"UUID sstable identifiers are disabled but some sstables have been " +
"created with UUID identifiers. You have to either delete those " +
"sstables or enable UUID based sstable identifers in cassandra.yaml " +
"(enable_uuid_sstable_identifiers). The list of affected sstables is: " +
Joiner.on(", ").join(withIllegalGenId) + ". If you decide to delete sstables, " +
"and have that data replicated over other healthy nodes, those will be brought" +
"back during repair");
}
};

14
src/java/org/apache/cassandra/utils/CassandraVersion.java
View File

@ -50,10 +50,24 @@ public class CassandraVersion implements Comparable<CassandraVersion>
private static final Pattern PATTERN = Pattern.compile(VERSION_REGEXP);
public static final CassandraVersion CASSANDRA_4_1 = new CassandraVersion("4.1").familyLowerBound.get();
public static final CassandraVersion CASSANDRA_4_0 = new CassandraVersion("4.0").familyLowerBound.get();
public static final CassandraVersion CASSANDRA_4_0_RC2 = new CassandraVersion(4, 0, 0, NO_HOTFIX, new String[] {"rc2"}, null);
public static final CassandraVersion CASSANDRA_3_4 = new CassandraVersion("3.4").familyLowerBound.get();
/**
* Used to indicate that there was a previous version written to the legacy (pre 1.2)
* system.Versions table, but that we cannot read it. Suffice to say, any upgrade should
* proceed through 1.2.x before upgrading to the current version.
*/
public static final CassandraVersion UNREADABLE_VERSION = new CassandraVersion("0.0.0-unknown");
/**
* Used to indicate that no previous version information was found. When encountered, we assume that
* Cassandra was not previously installed and we're in the process of starting a fresh node.
*/
public static final CassandraVersion NULL_VERSION = new CassandraVersion("0.0.0-absent");
public final int major;
public final int minor;
public final int patch;

104302
test/data/jmxdump/cassandra-4.1-jmx.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

4
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -63,7 +63,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Memtable;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.SystemKeyspaceMigrator40;
import org.apache.cassandra.db.SystemKeyspaceMigrator41;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionLogger;
import org.apache.cassandra.db.compaction.CompactionManager;
@ -566,7 +566,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
// We need to persist this as soon as possible after startup checks.
// This should be the first write to SystemKeyspace (CASSANDRA-11742)
SystemKeyspace.persistLocalMetadata();
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migrate();
// Same order to populate tokenMetadata for the first time,
// see org.apache.cassandra.service.CassandraDaemon.setup

24
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.distributed.shared;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.security.Permission;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -52,6 +53,7 @@ import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.impl.AbstractCluster;
import org.apache.cassandra.distributed.impl.InstanceConfig;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tools.SystemExitException;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Isolated;
@ -858,4 +860,26 @@ public class ClusterUtils
return Arrays.asList(address, rack, status, state, token).toString();
}
}
public static void preventSystemExit()
{
System.setSecurityManager(new SecurityManager()
{
@Override
public void checkExit(int status)
{
throw new SystemExitException(status);
}
@Override
public void checkPermission(Permission perm)
{
}
@Override
public void checkPermission(Permission perm, Object context)
{
}
});
}
}

17
test/distributed/org/apache/cassandra/distributed/test/FailingTruncationTest.java
View File

@ -19,6 +19,7 @@
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Test;
@ -35,13 +36,17 @@ import static org.junit.Assert.fail;
public class FailingTruncationTest extends TestBaseImpl
{
private static final String BB_FAIL_HELPER_PROP = "test.bbfailhelper.enabled";
@Test
public void testFailingTruncation() throws IOException
{
try(Cluster cluster = init(Cluster.build(2)
.withInstanceInitializer(BBFailHelper::install)
.start()))
try (Cluster cluster = init(Cluster.build(2)
.withInstanceInitializer(BBFailHelper::install)
.start()))
{
System.setProperty(BB_FAIL_HELPER_PROP, "true");
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int)");
try
{
@ -53,11 +58,11 @@ public class FailingTruncationTest extends TestBaseImpl
assertTrue(e.getMessage().contains("Truncate failed on replica /127.0.0.2"));
}
}
}
public static class BBFailHelper
{
static void install(ClassLoader cl, int nodeNumber)
{
if (nodeNumber == 2)
@ -72,8 +77,8 @@ public class FailingTruncationTest extends TestBaseImpl
public static void truncateBlocking()
{
throw new RuntimeException();
if (Boolean.getBoolean(BB_FAIL_HELPER_PROP))
throw new RuntimeException();
}
}
}

521
test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java Normal file
View File

@ -0,0 +1,521 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.FileUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.compaction.DateTieredCompactionStrategy;
import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.tools.SystemExitException;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
import org.assertj.core.data.Offset;
import static java.lang.String.format;
import static org.apache.cassandra.Util.bulkLoadSSTables;
import static org.apache.cassandra.Util.getBackups;
import static org.apache.cassandra.Util.getSSTables;
import static org.apache.cassandra.Util.getSnapshots;
import static org.apache.cassandra.Util.relativizePath;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.db.SystemKeyspace.LEGACY_SSTABLE_ACTIVITY;
import static org.apache.cassandra.db.SystemKeyspace.SSTABLE_ACTIVITY_V2;
import static org.apache.cassandra.distributed.shared.FutureUtils.waitOn;
import static org.apache.cassandra.distributed.test.ExecUtil.rethrow;
import static org.assertj.core.api.Assertions.assertThat;
public class SSTableIdGenerationTest extends TestBaseImpl
{
private final static String ENABLE_UUID_FIELD_NAME = "enable_uuid_sstable_identifiers";
private final static String SNAPSHOT_TAG = "test";
private int v;
@BeforeClass
public static void beforeClass() throws Throwable
{
TestBaseImpl.beforeClass();
// we prevent system exit and convert it to exception becuase this is one of the expected test outcomes,
// and we want to make an assertion on that
ClusterUtils.preventSystemExit();
}
/**
* This test verifies that a node with uuid disabled actually creates sstables with sequential ids and
* both the current and legacy sstable activity tables are updated.
* Then, when enable uuid, we actually create sstables with uuid but keep and can read the old sstables. Also, only
* update the current sstable activity table.
*/
@Test
public void testRestartWithUUIDEnabled() throws IOException
{
try (Cluster cluster = init(Cluster.build(1)
.withDataDirCount(1)
.withConfig(config -> config.set(ENABLE_UUID_FIELD_NAME, false))
.start()))
{
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl", null));
createSSTables(cluster.get(1), KEYSPACE, "tbl", 1, 2);
assertSSTablesCount(cluster.get(1), 2, 0, KEYSPACE, "tbl");
verfiySSTableActivity(cluster, true);
restartNode(cluster, 1, true);
createSSTables(cluster.get(1), KEYSPACE, "tbl", 3, 4);
assertSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, "tbl");
verfiySSTableActivity(cluster, false);
checkRowsNumber(cluster.get(1), KEYSPACE, "tbl", 9);
}
}
/**
* This test verifies that we should not be able to start a node with uuid disabled when there are uuid sstables
*/
@Test
public void testRestartWithUUIDDisabled() throws IOException
{
try (Cluster cluster = init(Cluster.build(1)
.withDataDirCount(1)
.withConfig(config -> config.set(ENABLE_UUID_FIELD_NAME, true))
.start()))
{
cluster.disableAutoCompaction(KEYSPACE);
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl", null));
createSSTables(cluster.get(1), KEYSPACE, "tbl", 1, 2);
assertSSTablesCount(cluster.get(1), 0, 2, KEYSPACE, "tbl");
verfiySSTableActivity(cluster, false);
Assertions.assertThatExceptionOfType(RuntimeException.class)
.isThrownBy(() -> restartNode(cluster, 1, false))
.withCauseInstanceOf(SystemExitException.class);
}
}
@Test
public final void testCompactionStrategiesWithMixedSSTables() throws Exception
{
testCompactionStrategiesWithMixedSSTables(SizeTieredCompactionStrategy.class,
DateTieredCompactionStrategy.class,
TimeWindowCompactionStrategy.class,
LeveledCompactionStrategy.class);
}
/**
* The purpose of this test is to verify that we can compact using the given strategy the mix of sstables created
* with sequential id and with uuid. Then we verify whether the number results matches the number of rows which we
* would get by merging data from the initial sstables.
*/
@SafeVarargs
private final void testCompactionStrategiesWithMixedSSTables(final Class<? extends AbstractCompactionStrategy>... compactionStrategyClasses) throws Exception
{
try (Cluster cluster = init(Cluster.build(1)
.withDataDirCount(1)
.withConfig(config -> config.set(ENABLE_UUID_FIELD_NAME, false))
.start()))
{
// create a table and two sstables with sequential id for each strategy, the sstables will contain overlapping partitions
for (Class<? extends AbstractCompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
{
String tableName = "tbl_" + compactionStrategyClass.getSimpleName().toLowerCase();
cluster.schemaChange(createTableStmt(KEYSPACE, tableName, compactionStrategyClass));
createSSTables(cluster.get(1), KEYSPACE, tableName, 1, 2);
assertSSTablesCount(cluster.get(1), 2, 0, KEYSPACE, tableName);
}
// restart the node with uuid enabled
restartNode(cluster, 1, true);
// create another two sstables with uuid for each previously created table
for (Class<? extends AbstractCompactionStrategy> compactionStrategyClass : compactionStrategyClasses)
{
String tableName = "tbl_" + compactionStrategyClass.getSimpleName().toLowerCase();
createSSTables(cluster.get(1), KEYSPACE, tableName, 3, 4);
// expect to have a mix of sstables with sequential id and uuid
assertSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, tableName);
// after compaction, we expect to have a single sstable with uuid
cluster.get(1).forceCompact(KEYSPACE, tableName);
assertSSTablesCount(cluster.get(1), 0, 1, KEYSPACE, tableName);
// verify the number of rows
checkRowsNumber(cluster.get(1), KEYSPACE, tableName, 9);
}
}
}
@Test
public void testStreamingToNodeWithUUIDEnabled() throws Exception
{
testStreaming(true);
}
@Test
public void testStreamingToNodeWithUUIDDisabled() throws Exception
{
testStreaming(false);
}
/**
* The purpose of this test case is to verify the scenario when we need to stream mixed UUID and seq sstables to
* a node which have: 1) UUID disabled, and 2) UUID enabled; then verify that we can read all the data properly
* from that node alone.
*/
private void testStreaming(boolean uuidEnabledOnTargetNode) throws Exception
{
// start both nodes with uuid disabled
try (Cluster cluster = init(Cluster.build(2)
.withDataDirCount(1)
.withConfig(config -> config.set(ENABLE_UUID_FIELD_NAME, false).with(Feature.NETWORK))
.start()))
{
// create an empty table and shutdown nodes 2, 3
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl", null));
waitOn(cluster.get(2).shutdown());
// create 2 sstables with overlapping partitions on node 1 (with seq ids)
createSSTables(cluster.get(1), KEYSPACE, "tbl", 1, 2);
// restart node 1 with uuid enabled
restartNode(cluster, 1, true);
// create 2 sstables with overlapping partitions on node 1 (with UUID ids)
createSSTables(cluster.get(1), KEYSPACE, "tbl", 3, 4);
assertSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, "tbl");
// now start node with UUID disabled and perform repair
cluster.get(2).config().set(ENABLE_UUID_FIELD_NAME, uuidEnabledOnTargetNode);
cluster.get(2).startup();
assertSSTablesCount(cluster.get(2), 0, 0, KEYSPACE, "tbl");
// at this point we have sstables with seq and uuid on nodes and no sstables on node
// when we run repair, we expect streaming all 5 sstables from node 1 to node 2
cluster.get(2).nodetool("repair", KEYSPACE);
if (uuidEnabledOnTargetNode)
assertSSTablesCount(cluster.get(2), 0, 5, KEYSPACE, "tbl");
else
assertSSTablesCount(cluster.get(2), 5, 0, KEYSPACE, "tbl");
waitOn(cluster.get(1).shutdown());
checkRowsNumber(cluster.get(2), KEYSPACE, "tbl", 9);
}
}
@Test
public void testSnapshot() throws Exception
{
File tmpDir = new File(Files.createTempDirectory("test"));
Set<String> seqOnlyBackupDirs;
Set<String> seqAndUUIDBackupDirs;
Set<String> uuidOnlyBackupDirs;
try (Cluster cluster = init(Cluster.build(1)
.withDataDirCount(1)
.withConfig(config -> config.with(Feature.NETWORK)
.set("incremental_backups", true)
.set("snapshot_before_compaction", false)
.set("auto_snapshot", false)
.set(ENABLE_UUID_FIELD_NAME, false))
.start()))
{
// create the tables
cluster.schemaChange("CREATE KEYSPACE new_ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};");
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_seq_only", null));
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_seq_and_uuid", null));
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_uuid_only", null));
cluster.schemaChange(createTableStmt("new_ks", "tbl_seq_only", null));
cluster.schemaChange(createTableStmt("new_ks", "tbl_seq_and_uuid", null));
cluster.schemaChange(createTableStmt("new_ks", "tbl_uuid_only", null));
// creating sstables
createSSTables(cluster.get(1), KEYSPACE, "tbl_seq_only", 1, 2, 3, 4);
createSSTables(cluster.get(1), KEYSPACE, "tbl_seq_and_uuid", 1, 2);
createSSTables(cluster.get(1), "new_ks", "tbl_seq_only", 5, 6, 7, 8);
createSSTables(cluster.get(1), "new_ks", "tbl_seq_and_uuid", 5, 6);
restartNode(cluster, 1, true);
createSSTables(cluster.get(1), KEYSPACE, "tbl_seq_and_uuid", 3, 4);
createSSTables(cluster.get(1), KEYSPACE, "tbl_uuid_only", 1, 2, 3, 4);
createSSTables(cluster.get(1), "new_ks", "tbl_seq_and_uuid", 7, 8);
createSSTables(cluster.get(1), "new_ks", "tbl_uuid_only", 5, 6, 7, 8);
Set<String> seqOnlySnapshotDirs = snapshot(cluster.get(1), KEYSPACE, "tbl_seq_only");
Set<String> seqAndUUIDSnapshotDirs = snapshot(cluster.get(1), KEYSPACE, "tbl_seq_and_uuid");
Set<String> uuidOnlySnapshotDirs = snapshot(cluster.get(1), KEYSPACE, "tbl_uuid_only");
seqOnlyBackupDirs = getBackupDirs(cluster.get(1), KEYSPACE, "tbl_seq_only");
seqAndUUIDBackupDirs = getBackupDirs(cluster.get(1), KEYSPACE, "tbl_seq_and_uuid");
uuidOnlyBackupDirs = getBackupDirs(cluster.get(1), KEYSPACE, "tbl_uuid_only");
// at this point, we should have sstables with backups and snapshots for all tables
assertSSTablesCount(cluster.get(1), 4, 1, KEYSPACE, "tbl_seq_only");
assertSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, "tbl_seq_and_uuid");
assertSSTablesCount(cluster.get(1), 0, 4, KEYSPACE, "tbl_uuid_only");
assertBackupSSTablesCount(cluster.get(1), 4, 1, KEYSPACE, "tbl_seq_only");
assertBackupSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, "tbl_seq_and_uuid");
assertBackupSSTablesCount(cluster.get(1), 0, 4, KEYSPACE, "tbl_uuid_only");
assertSnapshotSSTablesCount(cluster.get(1), 4, 1, KEYSPACE, "tbl_seq_only");
assertSnapshotSSTablesCount(cluster.get(1), 2, 3, KEYSPACE, "tbl_seq_and_uuid");
assertSnapshotSSTablesCount(cluster.get(1), 0, 4, KEYSPACE, "tbl_uuid_only");
checkRowsNumber(cluster.get(1), KEYSPACE, "tbl_seq_only", 9);
checkRowsNumber(cluster.get(1), KEYSPACE, "tbl_seq_and_uuid", 9);
checkRowsNumber(cluster.get(1), KEYSPACE, "tbl_uuid_only", 9);
// truncate the first set of tables
truncateAndAssertEmpty(cluster.get(1), KEYSPACE, "tbl_seq_only", "tbl_seq_and_uuid", "tbl_uuid_only");
restore(cluster.get(1), seqOnlySnapshotDirs, "tbl_seq_only", 9);
restore(cluster.get(1), seqAndUUIDSnapshotDirs, "tbl_seq_and_uuid", 9);
restore(cluster.get(1), uuidOnlySnapshotDirs, "tbl_uuid_only", 9);
truncateAndAssertEmpty(cluster.get(1), KEYSPACE, "tbl_seq_only", "tbl_seq_and_uuid", "tbl_uuid_only");
restore(cluster.get(1), seqOnlyBackupDirs, "tbl_seq_only", 9);
restore(cluster.get(1), seqAndUUIDBackupDirs, "tbl_seq_and_uuid", 9);
restore(cluster.get(1), uuidOnlyBackupDirs, "tbl_uuid_only", 9);
ImmutableSet<String> allBackupDirs = ImmutableSet.<String>builder().addAll(seqOnlyBackupDirs).addAll(seqAndUUIDBackupDirs).addAll(uuidOnlyBackupDirs).build();
cluster.get(1).runOnInstance(rethrow(() -> allBackupDirs.forEach(dir -> bulkLoadSSTables(new File(dir), "new_ks"))));
checkRowsNumber(cluster.get(1), "new_ks", "tbl_seq_only", 17);
checkRowsNumber(cluster.get(1), "new_ks", "tbl_seq_and_uuid", 17);
checkRowsNumber(cluster.get(1), "new_ks", "tbl_uuid_only", 17);
for (String dir : allBackupDirs)
{
File src = new File(dir);
File dest = relativizePath(tmpDir, src, 3);
Files.createDirectories(dest.parent().toPath());
FileUtils.moveDirectory(src.toJavaIOFile(), dest.toJavaIOFile());
}
}
try (Cluster cluster = init(Cluster.build(1)
.withDataDirCount(1)
.withConfig(config -> config.with(Feature.NETWORK, Feature.NATIVE_PROTOCOL)
.set("incremental_backups", true)
.set("snapshot_before_compaction", false)
.set("auto_snapshot", false)
.set(ENABLE_UUID_FIELD_NAME, false))
.start()))
{
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_seq_only", null));
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_seq_and_uuid", null));
cluster.schemaChange(createTableStmt(KEYSPACE, "tbl_uuid_only", null));
Function<String, String> relativeToTmpDir = d -> relativizePath(tmpDir, new File(d), 3).toString();
restore(cluster.get(1), seqOnlyBackupDirs.stream().map(relativeToTmpDir).collect(Collectors.toSet()), "tbl_seq_only", 9);
restore(cluster.get(1), seqAndUUIDBackupDirs.stream().map(relativeToTmpDir).collect(Collectors.toSet()), "tbl_seq_and_uuid", 9);
restore(cluster.get(1), uuidOnlyBackupDirs.stream().map(relativeToTmpDir).collect(Collectors.toSet()), "tbl_uuid_only", 9);
}
}
private static void restore(IInvokableInstance instance, Set<String> dirs, String targetTableName, int expectedRowsNum)
{
List<String> failedImports = instance.callOnInstance(() -> ColumnFamilyStore.getIfExists(KEYSPACE, targetTableName)
.importNewSSTables(dirs, false, false, true, true, true, true, true));
assertThat(failedImports).isEmpty();
checkRowsNumber(instance, KEYSPACE, targetTableName, expectedRowsNum);
}
private static void truncateAndAssertEmpty(IInvokableInstance instance, String ks, String... tableNames)
{
for (String tableName : tableNames)
{
instance.executeInternal(format("TRUNCATE %s.%s", ks, tableName));
assertSSTablesCount(instance, 0, 0, ks, tableName);
checkRowsNumber(instance, ks, tableName, 0);
}
}
private static Set<String> snapshot(IInvokableInstance instance, String ks, String tableName)
{
Set<String> snapshotDirs = instance.callOnInstance(() -> ColumnFamilyStore.getIfExists(ks, tableName)
.snapshot(SNAPSHOT_TAG)
.getDirectories()
.stream()
.map(File::toString)
.collect(Collectors.toSet()));
assertThat(snapshotDirs).isNotEmpty();
return snapshotDirs;
}
private static String createTableStmt(String ks, String name, Class<? extends AbstractCompactionStrategy> compactionStrategy)
{
if (compactionStrategy == null)
compactionStrategy = SizeTieredCompactionStrategy.class;
return format("CREATE TABLE %s.%s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) " +
"WITH compaction = {'class':'%s', 'enabled':'false'}",
ks, name, compactionStrategy.getCanonicalName());
}
private void createSSTables(IInstance instance, String ks, String tableName, int... records)
{
String insert = format("INSERT INTO %s.%s (pk, ck, v) VALUES (?, ?, ?)", ks, tableName);
for (int record : records)
{
instance.executeInternal(insert, record, record, ++v);
instance.executeInternal(insert, record, record + 1, ++v);
instance.executeInternal(insert, record + 1, record + 1, ++v);
instance.flush(ks);
}
}
private static void assertSSTablesCount(Set<Descriptor> descs, String tableName, int expectedSeqGenIds, int expectedUUIDGenIds)
{
List<String> seqSSTables = descs.stream().filter(desc -> desc.id instanceof SequenceBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList());
List<String> uuidSSTables = descs.stream().filter(desc -> desc.id instanceof UUIDBasedSSTableId).map(Descriptor::baseFilename).sorted().collect(Collectors.toList());
assertThat(seqSSTables).describedAs("SSTables of %s with sequence based id", tableName).hasSize(expectedSeqGenIds);
assertThat(uuidSSTables).describedAs("SSTables of %s with UUID based id", tableName).hasSize(expectedUUIDGenIds);
}
private static void assertSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames)
{
instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getSSTables(ks, tableName), tableName, expectedSeqGenIds, expectedUUIDGenIds))));
}
private static void assertSnapshotSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames)
{
instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getSnapshots(ks, tableName, SNAPSHOT_TAG), tableName, expectedSeqGenIds, expectedUUIDGenIds))));
}
private static void assertBackupSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames)
{
instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getBackups(ks, tableName), tableName, expectedSeqGenIds, expectedUUIDGenIds))));
}
private static Set<String> getBackupDirs(IInvokableInstance instance, String ks, String tableName)
{
return instance.callOnInstance(() -> getBackups(ks, tableName).stream()
.map(d -> d.directory)
.map(File::toString)
.collect(Collectors.toSet()));
}
private static void verfiySSTableActivity(Cluster cluster, boolean expectLegacyTableIsPopulated)
{
cluster.get(1).runOnInstance(() -> {
RestorableMeter meter = new RestorableMeter(15, 120);
SequenceBasedSSTableId seqGenId = new SequenceBasedSSTableId(1);
SystemKeyspace.persistSSTableReadMeter("ks", "tab", seqGenId, meter);
assertThat(SystemKeyspace.getSSTableReadMeter("ks", "tab", seqGenId)).matches(m -> m.fifteenMinuteRate() == meter.fifteenMinuteRate()
&& m.twoHourRate() == meter.twoHourRate());
checkSSTableActivityRow(SSTABLE_ACTIVITY_V2, seqGenId.asBytes(), true);
if (expectLegacyTableIsPopulated)
checkSSTableActivityRow(LEGACY_SSTABLE_ACTIVITY, seqGenId.generation, true);
SystemKeyspace.clearSSTableReadMeter("ks", "tab", seqGenId);
checkSSTableActivityRow(SSTABLE_ACTIVITY_V2, seqGenId.asBytes(), false);
if (expectLegacyTableIsPopulated)
checkSSTableActivityRow(LEGACY_SSTABLE_ACTIVITY, seqGenId.generation, false);
UUIDBasedSSTableId uuidGenId = new UUIDBasedSSTableId(TimeUUID.Generator.nextTimeUUID());
SystemKeyspace.persistSSTableReadMeter("ks", "tab", uuidGenId, meter);
assertThat(SystemKeyspace.getSSTableReadMeter("ks", "tab", uuidGenId)).matches(m -> m.fifteenMinuteRate() == meter.fifteenMinuteRate()
&& m.twoHourRate() == meter.twoHourRate());
checkSSTableActivityRow(SSTABLE_ACTIVITY_V2, uuidGenId.asBytes(), true);
SystemKeyspace.clearSSTableReadMeter("ks", "tab", uuidGenId);
checkSSTableActivityRow(SSTABLE_ACTIVITY_V2, uuidGenId.asBytes(), false);
});
}
private static void checkSSTableActivityRow(String table, Object genId, boolean expectExists)
{
String tableColName = SSTABLE_ACTIVITY_V2.equals(table) ? "table_name" : "columnfamily_name";
String idColName = SSTABLE_ACTIVITY_V2.equals(table) ? "id" : "generation";
String cql = "SELECT rate_15m, rate_120m FROM system.%s WHERE keyspace_name=? and %s=? and %s=?";
UntypedResultSet results = executeInternal(format(cql, table, tableColName, idColName), "ks", "tab", genId);
assertThat(results).isNotNull();
if (expectExists)
{
assertThat(results.isEmpty()).isFalse();
UntypedResultSet.Row row = results.one();
assertThat(row.getDouble("rate_15m")).isEqualTo(15d, Offset.offset(0.001d));
assertThat(row.getDouble("rate_120m")).isEqualTo(120d, Offset.offset(0.001d));
}
else
{
assertThat(results.isEmpty()).isTrue();
}
}
private static void restartNode(Cluster cluster, int node, boolean uuidEnabled)
{
waitOn(cluster.get(node).shutdown());
cluster.get(node).config().set(ENABLE_UUID_FIELD_NAME, uuidEnabled);
cluster.get(node).startup();
}
private static void checkRowsNumber(IInstance instance, String ks, String tableName, int expectedNumber)
{
SimpleQueryResult result = instance.executeInternalWithResult(format("SELECT * FROM %s.%s", ks, tableName));
Object[][] rows = result.toObjectArrays();
assertThat(rows).withFailMessage("Invalid results for %s.%s - should have %d rows but has %d: \n%s", ks, tableName, expectedNumber,
rows.length, result.toString()).hasSize(expectedNumber);
}
}

180
test/unit/org/apache/cassandra/Util.java
View File

@ -26,50 +26,80 @@ import java.io.IOException;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.time.Duration;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.AbstractReadCommandBuilder;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Directories.DataDirectory;
import org.apache.cassandra.db.DisallowedDirectories;
import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.PartitionRangeReadCommand;
import org.apache.cassandra.db.ReadCommand;
import org.apache.cassandra.db.ReadExecutionController;
import org.apache.cassandra.db.compaction.AbstractCompactionTask;
import org.apache.cassandra.db.compaction.ActiveCompactionsTracker;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.CompactionTasks;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaCollection;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.Directories.DataDirectory;
import org.apache.cassandra.db.compaction.AbstractCompactionTask;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.partitions.ImmutableBTreePartition;
import org.apache.cassandra.db.partitions.Partition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Cells;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
@ -77,24 +107,41 @@ import org.apache.cassandra.gms.ApplicationState;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableLoader;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.ReplicaCollection;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.pager.PagingState;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.CounterId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.FilterFactory;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Throwables;
import org.awaitility.Awaitility;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
public class Util
{
@ -792,7 +839,9 @@ public class Util
{
LifecycleTransaction.waitForDeletions();
assertEquals(expectedSSTableCount, cfs.getLiveSSTables().size());
Set<Integer> liveGenerations = cfs.getLiveSSTables().stream().map(sstable -> sstable.descriptor.generation).collect(Collectors.toSet());
Set<SSTableId> liveIdentifiers = cfs.getLiveSSTables().stream()
.map(sstable -> sstable.descriptor.id)
.collect(Collectors.toSet());
int fileCount = 0;
for (File f : cfs.getDirectories().getCFDirectories())
{
@ -801,7 +850,7 @@ public class Util
if (sst.name().contains("Data"))
{
Descriptor d = Descriptor.fromFilename(sst.absolutePath());
assertTrue(liveGenerations.contains(d.generation));
assertTrue(liveIdentifiers.contains(d.id));
fileCount++;
}
}
@ -877,4 +926,85 @@ public class Util
}
}
}
public static Supplier<SequenceBasedSSTableId> newSeqGen(int ... existing)
{
return SequenceBasedSSTableId.Builder.instance.generator(IntStream.of(existing).mapToObj(SequenceBasedSSTableId::new));
}
public static Supplier<UUIDBasedSSTableId> newUUIDGen()
{
return UUIDBasedSSTableId.Builder.instance.generator(Stream.empty());
}
public static Set<Descriptor> getSSTables(String ks, String tableName)
{
return Keyspace.open(ks)
.getColumnFamilyStore(tableName)
.getLiveSSTables()
.stream()
.map(sstr -> sstr.descriptor)
.collect(Collectors.toSet());
}
public static Set<Descriptor> getSnapshots(String ks, String tableName, String snapshotTag)
{
try
{
return Keyspace.open(ks)
.getColumnFamilyStore(tableName)
.getSnapshotSSTableReaders(snapshotTag)
.stream()
.map(sstr -> sstr.descriptor)
.collect(Collectors.toSet());
}
catch (IOException e)
{
throw Throwables.unchecked(e);
}
}
public static Set<Descriptor> getBackups(String ks, String tableName)
{
return Keyspace.open(ks)
.getColumnFamilyStore(tableName)
.getDirectories()
.sstableLister(Directories.OnTxnErr.THROW)
.onlyBackups(true)
.list()
.keySet();
}
public static StreamState bulkLoadSSTables(File dir, String targetKeyspace)
{
SSTableLoader.Client client = new SSTableLoader.Client()
{
private String keyspace;
public void init(String keyspace)
{
this.keyspace = keyspace;
for (Replica replica : StorageService.instance.getLocalReplicas(keyspace))
addRangeForEndpoint(replica.range(), FBUtilities.getBroadcastAddressAndPort());
}
public TableMetadataRef getTableMetadata(String tableName)
{
return Schema.instance.getTableMetadataRef(keyspace, tableName);
}
};
SSTableLoader loader = new SSTableLoader(dir, client, new OutputHandler.LogOutput(), 1, targetKeyspace);
StreamResultFuture result = loader.stream();
return FBUtilities.waitOnFuture(result);
}
public static File relativizePath(File targetBasePath, File path, int components)
{
Preconditions.checkArgument(components > 0);
Preconditions.checkArgument(path.toPath().getNameCount() >= components);
Path relative = path.toPath().subpath(path.toPath().getNameCount() - components, path.toPath().getNameCount());
return new File(targetBasePath.toPath().resolve(relative));
}
}

7
test/unit/org/apache/cassandra/cql3/CQLTester.java
View File

@ -612,7 +612,12 @@ public abstract class CQLTester
String currentTable = currentTable();
return currentTable == null
? null
: Keyspace.open(keyspace).getColumnFamilyStore(currentTable);
: getColumnFamilyStore(keyspace, currentTable);
}
public ColumnFamilyStore getColumnFamilyStore(String keyspace, String table)
{
return Keyspace.open(keyspace).getColumnFamilyStore(table);
}
public void flush(boolean forceFlush)

16
test/unit/org/apache/cassandra/cql3/GcCompactionTest.java
View File

@ -34,6 +34,8 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.CompactionParams.TombstoneOption;
@ -173,7 +175,7 @@ public class GcCompactionTest extends CQLTester
flush();
assertEquals(1, cfs.getLiveSSTables().size());
SSTableReader table = cfs.getLiveSSTables().iterator().next();
int gen = table.descriptor.generation;
SSTableId gen = table.descriptor.id;
assertEquals(KEY_COUNT * CLUSTERING_COUNT, countRows(table));
assertEquals(0, table.getSSTableLevel()); // flush writes to L0
@ -184,8 +186,8 @@ public class GcCompactionTest extends CQLTester
assertEquals(1, cfs.getLiveSSTables().size());
SSTableReader collected = cfs.getLiveSSTables().iterator().next();
int collectedGen = collected.descriptor.generation;
assertTrue(collectedGen > gen);
SSTableId collectedGen = collected.descriptor.id;
assertTrue(SSTableIdFactory.COMPARATOR.compare(collectedGen, gen) > 0);
assertEquals(KEY_COUNT * CLUSTERING_COUNT, countRows(collected));
assertEquals(0, collected.getSSTableLevel()); // garbagecollect should leave the LCS level where it was
@ -194,8 +196,8 @@ public class GcCompactionTest extends CQLTester
assertEquals(1, cfs.getLiveSSTables().size());
SSTableReader compacted = cfs.getLiveSSTables().iterator().next();
int compactedGen = compacted.descriptor.generation;
assertTrue(compactedGen > collectedGen);
SSTableId compactedGen = compacted.descriptor.id;
assertTrue(SSTableIdFactory.COMPARATOR.compare(compactedGen, collectedGen) > 0);
assertEquals(KEY_COUNT * CLUSTERING_COUNT, countRows(compacted));
assertEquals(1, compacted.getSSTableLevel()); // full compaction with LCS should move to L1
@ -205,7 +207,7 @@ public class GcCompactionTest extends CQLTester
assertEquals(1, cfs.getLiveSSTables().size());
SSTableReader collected2 = cfs.getLiveSSTables().iterator().next();
assertTrue(collected2.descriptor.generation > compactedGen);
assertTrue(SSTableIdFactory.COMPARATOR.compare(collected2.descriptor.id, compactedGen) > 0);
assertEquals(KEY_COUNT * CLUSTERING_COUNT, countRows(collected2));
assertEquals(1, collected2.getSSTableLevel()); // garbagecollect should leave the LCS level where it was
@ -264,7 +266,7 @@ public class GcCompactionTest extends CQLTester
assertEquals(CompactionManager.AllSSTableOpStatus.SUCCESSFUL, status);
SSTableReader[] tables = cfs.getLiveSSTables().toArray(new SSTableReader[0]);
Arrays.sort(tables, (o1, o2) -> Integer.compare(o1.descriptor.generation, o2.descriptor.generation)); // by order of compaction
Arrays.sort(tables, SSTableReader.idComparator); // by order of compaction
// Make sure deleted data was removed
assertTrue(rowCount0 > countRows(tables[0]));

3
test/unit/org/apache/cassandra/cql3/ViewComplexDeletionsTest.java
View File

@ -19,7 +19,6 @@
package org.apache.cassandra.cql3;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@ -114,7 +113,7 @@ public class ViewComplexDeletionsTest extends ViewAbstractParameterizedTest
ColumnFamilyStore cfs = ks.getColumnFamilyStore(currentView());
List<String> sstables = cfs.getLiveSSTables()
.stream()
.sorted(Comparator.comparingInt(s -> s.descriptor.generation))
.sorted(SSTableReader.idComparator)
.map(SSTableReader::getFilename)
.collect(Collectors.toList());
String dataFiles = String.join(",", Arrays.asList(sstables.get(1), sstables.get(3), sstables.get(4)));

1
test/unit/org/apache/cassandra/cql3/ViewComplexTest.java
View File

@ -27,7 +27,6 @@ import java.util.List;
import java.util.Map;
import com.google.common.base.Objects;
import org.junit.Test;
import org.apache.cassandra.db.ColumnFamilyStore;

3
test/unit/org/apache/cassandra/cql3/ViewComplexTombstoneTest.java
View File

@ -28,6 +28,7 @@ import org.junit.Test;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.FBUtilities;
@ -101,7 +102,7 @@ public class ViewComplexTombstoneTest extends ViewAbstractParameterizedTest
ColumnFamilyStore cfs = ks.getColumnFamilyStore(currentView());
List<String> sstables = cfs.getLiveSSTables()
.stream()
.sorted(Comparator.comparingInt(s -> s.descriptor.generation))
.sorted(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR))
.map(SSTableReader::getFilename)
.collect(Collectors.toList());
String dataFiles = String.join(",", Arrays.asList(sstables.get(1), sstables.get(2)));

57
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
View File

@ -22,34 +22,42 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.collect.Iterators;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import com.google.common.collect.Iterators;
import org.apache.cassandra.*;
import com.googlecode.concurrenttrees.common.Iterables;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.ClearableHistogram;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
@ -57,8 +65,8 @@ import org.apache.cassandra.utils.WrappedRunnable;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class ColumnFamilyStoreTest
{
@ -301,12 +309,15 @@ public class ColumnFamilyStoreTest
new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key2")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
cfs.forceBlockingFlush();
for (int version = 1; version <= 2; ++version)
for (SSTableReader liveSSTable : cfs.getLiveSSTables())
{
Descriptor existing = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), KEYSPACE2, CF_STANDARD1, version,
SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(Directories.getBackupsDirectory(existing), KEYSPACE2, CF_STANDARD1, version, SSTableFormat.Type.BIG);
for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.STATS })
Descriptor existing = liveSSTable.descriptor;
Descriptor desc = new Descriptor(Directories.getBackupsDirectory(existing),
KEYSPACE2,
CF_STANDARD1,
liveSSTable.descriptor.id,
liveSSTable.descriptor.formatType);
for (Component c : liveSSTable.getComponents())
assertTrue("Cannot find backed-up file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists());
}
}
@ -493,7 +504,13 @@ public class ColumnFamilyStoreTest
String indexTableFile = manifest.getFiles().get(1);
assertThat(baseTableFile).isNotEqualTo(indexTableFile);
assertThat(Directories.isSecondaryIndexFolder(new File(indexTableFile).parent())).isTrue();
assertThat(indexTableFile).endsWith(baseTableFile);
Set<String> originalFiles = new HashSet<>();
Iterables.toList(cfs.concatWithIndexes()).stream()
.flatMap(c -> c.getLiveSSTables().stream().map(t -> t.descriptor.filenameFor(Component.DATA)))
.forEach(originalFiles::add);
assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(indexTableFile))).isTrue();
assertThat(originalFiles.stream().anyMatch(f -> f.endsWith(baseTableFile))).isTrue();
}
private void createSnapshotAndDelete(String ks, String table, boolean writeData)

88
test/unit/org/apache/cassandra/db/DirectoriesTest.java
View File

@ -21,31 +21,40 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.commons.lang3.StringUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspaceTables;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.Util;
import org.apache.cassandra.auth.AuthKeyspace;
import org.apache.cassandra.config.Config.DiskFailurePolicy;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.Directories.DataDirectories;
import org.apache.cassandra.db.Directories.DataDirectory;
@ -54,23 +63,35 @@ import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspaceTables;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.DefaultFSErrorHandler;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.JVMStabilityInspector;
import static org.apache.cassandra.schema.MockSchema.sstableId;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.assertj.core.api.Assertions.assertThat;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(Parameterized.class)
public class DirectoriesTest
{
public static final String TABLE_NAME = "FakeTable";
@ -84,6 +105,19 @@ public class DirectoriesTest
private static Set<TableMetadata> CFM;
private static Map<String, List<File>> sstablesByTableName;
@Parameterized.Parameter(0)
public SSTableId.Builder<? extends SSTableId> idBuilder;
@Parameterized.Parameter(1)
public Supplier<? extends SSTableId> idGenerator;
@Parameterized.Parameters
public static Collection<Object[]> idBuilders()
{
return Arrays.asList(new Object[]{ SequenceBasedSSTableId.Builder.instance, Util.newSeqGen() },
new Object[]{ UUIDBasedSSTableId.Builder.instance, Util.newUUIDGen() });
}
@BeforeClass
public static void beforeClass()
{
@ -94,6 +128,9 @@ public class DirectoriesTest
@Before
public void beforeTest() throws IOException
{
MockSchema.sstableIds.clear();
MockSchema.sstableIdGenerator = idGenerator;
TABLES = new String[] { "cf1", "ks" };
CFM = new HashSet<>(TABLES.length);
sstablesByTableName = new HashMap<>();
@ -122,7 +159,7 @@ public class DirectoriesTest
return new DataDirectory[] { new DataDirectory(location) };
}
private static void createTestFiles() throws IOException
private void createTestFiles() throws IOException
{
for (TableMetadata cfm : CFM)
{
@ -181,7 +218,7 @@ public class DirectoriesTest
File snapshotDir = new File(tableDir, Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + tag);
snapshotDir.tryCreateDirectories();
Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, 1, SSTableFormat.Type.BIG);
Descriptor sstableDesc = new Descriptor(snapshotDir, KS, table.name, sstableId(1), SSTableFormat.Type.BIG);
createFakeSSTable(sstableDesc);
SnapshotManifest manifest = null;
@ -195,13 +232,13 @@ public class DirectoriesTest
return new FakeSnapshot(table, tag, snapshotDir, manifest);
}
private static List<File> createFakeSSTable(File dir, String cf, int gen) throws IOException
private List<File> createFakeSSTable(File dir, String cf, int gen)
{
Descriptor desc = new Descriptor(dir, KS, cf, gen, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(dir, KS, cf, sstableId(gen), SSTableFormat.Type.BIG);
return createFakeSSTable(desc);
}
private static List<File> createFakeSSTable(Descriptor desc) throws IOException
private List<File> createFakeSSTable(Descriptor desc)
{
List<File> components = new ArrayList<>(3);
for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER })
@ -239,12 +276,15 @@ public class DirectoriesTest
Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, 1, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
File snapshotDir = new File(cfDir(cfm), File.pathSeparator() + Directories.SNAPSHOT_SUBDIR + File.pathSeparator() + LEGACY_SNAPSHOT_NAME);
assertEquals(snapshotDir.toCanonical(), Directories.getSnapshotDirectory(desc, LEGACY_SNAPSHOT_NAME));
File backupsDir = new File(cfDir(cfm), File.pathSeparator() + Directories.BACKUPS_SUBDIR);
assertEquals(backupsDir.toCanonical(), Directories.getBackupsDirectory(desc));
Supplier<? extends SSTableId> uidGen = directories.getUIDGenerator(idBuilder);
assertThat(Stream.generate(uidGen).limit(100).filter(MockSchema.sstableIds::containsValue).collect(Collectors.toList())).isEmpty();
}
}
@ -305,7 +345,7 @@ public class DirectoriesTest
{
String tag = "test";
Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, 0, SSTableFormat.Type.BIG);
Descriptor parentDesc = new Descriptor(directories.getDirectoryForNewSSTables(), KS, cfm.name, sstableId(0), SSTableFormat.Type.BIG);
File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, tag);
List<String> files = new LinkedList<>();
@ -352,8 +392,8 @@ public class DirectoriesTest
{
assertEquals(cfDir(INDEX_CFM), dir);
}
Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, 0, SSTableFormat.Type.BIG);
Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, 0, SSTableFormat.Type.BIG);
Descriptor parentDesc = new Descriptor(parentDirectories.getDirectoryForNewSSTables(), KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
Descriptor indexDesc = new Descriptor(indexDirectories.getDirectoryForNewSSTables(), KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
// snapshot dir should be created under its parent's
File parentSnapshotDirectory = Directories.getSnapshotDirectory(parentDesc, "test");
@ -366,9 +406,9 @@ public class DirectoriesTest
assertTrue(indexDirectories.snapshotExists("test"));
// check true snapshot size
Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, 0, SSTableFormat.Type.BIG);
Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
createFile(parentSnapshot.filenameFor(Component.DATA), 30);
Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, 0, SSTableFormat.Type.BIG);
Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), SSTableFormat.Type.BIG);
createFile(indexSnapshot.filenameFor(Component.DATA), 40);
assertEquals(30, parentDirectories.trueSnapshotsSize());
@ -517,7 +557,7 @@ public class DirectoriesTest
final String n = Long.toString(nanoTime());
Callable<File> directoryGetter = new Callable<File>() {
public File call() throws Exception {
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, 1, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
return Directories.getSnapshotDirectory(desc, n);
}
};

5
test/unit/org/apache/cassandra/db/DiskBoundaryManagerTest.java
View File

@ -22,13 +22,14 @@ import java.net.UnknownHostException;
import java.util.List;
import com.google.common.collect.Lists;
import org.apache.cassandra.io.util.File;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.dht.BootStrapper;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.StorageService;
@ -120,7 +121,7 @@ public class DiskBoundaryManagerTest extends CQLTester
{
MockCFS(ColumnFamilyStore cfs, Directories dirs)
{
super(cfs.keyspace, cfs.getTableName(), 0, cfs.metadata, dirs, false, false, true);
super(cfs.keyspace, cfs.getTableName(), Util.newSeqGen(), cfs.metadata, dirs, false, false, true);
}
}
}

3
test/unit/org/apache/cassandra/db/ImportTest.java
View File

@ -34,6 +34,7 @@ import org.apache.commons.lang3.StringUtils;
import org.junit.Assert;
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.cache.RowCacheKey;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
@ -667,7 +668,7 @@ public class ImportTest extends CQLTester
{
public MockCFS(ColumnFamilyStore cfs, Directories dirs)
{
super(cfs.keyspace, cfs.getTableName(), 0, cfs.metadata, dirs, false, false, true);
super(cfs.keyspace, cfs.getTableName(), Util.newSeqGen(), cfs.metadata, dirs, false, false, true);
}
}
}

2
test/unit/org/apache/cassandra/db/KeyCacheTest.java
View File

@ -202,7 +202,7 @@ public class KeyCacheTest
{
return ColumnFamilyStore.getIfExists(k.desc.ksname, k.desc.cfname).getLiveSSTables()
.stream()
.filter(sstreader -> sstreader.descriptor.generation == k.desc.generation)
.filter(sstreader -> sstreader.descriptor.id == k.desc.id)
.findFirst().get();
}

9
test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
View File

@ -19,6 +19,8 @@
package org.apache.cassandra.db;
import com.google.common.io.Files;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.compaction.OperationType;
@ -32,6 +34,7 @@ import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
@ -47,9 +50,9 @@ import org.junit.Test;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Supplier;
import org.apache.cassandra.io.util.File;
@ -84,12 +87,12 @@ public class SerializationHeaderTest
schemaWithStatic = schemaWithStatic.unbuild().recordColumnDrop(columnRegular, 0L).build();
schemaWithRegular = schemaWithRegular.unbuild().recordColumnDrop(columnStatic, 0L).build();
final AtomicInteger generation = new AtomicInteger();
Supplier<SequenceBasedSSTableId> id = Util.newSeqGen();
File dir = new File(Files.createTempDir());
try
{
BiFunction<TableMetadata, Function<ByteBuffer, Clustering<?>>, Callable<Descriptor>> writer = (schema, clusteringFunction) -> () -> {
Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.keyspace, schema.name, generation.incrementAndGet(), SSTableFormat.Type.BIG);
Descriptor descriptor = new Descriptor(BigFormat.latestVersion, dir, schema.keyspace, schema.name, id.get(), SSTableFormat.Type.BIG);
SerializationHeader header = SerializationHeader.makeWithoutStats(schema);
try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);

106
test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator40Test.java → test/unit/org/apache/cassandra/db/SystemKeyspaceMigrator41Test.java
View File

@ -22,9 +22,9 @@ import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.UUID;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
@ -36,16 +36,22 @@ import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.junit.Assert.assertEquals;
public class SystemKeyspaceMigrator40Test extends CQLTester
public class SystemKeyspaceMigrator41Test extends CQLTester
{
@Test
public void testMigratePeers() throws Throwable
{
String legacyTab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEERS);
String tab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEERS_V2);
String insert = String.format("INSERT INTO %s ("
+ "peer, "
+ "data_center, "
@ -57,7 +63,7 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
+ "schema_version, "
+ "tokens) "
+ " values ( ?, ?, ? , ? , ?, ?, ?, ?, ?)",
SystemKeyspaceMigrator40.legacyPeersName);
legacyTab);
UUID hostId = UUID.randomUUID();
UUID schemaVersion = UUID.randomUUID();
execute(insert,
@ -69,10 +75,10 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
InetAddress.getByName("127.0.0.3"),
schemaVersion,
ImmutableSet.of("foobar"));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migratePeers();
int rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peersName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
@ -91,15 +97,15 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
assertEquals(1, rowCount);
//Test nulls/missing don't prevent the row from propagating
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyPeersName));
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.peersName));
execute(String.format("TRUNCATE %s", legacyTab));
execute(String.format("TRUNCATE %s", tab));
execute(String.format("INSERT INTO %s (peer) VALUES (?)", SystemKeyspaceMigrator40.legacyPeersName),
execute(String.format("INSERT INTO %s (peer) VALUES (?)", legacyTab),
InetAddress.getByName("127.0.0.1"));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migratePeers();
rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peersName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
}
@ -109,19 +115,21 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
@Test
public void testMigratePeerEvents() throws Throwable
{
String legacyTab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_PEER_EVENTS);
String tab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.PEER_EVENTS_V2);
String insert = String.format("INSERT INTO %s ("
+ "peer, "
+ "hints_dropped) "
+ " values ( ?, ? )",
SystemKeyspaceMigrator40.legacyPeerEventsName);
legacyTab);
TimeUUID uuid = nextTimeUUID();
execute(insert,
InetAddress.getByName("127.0.0.1"),
ImmutableMap.of(uuid, 42));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migratePeerEvents();
int rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peerEventsName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
assertEquals(InetAddress.getByName("127.0.0.1"), row.getInetAddress("peer"));
@ -131,15 +139,15 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
assertEquals(1, rowCount);
//Test nulls/missing don't prevent the row from propagating
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyPeerEventsName));
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.peerEventsName));
execute(String.format("TRUNCATE %s", legacyTab));
execute(String.format("TRUNCATE %s", tab));
execute(String.format("INSERT INTO %s (peer) VALUES (?)", SystemKeyspaceMigrator40.legacyPeerEventsName),
execute(String.format("INSERT INTO %s (peer) VALUES (?)", legacyTab),
InetAddress.getByName("127.0.0.1"));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migratePeerEvents();
rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.peerEventsName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
}
@ -149,22 +157,24 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
@Test
public void testMigrateTransferredRanges() throws Throwable
{
String legacyTab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_TRANSFERRED_RANGES);
String tab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.TRANSFERRED_RANGES_V2);
String insert = String.format("INSERT INTO %s ("
+ "operation, "
+ "peer, "
+ "keyspace_name, "
+ "ranges) "
+ " values ( ?, ?, ?, ? )",
SystemKeyspaceMigrator40.legacyTransferredRangesName);
legacyTab);
execute(insert,
"foo",
InetAddress.getByName("127.0.0.1"),
"bar",
ImmutableSet.of(ByteBuffer.wrap(new byte[] { 42 })));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migrateTransferredRanges();
int rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.transferredRangesName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
assertEquals("foo", row.getString("operation"));
@ -176,17 +186,17 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
assertEquals(1, rowCount);
//Test nulls/missing don't prevent the row from propagating
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.legacyTransferredRangesName));
execute(String.format("TRUNCATE %s", SystemKeyspaceMigrator40.transferredRangesName));
execute(String.format("TRUNCATE %s", legacyTab));
execute(String.format("TRUNCATE %s", tab));
execute(String.format("INSERT INTO %s (operation, peer, keyspace_name) VALUES (?, ?, ?)", SystemKeyspaceMigrator40.legacyTransferredRangesName),
execute(String.format("INSERT INTO %s (operation, peer, keyspace_name) VALUES (?, ?, ?)", legacyTab),
"foo",
InetAddress.getByName("127.0.0.1"),
"bar");
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migrateTransferredRanges();
rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.transferredRangesName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
}
@ -196,19 +206,21 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
@Test
public void testMigrateAvailableRanges() throws Throwable
{
String legacyTab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_AVAILABLE_RANGES);
String tab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.AVAILABLE_RANGES_V2);
Range<Token> testRange = new Range<>(DatabaseDescriptor.getPartitioner().getRandomToken(), DatabaseDescriptor.getPartitioner().getRandomToken());
String insert = String.format("INSERT INTO %s ("
+ "keyspace_name, "
+ "ranges) "
+ " values ( ?, ? )",
SystemKeyspaceMigrator40.legacyAvailableRangesName);
legacyTab);
execute(insert,
"foo",
ImmutableSet.of(SystemKeyspace.rangeToBytes(testRange)));
SystemKeyspaceMigrator40.migrate();
SystemKeyspaceMigrator41.migrateAvailableRanges();
int rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", SystemKeyspaceMigrator40.availableRangesName)))
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
assertEquals("foo", row.getString("keyspace_name"));
@ -216,4 +228,40 @@ public class SystemKeyspaceMigrator40Test extends CQLTester
}
assertEquals(1, rowCount);
}
@Test
public void testMigrateSSTableActivity() throws Throwable
{
FBUtilities.setPreviousReleaseVersionString(CassandraVersion.NULL_VERSION.toString());
String legacyTab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_SSTABLE_ACTIVITY);
String tab = String.format("%s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.SSTABLE_ACTIVITY_V2);
String insert = String.format("INSERT INTO %s (%s) VALUES (%s)",
legacyTab,
StringUtils.join(new String[] {"keyspace_name",
"columnfamily_name",
"generation",
"rate_120m",
"rate_15m"}, ", "),
StringUtils.repeat("?", ", ", 5));
execute(insert, "ks", "tab", 5, 123.234d, 345.456d);
ColumnFamilyStore cf = getColumnFamilyStore(SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.SSTABLE_ACTIVITY_V2);
cf.truncateBlocking();
cf.clearUnsafe();
SystemKeyspaceMigrator41.migrateSSTableActivity();
int rowCount = 0;
for (UntypedResultSet.Row row : execute(String.format("SELECT * FROM %s", tab)))
{
rowCount++;
assertEquals("ks", row.getString("keyspace_name"));
assertEquals("tab", row.getString("table_name"));
assertEquals(new SequenceBasedSSTableId(5).asBytes(), row.getBytes("id"));
assertEquals(123.234d, row.getDouble("rate_120m"), 0.001d);
assertEquals(345.456d, row.getDouble("rate_15m"), 0.001d);
}
assertEquals(1, rowCount);
}
}

9
test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
View File

@ -32,12 +32,10 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import org.apache.cassandra.io.util.File;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,6 +53,7 @@ import org.apache.cassandra.db.compaction.AbstractStrategyHolder.GroupedSSTableC
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.notifications.SSTableAddedNotification;
import org.apache.cassandra.notifications.SSTableDeletingNotification;
import org.apache.cassandra.schema.CompactionParams;
@ -461,7 +460,7 @@ public class CompactionStrategyManagerTest
int firstKey = Integer.parseInt(new String(ByteBufferUtil.getArray(reader.first.getKey())));
while (boundaries[index] <= firstKey)
index++;
logger.debug("Index for SSTable {} on boundary {} is {}", reader.descriptor.generation, Arrays.toString(boundaries), index);
logger.debug("Index for SSTable {} on boundary {} is {}", reader.descriptor.id, Arrays.toString(boundaries), index);
return index;
}
@ -520,7 +519,7 @@ public class CompactionStrategyManagerTest
{
MockCFS(ColumnFamilyStore cfs, Directories dirs)
{
super(cfs.keyspace, cfs.getTableName(), 0, cfs.metadata, dirs, false, false, true);
super(cfs.keyspace, cfs.getTableName(), Util.newSeqGen(), cfs.metadata, dirs, false, false, true);
}
}
@ -531,7 +530,7 @@ public class CompactionStrategyManagerTest
private MockCFSForCSM(ColumnFamilyStore cfs, CountDownLatch latch, AtomicInteger upgradeTaskCount)
{
super(cfs.keyspace, cfs.name, 10, cfs.metadata, cfs.getDirectories(), true, false, false);
super(cfs.keyspace, cfs.name, Util.newSeqGen(10), cfs.metadata, cfs.getDirectories(), true, false, false);
this.latch = latch;
this.upgradeTaskCount = upgradeTaskCount;
}

7
test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
View File

@ -63,6 +63,8 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.SSTableIdFactory;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.File;
@ -73,6 +75,7 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@ -270,7 +273,7 @@ public class CompactionsTest
assertEquals(1, sstables.size());
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
SSTableId prevGeneration = sstable.descriptor.id;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).absolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
@ -282,7 +285,7 @@ public class CompactionsTest
// CF should have only one sstable with generation number advanced
sstables = cfs.getLiveSSTables();
assertEquals(1, sstables.size());
assertEquals( prevGeneration + 1, sstables.iterator().next().descriptor.generation);
assertThat(SSTableIdFactory.COMPARATOR.compare(prevGeneration, sstables.iterator().next().descriptor.id)).isLessThan(0);
}
public static void writeSSTableWithRangeTombstoneMaskingOneColumn(ColumnFamilyStore cfs, TableMetadata table, int[] dks) {

2
test/unit/org/apache/cassandra/db/compaction/LeveledGenerationsTest.java
View File

@ -194,6 +194,6 @@ public class LeveledGenerationsTest extends CQLTester
private void print(SSTableReader sstable)
{
System.out.println(String.format("%d %s %s %d", sstable.descriptor.generation, sstable.first, sstable.last, sstable.getSSTableLevel()));
System.out.println(String.format("%d %s %s %d", sstable.descriptor.id, sstable.first, sstable.last, sstable.getSSTableLevel()));
}
}

3
test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
View File

@ -47,6 +47,7 @@ import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
@ -1260,7 +1261,7 @@ public class LogTransactionTest extends AbstractTransactionalTest
private static SSTableReader sstable(File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException
{
Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation, SSTableFormat.Type.BIG);
Descriptor descriptor = new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), new SequenceBasedSSTableId(generation), SSTableFormat.Type.BIG);
Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components)
{

2
test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
View File

@ -189,7 +189,7 @@ public class SASIIndexTest
Descriptor snapshotSSTable = new Descriptor(snapshotDirectory,
sstable.getKeyspaceName(),
sstable.getColumnFamilyName(),
sstable.descriptor.generation,
sstable.descriptor.id,
sstable.descriptor.formatType);
Set<Component> components = snapshotSSTables.get(snapshotSSTable);

10
test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java
View File

@ -84,12 +84,12 @@ public class DescriptorTest
private void testFromFilenameFor(File dir)
{
checkFromFilename(new Descriptor(dir, ksname, cfname, 1, SSTableFormat.Type.BIG));
checkFromFilename(new Descriptor(dir, ksname, cfname, new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG));
// secondary index
String idxName = "myidx";
File idxDir = new File(dir.absolutePath() + File.pathSeparator() + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName);
checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 4, SSTableFormat.Type.BIG));
checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, new SequenceBasedSSTableId(4), SSTableFormat.Type.BIG));
}
private void checkFromFilename(Descriptor original)
@ -103,7 +103,7 @@ public class DescriptorTest
assertEquals(original.ksname, desc.ksname);
assertEquals(original.cfname, desc.cfname);
assertEquals(original.version, desc.version);
assertEquals(original.generation, desc.generation);
assertEquals(original.id, desc.id);
assertEquals(Component.DATA, pair.right);
}
@ -112,8 +112,8 @@ public class DescriptorTest
{
// Descriptor should be equal when parent directory points to the same directory
File dir = new File(".");
Descriptor desc1 = new Descriptor(dir, "ks", "cf", 1, SSTableFormat.Type.BIG);
Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", 1, SSTableFormat.Type.BIG);
Descriptor desc1 = new Descriptor(dir, "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG);
Descriptor desc2 = new Descriptor(dir.toAbsolute(), "ks", "cf", new SequenceBasedSSTableId(1), SSTableFormat.Type.BIG);
assertEquals(desc1, desc2);
assertEquals(desc1.hashCode(), desc2.hashCode());
}

13
test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
View File

@ -19,6 +19,8 @@ package org.apache.cassandra.io.sstable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -56,7 +58,6 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
@ -141,14 +142,10 @@ public class LegacySSTableTest
/**
* Get a descriptor for the legacy sstable at the given version.
*/
protected Descriptor getDescriptor(String legacyVersion, String table)
protected Descriptor getDescriptor(String legacyVersion, String table) throws IOException
{
return new Descriptor(SSTableFormat.Type.BIG.info.getVersion(legacyVersion),
getTableDir(legacyVersion, table),
"legacy_tables",
table,
1,
SSTableFormat.Type.BIG);
Path file = Files.list(getTableDir(legacyVersion, table).toPath()).findFirst().orElseThrow(() -> new RuntimeException(String.format("No files for version=%s and table=%s", legacyVersion, table)));
return Descriptor.fromFilename(new File(file));
}
@Test

19
test/unit/org/apache/cassandra/io/sstable/SSTableHeaderFixTest.java
View File

@ -19,12 +19,14 @@ package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -36,6 +38,8 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
@ -63,6 +67,7 @@ import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.SequentialWriter;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -77,6 +82,7 @@ import static org.junit.Assert.fail;
* Test the functionality of {@link SSTableHeaderFix}.
* It writes an 'big-m' version sstable(s) and executes against these.
*/
@RunWith(Parameterized.class)
public class SSTableHeaderFixTest
{
static
@ -86,9 +92,20 @@ public class SSTableHeaderFixTest
private File temporaryFolder;
@Parameterized.Parameter
public Supplier<? extends SSTableId> sstableIdGen;
@Parameterized.Parameters
public static Collection<Object[]> parameters()
{
return MockSchema.sstableIdGenerators();
}
@Before
public void setup()
{
MockSchema.sstableIds.clear();
MockSchema.sstableIdGenerator = sstableIdGen;
File f = FileUtils.createTempFile("SSTableUDTFixTest", "");
f.tryDelete();
f.tryCreateDirectories();
@ -805,7 +822,7 @@ public class SSTableHeaderFixTest
try
{
Descriptor desc = new Descriptor(version, dir, "ks", "cf", generation, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(version, dir, "ks", "cf", MockSchema.sstableId(generation), SSTableFormat.Type.BIG);
// Just create the component files - we don't really need those.
for (Component component : requiredComponents)

172
test/unit/org/apache/cassandra/io/sstable/SSTableIdTest.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
import com.google.common.primitives.UnsignedBytes;
import org.junit.Test;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.utils.TimeUUID;
import org.awaitility.Awaitility;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.quicktheories.QuickTheory.qt;
import static org.quicktheories.generators.SourceDSL.longs;
public class SSTableIdTest
{
@Test
public void testSequenceBasedIdProperties()
{
testSSTableIdProperties(SequenceBasedSSTableId.Builder.instance);
}
@Test
public void testUUIDBasedIdProperties()
{
testSSTableIdProperties(UUIDBasedSSTableId.Builder.instance);
}
private void testSSTableIdProperties(SSTableId.Builder<?> builder)
{
List<SSTableId> ids = Stream.generate(builder.generator(Stream.empty()))
.limit(100).collect(Collectors.toList());
assertThat(ids).isSorted();
assertThat(Sets.newHashSet(ids)).hasSameSizeAs(ids);
List<ByteBuffer> serIds = ids.stream().map(SSTableId::asBytes).collect(Collectors.toList());
assertThat(serIds).isSortedAccordingTo((o1, o2) -> UnsignedBytes.lexicographicalComparator().compare(o1.array(), o2.array()));
List<SSTableId> deserIds = serIds.stream().map(builder::fromBytes).collect(Collectors.toList());
assertThat(deserIds).containsExactlyElementsOf(ids);
List<String> stringifiedIds = ids.stream().map(SSTableId::asString).collect(Collectors.toList());
if (!(builder instanceof SequenceBasedSSTableId.Builder))
{
// the legacy string representation is not sortable
assertThat(stringifiedIds).isSorted();
}
List<SSTableId> destringifiedIds = stringifiedIds.stream().map(builder::fromString).collect(Collectors.toList());
assertThat(destringifiedIds).containsExactlyElementsOf(ids);
generatorFuzzTest(builder);
}
@Test
public void testUUIDBytesSerDe()
{
qt().forAll(longs().all(), longs().all()).checkAssert((msb, lsb) -> {
msb = (msb & ~0xf000) | 0x1000; // v1
TimeUUID uuid = TimeUUID.fromBytes(msb, lsb);
UUIDBasedSSTableId id = new UUIDBasedSSTableId(uuid);
testBytesSerialization(id);
testStringSerialization(id);
});
}
private void testBytesSerialization(UUIDBasedSSTableId id)
{
ByteBuffer buf = id.asBytes();
assertThat(buf.remaining()).isEqualTo(UUIDBasedSSTableId.BYTES_LEN);
assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isTrue();
assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(buf)).isFalse();
SSTableId fromBytes = SSTableIdFactory.instance.fromBytes(buf);
assertThat(fromBytes).isEqualTo(id);
}
private void testStringSerialization(UUIDBasedSSTableId id)
{
String s = id.asString();
assertThat(s).hasSize(UUIDBasedSSTableId.STRING_LEN);
assertThat(s).matches(Pattern.compile("[0-9a-z]{4}_[0-9a-z]{4}_[0-9a-z]{18}"));
assertThat(UUIDBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isTrue();
assertThat(SequenceBasedSSTableId.Builder.instance.isUniqueIdentifier(s)).isFalse();
SSTableId fromString = SSTableIdFactory.instance.fromString(s);
assertThat(fromString).isEqualTo(id);
}
@Test
public void testComparator()
{
List<SSTableId> ids = new ArrayList<>(Collections.nCopies(300, null));
for (int i = 0; i < 100; i++)
{
ids.set(i + 100, new SequenceBasedSSTableId(ThreadLocalRandom.current().nextInt(1000000)));
ids.set(i, new UUIDBasedSSTableId(TimeUUID.Generator.atUnixMillis(ThreadLocalRandom.current().nextLong(10000), 0)));
}
List<SSTableId> shuffledIds = new ArrayList<>(ids);
Collections.shuffle(shuffledIds);
assertThat(shuffledIds).isNotEqualTo(ids);
List<SSTableId> sortedIds = new ArrayList<>(shuffledIds);
sortedIds.sort(SSTableIdFactory.COMPARATOR);
assertThat(sortedIds).isNotEqualTo(shuffledIds);
assertThat(sortedIds).isSortedAccordingTo(SSTableIdFactory.COMPARATOR);
assertThat(sortedIds.subList(0, 100)).containsOnlyNulls();
assertThat(sortedIds.subList(100, 200)).allMatch(id -> id instanceof SequenceBasedSSTableId);
assertThat(sortedIds.subList(200, 300)).allMatch(id -> id instanceof UUIDBasedSSTableId);
assertThat(sortedIds.subList(100, 200)).isSortedAccordingTo(Comparator.comparing(o -> ((SequenceBasedSSTableId) o)));
assertThat(sortedIds.subList(200, 300)).isSortedAccordingTo(Comparator.comparing(o -> ((UUIDBasedSSTableId) o)));
}
private static <T extends SSTableId> void generatorFuzzTest(SSTableId.Builder<T> builder)
{
final int NUM_THREADS = 10, IDS_PER_THREAD = 10;
Set<SSTableId> ids = new CopyOnWriteArraySet<>();
Supplier<T> generator = builder.generator(Stream.empty());
ExecutorPlus service = executorFactory().pooled("test", NUM_THREADS);
CyclicBarrier barrier = new CyclicBarrier(NUM_THREADS);
for (int i = 0; i < NUM_THREADS; i++)
{
service.submit(() -> {
for (int k = 0; k < IDS_PER_THREAD; k++)
{
barrier.await();
ids.add(generator.get());
}
return null;
});
}
Awaitility.await().atMost(Duration.ofSeconds(10)).untilAsserted(() -> assertThat(service.getCompletedTaskCount()).isEqualTo(NUM_THREADS));
assertThat(ids).hasSize(NUM_THREADS * IDS_PER_THREAD);
}
}

10
test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
View File

@ -23,6 +23,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Stream;
import com.google.common.collect.Sets;
import org.apache.cassandra.io.util.File;
@ -751,7 +752,7 @@ public class SSTableReaderTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
SSTableReader sstable = getNewSSTable(cfs);
Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", 0);
Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
SSTableReader.moveAndOpenSSTable(cfs, sstable.descriptor, notLiveDesc, sstable.components, false);
}
@ -761,7 +762,7 @@ public class SSTableReaderTest
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD);
SSTableReader sstable = getNewSSTable(cfs);
Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", 0);
Descriptor notLiveDesc = new Descriptor(new File("/tmp"), "", "", SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get());
SSTableReader.moveAndOpenSSTable(cfs, notLiveDesc, sstable.descriptor, sstable.components, false);
}
@ -775,7 +776,8 @@ public class SSTableReaderTest
sstable.selfRef().release();
File tmpdir = new File(Files.createTempDirectory("testMoveAndOpen"));
tmpdir.deleteOnExit();
Descriptor notLiveDesc = new Descriptor(tmpdir, sstable.descriptor.ksname, sstable.descriptor.cfname, 100);
SSTableId id = SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get();
Descriptor notLiveDesc = new Descriptor(tmpdir, sstable.descriptor.ksname, sstable.descriptor.cfname, id);
// make sure the new directory is empty and that the old files exist:
for (Component c : sstable.components)
{
@ -789,7 +791,7 @@ public class SSTableReaderTest
{
File f = new File(notLiveDesc.filenameFor(c));
assertTrue(f.exists());
assertTrue(f.toString().contains("-100-"));
assertTrue(f.toString().contains(String.format("-%s-", id)));
f.deleteOnExit();
assertFalse(new File(sstable.descriptor.filenameFor(c)).exists());
}

15
test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
View File

@ -22,6 +22,7 @@ package org.apache.cassandra.io.sstable;
import org.apache.cassandra.io.util.File;
import java.io.IOException;
import java.util.*;
import java.util.stream.Stream;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.TableMetadata;
@ -70,7 +71,7 @@ public class SSTableUtils
}
*/
public static File tempSSTableFile(String keyspaceName, String cfname, int generation) throws IOException
public static File tempSSTableFile(String keyspaceName, String cfname, SSTableId id) throws IOException
{
File tempdir = FileUtils.createTempFile(keyspaceName, cfname);
if(!tempdir.tryDelete() || !tempdir.tryCreateDirectory())
@ -79,7 +80,7 @@ public class SSTableUtils
File cfDir = new File(tempdir, keyspaceName + File.pathSeparator() + cfname);
cfDir.tryCreateDirectories();
cfDir.deleteOnExit();
File datafile = new File(new Descriptor(cfDir, keyspaceName, cfname, generation, SSTableFormat.Type.BIG).filenameFor(Component.DATA));
File datafile = new File(new Descriptor(cfDir, keyspaceName, cfname, id, SSTableFormat.Type.BIG).filenameFor(Component.DATA));
if (!datafile.createFileIfNotExists())
throw new IOException("unable to create file " + datafile);
datafile.deleteOnExit();
@ -132,7 +133,7 @@ public class SSTableUtils
private String cfname = CFNAME;
private Descriptor dest = null;
private boolean cleanup = true;
private int generation = 0;
private SSTableId id = SSTableIdFactory.instance.defaultBuilder().generator(Stream.empty()).get();
Context() {}
@ -160,11 +161,11 @@ public class SSTableUtils
}
/**
* Sets the generation number for the generated SSTable. Ignored if "dest()" is set.
* Sets the identifier for the generated SSTable. Ignored if "dest()" is set.
*/
public Context generation(int generation)
public Context id(SSTableId id)
{
this.generation = generation;
this.id = id;
return this;
}
@ -215,7 +216,7 @@ public class SSTableUtils
public Collection<SSTableReader> write(int expectedSize, Appender appender) throws IOException
{
File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, generation) : new File(dest.filenameFor(Component.DATA));
File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : new File(dest.filenameFor(Component.DATA));
TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname);
ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id);
SerializationHeader header = appender.header();

6
test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
View File

@ -124,13 +124,13 @@ public class SSTableWriterTestBase extends SchemaLoader
*/
public static void validateCFS(ColumnFamilyStore cfs)
{
Set<Integer> liveDescriptors = new HashSet<>();
Set<SSTableId> liveDescriptors = new HashSet<>();
long spaceUsed = 0;
for (SSTableReader sstable : cfs.getLiveSSTables())
{
assertFalse(sstable.isMarkedCompacted());
assertEquals(1, sstable.selfRef().globalCount());
liveDescriptors.add(sstable.descriptor.generation);
liveDescriptors.add(sstable.descriptor.id);
spaceUsed += sstable.bytesOnDisk();
}
for (File dir : cfs.getDirectories().getCFDirectories())
@ -140,7 +140,7 @@ public class SSTableWriterTestBase extends SchemaLoader
if (f.name().contains("Data"))
{
Descriptor d = Descriptor.fromFilename(f.absolutePath());
assertTrue(d.toString(), liveDescriptors.contains(d.generation));
assertTrue(d.toString(), liveDescriptors.contains(d.id));
}
}
}

13
test/unit/org/apache/cassandra/io/sstable/format/SSTableFlushObserverTest.java
View File

@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.Iterator;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.ColumnMetadata;
@ -92,12 +93,14 @@ public class SSTableFlushObserverTest
throw new FSWriteError(new IOException("failed to create tmp directory"), directory.absolutePath());
SSTableFormat.Type sstableFormat = SSTableFormat.Type.current();
Descriptor descriptor = new Descriptor(sstableFormat.info.getLatestVersion(),
directory,
cfm.keyspace,
cfm.name,
new SequenceBasedSSTableId(0),
sstableFormat);
BigTableWriter writer = new BigTableWriter(new Descriptor(sstableFormat.info.getLatestVersion(),
directory,
KS_NAME, CF_NAME,
0,
sstableFormat),
BigTableWriter writer = new BigTableWriter(descriptor,
10L, 0L, null, false, TableMetadataRef.forOfflineTools(cfm),
new MetadataCollector(cfm.comparator).sstableLevel(0),
new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS),

7
test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.cassandra.io.sstable.metadata;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.util.*;
import java.io.IOException;
import java.util.Arrays;
@ -61,7 +62,7 @@ public class MetadataSerializerTest
MetadataSerializer serializer = new MetadataSerializer();
File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
Descriptor desc = new Descriptor(statsFile.parent(), "", "", 0, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG);
try (RandomAccessReader in = RandomAccessReader.open(statsFile))
{
Map<MetadataType, MetadataComponent> deserialized = serializer.deserialize(desc, in, EnumSet.allOf(MetadataType.class));
@ -88,7 +89,7 @@ public class MetadataSerializerTest
// Serialize w/ overflowed histograms:
MetadataSerializer serializer = new MetadataSerializer();
File statsFile = serialize(originalMetadata, serializer, BigFormat.latestVersion);
Descriptor desc = new Descriptor(statsFile.parent(), "", "", 0, SSTableFormat.Type.BIG);
Descriptor desc = new Descriptor(statsFile.parent(), "", "", new SequenceBasedSSTableId(0), SSTableFormat.Type.BIG);
try (RandomAccessReader in = RandomAccessReader.open(statsFile))
{
@ -171,7 +172,7 @@ public class MetadataSerializerTest
File statsFileLa = serialize(originalMetadata, serializer, BigFormat.instance.getVersion(oldV));
// Reading both as earlier version should yield identical results.
SSTableFormat.Type stype = SSTableFormat.Type.current();
Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.parent(), "", "", 0, stype);
Descriptor desc = new Descriptor(stype.info.getVersion(oldV), statsFileLb.parent(), "", "", new SequenceBasedSSTableId(0), stype);
try (RandomAccessReader inLb = RandomAccessReader.open(statsFileLb);
RandomAccessReader inLa = RandomAccessReader.open(statsFileLa))
{

32
test/unit/org/apache/cassandra/schema/MockSchema.java
View File

@ -18,15 +18,20 @@
*/
package org.apache.cassandra.schema;
import org.apache.cassandra.Util;
import org.apache.cassandra.io.util.File;
import java.io.IOException;
import java.util.*;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.UTF8Type;
@ -34,12 +39,14 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.IndexSummary;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.ChannelProxy;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.Memory;
@ -50,6 +57,21 @@ import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABL
public class MockSchema
{
public static Supplier<? extends SSTableId> sstableIdGenerator = Util.newSeqGen();
public static final ConcurrentMap<Integer, SSTableId> sstableIds = new ConcurrentHashMap<>();
public static SSTableId sstableId(int idx)
{
return sstableIds.computeIfAbsent(idx, ignored -> sstableIdGenerator.get());
}
public static Collection<Object[]> sstableIdGenerators()
{
return Arrays.asList(new Object[]{ Util.newSeqGen() },
new Object[]{ Util.newUUIDGen() });
}
static
{
Memory offsets = Memory.allocate(4);
@ -112,7 +134,7 @@ public class MockSchema
Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(),
cfs.keyspace.getName(),
cfs.getTableName(),
generation, SSTableFormat.Type.BIG);
sstableId(generation), SSTableFormat.Type.BIG);
Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components)
{
@ -174,7 +196,7 @@ public class MockSchema
public static ColumnFamilyStore newCFS(TableMetadata metadata)
{
return new ColumnFamilyStore(ks, metadata.name, 0, new TableMetadataRef(metadata), new Directories(metadata), false, false, false);
return new ColumnFamilyStore(ks, metadata.name, Util.newSeqGen(), new TableMetadataRef(metadata), new Directories(metadata), false, false, false);
}
public static TableMetadata newTableMetadata(String ksname)

3
test/unit/org/apache/cassandra/service/SSTablesGlobalTrackerTest.java
View File

@ -28,6 +28,7 @@ import java.util.stream.Collectors;
import org.junit.Test;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.VersionAndType;
import org.apache.cassandra.io.util.File;
@ -100,7 +101,7 @@ public class SSTablesGlobalTrackerTest
tables(),
generations(),
sstableVersionString(),
(f, k, t, g, v) -> new Descriptor(v, new File(Files.currentFolder()), k, t, g, f));
(f, k, t, g, v) -> new Descriptor(v, new File(Files.currentFolder()), k, t, new SequenceBasedSSTableId(g), f));
}
private Gen<List<Descriptor>> descriptorLists(int minSize)

3
test/unit/org/apache/cassandra/streaming/compression/CompressedInputStreamTest.java
View File

@ -23,6 +23,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.*;
import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.junit.BeforeClass;
import org.junit.Rule;
@ -119,7 +120,7 @@ public class CompressedInputStreamTest
// write compressed data file of longs
File parentDir = new File(tempFolder.newFolder());
Descriptor desc = new Descriptor(parentDir, "ks", "cf", 1);
Descriptor desc = new Descriptor(parentDir, "ks", "cf", new SequenceBasedSSTableId(1));
File tmp = new File(desc.filenameFor(Component.DATA));
MetadataCollector collector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
CompressionParams param = CompressionParams.snappy(32, minCompressRatio);

10
test/unit/org/apache/cassandra/tools/JMXCompatabilityTest.java
View File

@ -160,6 +160,16 @@ public class JMXCompatabilityTest extends CQLTester
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-4.0-jmx.yaml");
}
@Test
public void diff41() throws Throwable
{
List<String> excludeObjects = Arrays.asList();
List<String> excludeAttributes = Arrays.asList();
List<String> excludeOperations = Arrays.asList();
diff(excludeObjects, excludeAttributes, excludeOperations, "test/data/jmxdump/cassandra-4.1-jmx.yaml");
}
private void diff(List<String> excludeObjects, List<String> excludeAttributes, List<String> excludeOperations, String original) throws Throwable
{
setupStandardTables();