natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Consolidate all snapshot management to SnapshotManager 

patch by Stefan Miklosovic; reviewed by Francisco Guerrero for CASSANDRA-18111
This commit is contained in:
Stefan Miklosovic 2024-06-20 10:29:51 +02:00
parent 4f49ca5e29
commit f410b0fa0b
No known key found for this signature in database
GPG Key ID: 32F35CB2F546D93E
79 changed files with 5142 additions and 1523 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.1
* Consolidate all snapshot management to SnapshotManager and introduce SnapshotManagerMBean (CASSANDRA-18111)
* Fix RequestFailureReason constants codes (CASSANDRA-20126)
* Introduce SSTableSimpleScanner for compaction (CASSANDRA-20092)
* Include column drop timestamp in alter table transformation (CASSANDRA-18961)

3
NEWS.txt
View File

@ -89,6 +89,9 @@ New features
when 'GENERATED PASSWORD' clause is used. Character sets supported are: English, Cyrillic, modern Cyrillic,
German, Polish and Czech.
- JMX SSL configuration can be now done in cassandra.yaml via jmx_encryption_options section instead of cassandra-env.sh
- There is new MBean of name org.apache.cassandra.service.snapshot:type=SnapshotManager which exposes user-facing
snapshot operations. Snapshot-related methods on StorageServiceMBean are still present and functional
but marked as deprecated.
Upgrading

316
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -18,11 +18,9 @@
package org.apache.cassandra.db;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -32,7 +30,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -68,7 +65,6 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -79,7 +75,6 @@ import org.apache.cassandra.cache.RowCacheSentinel;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.concurrent.FutureTask;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
@ -116,8 +111,6 @@ import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.IScrubber;
@ -131,7 +124,6 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.metrics.Sampler;
import org.apache.cassandra.metrics.Sampler.Sample;
import org.apache.cassandra.metrics.Sampler.SamplerType;
@ -157,9 +149,7 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.TablePaxosRepairHistory;
import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.streaming.TableStreamManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
@ -185,7 +175,6 @@ import static org.apache.cassandra.config.DatabaseDescriptor.getFlushWriters;
import static org.apache.cassandra.db.commitlog.CommitLogPosition.NONE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.apache.cassandra.utils.Throwables.maybeFail;
import static org.apache.cassandra.utils.Throwables.merge;
import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch;
@ -260,8 +249,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
private static final String SAMPLING_RESULTS_NAME = "SAMPLING_RESULTS";
public static final String SNAPSHOT_TRUNCATE_PREFIX = "truncated";
public static final String SNAPSHOT_DROP_PREFIX = "dropped";
static final String TOKEN_DELIMITER = ":";
/** Special values used when the local ranges are not changed with ring changes (e.g. local tables). */
@ -517,6 +504,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
// Note that this needs to happen before we load the first sstables, or the global sstable tracker will not
// be notified on the initial loading.
data.subscribe(StorageService.instance.sstablesTracker);
data.subscribe(SnapshotManager.instance);
Collection<SSTableReader> sstables = null;
// scan for sstables corresponding to this cf and load them
@ -785,14 +773,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
* Removes unnecessary files from the cf directory at startup: these include temp files, orphans, zero-length files
* and compacted sstables. Files that cannot be recognized will be ignored.
*/
public static void scrubDataDirectories(TableMetadata metadata) throws StartupException
public static void scrubDataDirectories(TableMetadata metadata) throws StartupException
{
Directories directories = new Directories(metadata);
Set<File> cleanedDirectories = new HashSet<>();
// clear ephemeral snapshots that were not properly cleared last session (CASSANDRA-7357)
clearEphemeralSnapshots(directories);
directories.removeTemporaryDirectories();
logger.trace("Removing temporary or obsoleted files from unfinished operations for table {}", metadata.name);
@ -977,6 +962,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
return keyspace.getName();
}
public String getKeyspaceTableName()
{
return getKeyspaceName() + '.' + getTableName();
}
public Descriptor newSSTableDescriptor(File directory)
{
return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
@ -1757,12 +1747,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean alwaysFail, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException
{
// skip snapshot creation during scrub, SEE JIRA 5891
if(!disableSnapshot)
{
Instant creationTime = now();
String snapshotName = "pre-scrub-" + creationTime.toEpochMilli();
snapshotWithoutMemtable(snapshotName, creationTime);
}
if (!disableSnapshot)
data.notifyPreScrubbed();
try
{
@ -2127,262 +2113,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
return metadata().comparator;
}
public TableSnapshot snapshotWithoutMemtable(String snapshotName)
{
return snapshotWithoutMemtable(snapshotName, now());
}
public TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime)
{
return snapshotWithoutMemtable(snapshotName, null, false, null, null, creationTime);
}
/**
* @param ephemeral If this flag is set to true, the snapshot will be cleaned during next startup
*/
public TableSnapshot snapshotWithoutMemtable(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime)
{
if (ephemeral && ttl != null)
{
throw new IllegalStateException(String.format("can not take ephemeral snapshot (%s) while ttl is specified too", snapshotName));
}
if (rateLimiter == null)
rateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
Set<SSTableReader> snapshottedSSTables = new LinkedHashSet<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
{
try (RefViewFragment currentView = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.apply(x))))
{
for (SSTableReader ssTable : currentView.sstables)
{
File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName);
ssTable.createLinks(snapshotDirectory.path(), rateLimiter); // hard links
if (logger.isTraceEnabled())
logger.trace("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory);
snapshottedSSTables.add(ssTable);
}
}
}
return createSnapshot(snapshotName, ephemeral, ttl, snapshottedSSTables, creationTime);
}
protected TableSnapshot createSnapshot(String tag, boolean ephemeral, DurationSpec.IntSecondsBound ttl, Set<SSTableReader> sstables, Instant creationTime) {
Set<File> snapshotDirs = sstables.stream()
.map(s -> Directories.getSnapshotDirectory(s.descriptor, tag).toAbsolute())
.filter(dir -> !Directories.isSecondaryIndexFolder(dir)) // Remove secondary index subdirectory
.collect(Collectors.toCollection(HashSet::new));
// Create and write snapshot manifest
SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime, ephemeral);
File manifestFile = getDirectories().getSnapshotManifestFile(tag);
writeSnapshotManifest(manifest, manifestFile);
snapshotDirs.add(manifestFile.parent().toAbsolute()); // manifest may create empty snapshot dir
// Write snapshot schema
if (!SchemaConstants.isLocalSystemKeyspace(metadata.keyspace) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.keyspace))
{
File schemaFile = getDirectories().getSnapshotSchemaFile(tag);
writeSnapshotSchema(schemaFile);
snapshotDirs.add(schemaFile.parent().toAbsolute()); // schema may create empty snapshot dir
}
TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(),
tag, manifest.createdAt, manifest.expiresAt, snapshotDirs,
manifest.ephemeral);
StorageService.instance.addSnapshot(snapshot);
return snapshot;
}
private SnapshotManifest writeSnapshotManifest(SnapshotManifest manifest, File manifestFile)
{
try
{
manifestFile.parent().tryCreateDirectories();
manifest.serializeToJsonFile(manifestFile);
return manifest;
}
catch (IOException e)
{
throw new FSWriteError(e, manifestFile);
}
}
private List<String> mapToDataFilenames(Collection<SSTableReader> sstables)
{
return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Components.DATA)).collect(Collectors.toList());
}
private void writeSnapshotSchema(File schemaFile)
{
try
{
if (!schemaFile.parent().exists())
schemaFile.parent().tryCreateDirectories();
try (PrintStream out = new PrintStream(new FileOutputStreamPlus(schemaFile)))
{
SchemaCQLHelper.reCreateStatementsForSchemaCql(metadata(),
keyspace.getMetadata())
.forEach(out::println);
}
}
catch (IOException e)
{
throw new FSWriteError(e, schemaFile);
}
}
protected static void clearEphemeralSnapshots(Directories directories)
{
RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
List<TableSnapshot> ephemeralSnapshots = new SnapshotLoader(directories).loadSnapshots()
.stream()
.filter(TableSnapshot::isEphemeral)
.collect(Collectors.toList());
for (TableSnapshot ephemeralSnapshot : ephemeralSnapshots)
{
logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot.getId());
Directories.clearSnapshot(ephemeralSnapshot.getTag(), directories.getCFDirectories(), clearSnapshotRateLimiter);
}
}
public Refs<SSTableReader> getSnapshotSSTableReaders(String tag) throws IOException
{
Map<SSTableId, SSTableReader> active = new HashMap<>();
for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
active.put(sstable.descriptor.id, sstable);
Map<Descriptor, Set<Component>> snapshots = getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list();
Refs<SSTableReader> refs = new Refs<>();
try
{
for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
{
// Try acquire reference to an active sstable instead of snapshot if it exists,
// to avoid opening new sstables. If it fails, use the snapshot reference instead.
SSTableReader sstable = active.get(entries.getKey().id);
if (sstable == null || !refs.tryRef(sstable))
{
if (logger.isTraceEnabled())
logger.trace("using snapshot sstable {}", entries.getKey());
// open offline so we don't modify components or track hotness.
sstable = SSTableReader.open(this, entries.getKey(), entries.getValue(), metadata, true, true);
refs.tryRef(sstable);
// release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released
sstable.selfRef().release();
}
else if (logger.isTraceEnabled())
{
logger.trace("using active sstable {}", entries.getKey());
}
}
}
catch (FSReadError | RuntimeException e)
{
// In case one of the snapshot sstables fails to open,
// we must release the references to the ones we opened so far
refs.release();
throw e;
}
return refs;
}
/**
* Take a snap shot of this columnfamily store.
*
* @param snapshotName the name of the associated with the snapshot
*/
public TableSnapshot snapshot(String snapshotName)
{
return snapshot(snapshotName, null);
}
public TableSnapshot snapshot(String snapshotName, DurationSpec.IntSecondsBound ttl)
{
return snapshot(snapshotName, false, ttl, null, now());
}
/**
* Take a snap shot of this columnfamily store.
*
* @param snapshotName the name of the associated with the snapshot
* @param skipMemtable Skip flushing the memtable
* @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed
* @param rateLimiter Rate limiter for hardlinks-per-second
* @param creationTime time when this snapshot was taken
*/
public TableSnapshot snapshot(String snapshotName, boolean skipMemtable, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime)
{
return snapshot(snapshotName, null, false, skipMemtable, ttl, rateLimiter, creationTime);
}
/**
* @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup
* @param skipMemtable Skip flushing the memtable
*/
public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipMemtable)
{
return snapshot(snapshotName, predicate, ephemeral, skipMemtable, null, null, now());
}
/**
* @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup
* @param skipMemtable Skip flushing the memtable
* @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed
* @param rateLimiter Rate limiter for hardlinks-per-second
* @param creationTime time when this snapshot was taken
*/
public TableSnapshot snapshot(String snapshotName, Predicate<SSTableReader> predicate, boolean ephemeral, boolean skipMemtable, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime)
{
if (!skipMemtable)
{
Memtable current = getTracker().getView().getCurrentMemtable();
if (!current.isClean())
{
if (current.shouldSwitch(FlushReason.SNAPSHOT))
FBUtilities.waitOnFuture(switchMemtableIfCurrent(current, FlushReason.SNAPSHOT));
else
current.performSnapshot(snapshotName);
}
}
return snapshotWithoutMemtable(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime);
}
public boolean snapshotExists(String snapshotName)
{
return getDirectories().snapshotExists(snapshotName);
}
/**
* Clear all the snapshots for a given column family.
*
* @param snapshotName the user supplied snapshot name. If left empty,
* all the snapshots will be cleaned.
*/
public void clearSnapshot(String snapshotName)
{
RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
List<File> snapshotDirs = getDirectories().getCFDirectories();
Directories.clearSnapshot(snapshotName, snapshotDirs, clearSnapshotRateLimiter);
}
/**
*
* @return Return a map of all snapshots to space being used
* The pair for a snapshot has true size and size on disk.
*/
public Map<String, TableSnapshot> listSnapshots()
{
return getDirectories().listSnapshots();
}
/**
* @return the cached partition for @param key if it is already present in the cache.
* Not that this will not readAndCache the parition if it is not present, nor
@ -2771,15 +2501,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
// stream in data that is actually supposed to have been deleted
ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id),
"Stopping parent sessions {} due to truncation of tableId="+metadata.id);
data.notifyTruncated(truncatedAt);
data.notifyTruncated(noSnapshot, truncatedAt, DatabaseDescriptor.getAutoSnapshotTtl());
if (!noSnapshot && isAutoSnapshotEnabled())
snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
discardSSTables(truncatedAt);
discardSSTables(truncatedAt);
indexManager.truncateAllIndexesBlocking(truncatedAt);
viewManager.truncateBlocking(replayAfter, truncatedAt);
indexManager.truncateAllIndexesBlocking(truncatedAt);
viewManager.truncateBlocking(replayAfter, truncatedAt);
SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
logger.trace("cleaning out row cache");
@ -3303,10 +3030,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
return allColumns > 0 ? allDroppable / allColumns : 0;
}
public Set<String> getFilesOfCfs()
{
Set<String> files = new HashSet<>();
for (ColumnFamilyStore cfs : concatWithIndexes())
cfs.getTracker().getView().liveSSTables().forEach(s -> files.addAll(s.getAllFilePaths()));
return files;
}
@Override
public long trueSnapshotsSize()
{
return getDirectories().trueSnapshotsSize();
return SnapshotManager.instance.getTrueSnapshotsSize(getKeyspaceName(), getTableName());
}
/**
@ -3426,8 +3163,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true);
if (isAutoSnapshotEnabled())
snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl());
data.notifyDropped(DatabaseDescriptor.getAutoSnapshotTtl());
CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id));

184
src/java/org/apache/cassandra/db/Directories.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -49,8 +48,6 @@ import java.util.stream.StreamSupport;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -74,8 +71,6 @@ import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.DirectorySizeCalculator;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Pair;
@ -306,6 +301,11 @@ public class Directories
canonicalPathToDD = canonicalPathsBuilder.build();
}
public File[] getDataPaths()
{
return dataPaths;
}
/**
* Returns SSTable location which is inside given data directory.
*
@ -673,6 +673,16 @@ public class Directories
}
}
public Set<File> getSnapshotDirs(String tag)
{
Set<File> snapshotDirs = new HashSet<>();
for (File cfDir : getCFDirectories())
snapshotDirs.add(Directories.getSnapshotDirectory(cfDir, tag).toAbsolute());
return snapshotDirs;
}
public File getSnapshotManifestFile(String snapshotName)
{
File snapshotDir = getSnapshotDirectory(getDirectoryForNewSSTables(), snapshotName);
@ -1171,37 +1181,6 @@ public class Directories
}
}
public Map<String, TableSnapshot> listSnapshots()
{
Map<String, Set<File>> snapshotDirsByTag = listSnapshotDirsByTag();
Map<String, TableSnapshot> snapshots = Maps.newHashMapWithExpectedSize(snapshotDirsByTag.size());
for (Map.Entry<String, Set<File>> entry : snapshotDirsByTag.entrySet())
{
String tag = entry.getKey();
Set<File> snapshotDirs = entry.getValue();
SnapshotManifest manifest = maybeLoadManifest(metadata.keyspace, metadata.name, tag, snapshotDirs);
snapshots.put(tag, buildSnapshot(tag, manifest, snapshotDirs));
}
return snapshots;
}
private TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs)
{
boolean ephemeral = manifest != null ? manifest.isEphemeral() : isLegacyEphemeralSnapshot(snapshotDirs);
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
return new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag, createdAt, expiresAt,
snapshotDirs, ephemeral);
}
private static boolean isLegacyEphemeralSnapshot(Set<File> snapshotDirs)
{
return snapshotDirs.stream().map(d -> new File(d, "ephemeral.snapshot")).anyMatch(File::exists);
}
@VisibleForTesting
protected static SnapshotManifest maybeLoadManifest(String keyspace, String table, String tag, Set<File> snapshotDirs)
{
@ -1230,100 +1209,10 @@ public class Directories
return null;
}
@VisibleForTesting
protected Map<String, Set<File>> listSnapshotDirsByTag()
{
Map<String, Set<File>> snapshotDirsByTag = new HashMap<>();
for (final File dir : dataPaths)
{
File snapshotDir = isSecondaryIndexFolder(dir)
? new File(dir.parentPath(), SNAPSHOT_SUBDIR)
: new File(dir, SNAPSHOT_SUBDIR);
if (snapshotDir.exists() && snapshotDir.isDirectory())
{
final File[] snapshotDirs = snapshotDir.tryList();
if (snapshotDirs != null)
{
for (final File snapshot : snapshotDirs)
{
if (snapshot.isDirectory()) {
snapshotDirsByTag.computeIfAbsent(snapshot.name(), k -> new LinkedHashSet<>()).add(snapshot.toAbsolute());
}
}
}
}
}
return snapshotDirsByTag;
}
public boolean snapshotExists(String snapshotName)
{
for (File dir : dataPaths)
{
File snapshotDir;
if (isSecondaryIndexFolder(dir))
{
snapshotDir = new File(dir.parent(), join(SNAPSHOT_SUBDIR, snapshotName, dir.name()));
}
else
{
snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName));
}
if (snapshotDir.exists())
return true;
}
return false;
}
public static void clearSnapshot(String snapshotName, List<File> tableDirectories, RateLimiter snapshotRateLimiter)
{
// If snapshotName is empty or null, we will delete the entire snapshot directory
String tag = snapshotName == null ? "" : snapshotName;
for (File tableDir : tableDirectories)
{
File snapshotDir = new File(tableDir, join(SNAPSHOT_SUBDIR, tag));
removeSnapshotDirectory(snapshotRateLimiter, snapshotDir);
}
}
public static void removeSnapshotDirectory(RateLimiter snapshotRateLimiter, File snapshotDir)
{
if (snapshotDir.exists())
{
logger.trace("Removing snapshot directory {}", snapshotDir);
try
{
FileUtils.deleteRecursiveWithThrottle(snapshotDir, snapshotRateLimiter);
}
catch (RuntimeException ex)
{
if (!snapshotDir.exists())
return; // ignore
throw ex;
}
}
}
/**
* @return total snapshot size in byte for all snapshots.
*/
public long trueSnapshotsSize()
{
long result = 0L;
for (File dir : dataPaths)
{
File snapshotDir = isSecondaryIndexFolder(dir)
? new File(dir.parentPath(), SNAPSHOT_SUBDIR)
: new File(dir, SNAPSHOT_SUBDIR);
result += getTrueAllocatedSizeIn(snapshotDir);
}
return result;
}
/**
* @return Raw size on disk for all directories
*/
public long getRawDiretoriesSize()
public long getRawDirectoriesSize()
{
long totalAllocatedSize = 0L;
@ -1333,25 +1222,7 @@ public class Directories
return totalAllocatedSize;
}
public long getTrueAllocatedSizeIn(File snapshotDir)
{
if (!snapshotDir.isDirectory())
return 0;
SSTableSizeSummer visitor = new SSTableSizeSummer(sstableLister(OnTxnErr.THROW).listFiles());
try
{
Files.walkFileTree(snapshotDir.toPath(), visitor);
}
catch (IOException e)
{
logger.error("Could not calculate the size of {}. {}", snapshotDir, e.getMessage());
}
return visitor.getAllocatedSize();
}
// Recursively finds all the sub directories in the KS directory.
// Recursively finds all the subdirectories in the KS directory.
public static List<File> getKSChildDirectories(String ksName)
{
List<File> result = new ArrayList<>();
@ -1434,25 +1305,4 @@ public class Directories
{
return StringUtils.join(s, File.pathSeparator());
}
private class SSTableSizeSummer extends DirectorySizeCalculator
{
private final Set<String> toSkip;
SSTableSizeSummer(List<File> files)
{
toSkip = files.stream().map(File::name).collect(Collectors.toSet());
}
@Override
public boolean isAcceptable(Path path)
{
File file = new File(path);
Descriptor desc = SSTable.tryDescriptorFromFile(file);
return desc != null
&& desc.ksname.equals(metadata.keyspace)
&& desc.cfname.equals(metadata.name)
&& !toSkip.contains(file.name());
}
}
}

89
src/java/org/apache/cassandra/db/Keyspace.java
View File

@ -17,8 +17,6 @@
*/
package org.apache.cassandra.db;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -35,18 +33,17 @@ import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager;
import org.apache.cassandra.db.view.ViewManager;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
@ -61,7 +58,6 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -74,7 +70,6 @@ import org.apache.cassandra.utils.concurrent.Promise;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime;
/**
@ -226,80 +221,19 @@ public class Keyspace
return columnFamilyStores.containsKey(id);
}
/**
* Take a snapshot of the specific column family, or the entire set of column families
* if columnFamily is null with a given timestamp
*
* @param snapshotName the tag associated with the name of the snapshot. This value may not be null
* @param columnFamilyName the column family to snapshot or all on null
* @param skipFlush Skip blocking flush of memtable
* @param rateLimiter Rate limiter for hardlinks-per-second
* @throws IOException if the column family doesn't exist
*/
public void snapshot(String snapshotName, String columnFamilyName, boolean skipFlush, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime) throws IOException
public static void verifyKeyspaceIsValid(String keyspaceName)
{
assert snapshotName != null;
boolean tookSnapShot = false;
for (ColumnFamilyStore cfStore : columnFamilyStores.values())
{
if (columnFamilyName == null || cfStore.name.equals(columnFamilyName))
{
tookSnapShot = true;
cfStore.snapshot(snapshotName, skipFlush, ttl, rateLimiter, creationTime);
}
}
if (null != VirtualKeyspaceRegistry.instance.getKeyspaceNullable(keyspaceName))
throw new IllegalArgumentException("Cannot perform any operations against virtual keyspace " + keyspaceName);
if ((columnFamilyName != null) && !tookSnapShot)
throw new IOException("Failed taking snapshot. Table " + columnFamilyName + " does not exist.");
if (!Schema.instance.getKeyspaces().contains(keyspaceName))
throw new IllegalArgumentException("Keyspace " + keyspaceName + " does not exist");
}
/**
* Take a snapshot of the specific column family, or the entire set of column families
* if columnFamily is null with a given timestamp
*
* @param snapshotName the tag associated with the name of the snapshot. This value may not be null
* @param columnFamilyName the column family to snapshot or all on null
* @throws IOException if the column family doesn't exist
*/
public void snapshot(String snapshotName, String columnFamilyName) throws IOException
public static Keyspace getValidKeyspace(String keyspaceName)
{
snapshot(snapshotName, columnFamilyName, false, null, null, now());
}
/**
* @param clientSuppliedName may be null.
* @return the name of the snapshot
*/
public static String getTimestampedSnapshotName(String clientSuppliedName)
{
String snapshotName = Long.toString(currentTimeMillis());
if (clientSuppliedName != null && !clientSuppliedName.equals(""))
{
snapshotName = snapshotName + "-" + clientSuppliedName;
}
return snapshotName;
}
public static String getTimestampedSnapshotNameWithPrefix(String clientSuppliedName, String prefix)
{
return prefix + "-" + getTimestampedSnapshotName(clientSuppliedName);
}
/**
* Check whether snapshots already exists for a given name.
*
* @param snapshotName the user supplied snapshot name
* @return true if the snapshot exists
*/
public boolean snapshotExists(String snapshotName)
{
assert snapshotName != null;
for (ColumnFamilyStore cfStore : columnFamilyStores.values())
{
if (cfStore.snapshotExists(snapshotName))
return true;
}
return false;
verifyKeyspaceIsValid(keyspaceName);
return Keyspace.open(keyspaceName);
}
/**
@ -313,11 +247,6 @@ public class Keyspace
return list;
}
public Stream<TableSnapshot> getAllSnapshots()
{
return getColumnFamilyStores().stream().flatMap(cfs -> cfs.listSnapshots().values().stream());
}
public static Keyspace forSchema(String keyspaceName, SchemaProvider schema)
{
return new Keyspace(keyspaceName, schema, true);

27
src/java/org/apache/cassandra/db/SystemKeyspace.java
View File

@ -109,6 +109,9 @@ import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
@ -141,10 +144,10 @@ import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION;
import static org.apache.cassandra.gms.ApplicationState.STATUS_WITH_PORT;
import static org.apache.cassandra.gms.ApplicationState.TOKENS;
import static org.apache.cassandra.service.paxos.Commit.latest;
import static org.apache.cassandra.service.snapshot.SnapshotOptions.systemSnapshot;
import static org.apache.cassandra.utils.CassandraVersion.NULL_VERSION;
import static org.apache.cassandra.utils.CassandraVersion.UNREADABLE_VERSION;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.FBUtilities.now;
public final class SystemKeyspace
{
@ -1808,10 +1811,8 @@ public final class SystemKeyspace
* Compare the release version in the system.local table with the one included in the distro.
* If they don't match, snapshot all tables in the system and schema keyspaces. This is intended
* to be called at startup to create a backup of the system tables during an upgrade
*
* @throws IOException
*/
public static void snapshotOnVersionChange() throws IOException
public static void snapshotOnVersionChange()
{
String previous = getPreviousVersionString();
String next = FBUtilities.getReleaseVersionString();
@ -1820,16 +1821,18 @@ public final class SystemKeyspace
// if we're restarting after an upgrade, snapshot the system and schema keyspaces
if (!previous.equals(NULL_VERSION.toString()) && !previous.equals(next))
{
logger.info("Detected version upgrade from {} to {}, snapshotting system keyspaces", previous, next);
String snapshotName = Keyspace.getTimestampedSnapshotName(format("upgrade-%s-%s",
previous,
next));
Instant creationTime = now();
List<String> entities = new ArrayList<>();
for (String keyspace : SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES)
Keyspace.open(keyspace).snapshot(snapshotName, null, false, null, null, creationTime);
{
for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores())
entities.add(cfs.getKeyspaceTableName());
}
logger.info("Detected version upgrade from {} to {}, snapshotting system keyspaces", previous, next);
SnapshotOptions options = systemSnapshot(format("%s-%s", previous, next), SnapshotType.UPGRADE, entities.toArray(new String[0])).build();
SnapshotManager.instance.takeSnapshot(options);
}
}

9
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.cassandra.db.compaction;
import java.time.Instant;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@ -46,6 +45,9 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Refs;
@ -53,7 +55,6 @@ import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
import static org.apache.cassandra.utils.FBUtilities.now;
public class CompactionTask extends AbstractCompactionTask
{
@ -126,8 +127,8 @@ public class CompactionTask extends AbstractCompactionTask
if (DatabaseDescriptor.isSnapshotBeforeCompaction())
{
Instant creationTime = now();
cfs.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime);
SnapshotOptions options = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.COMPACT, cfs.getKeyspaceTableName()).skipFlush().build();
SnapshotManager.instance.takeSnapshot(options);
}
try (CompactionController controller = getCompactionController(transaction.originals()))

34
src/java/org/apache/cassandra/db/lifecycle/Tracker.java
View File

@ -35,6 +35,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
@ -56,6 +57,8 @@ import org.apache.cassandra.notifications.SSTableDeletingNotification;
import org.apache.cassandra.notifications.SSTableListChangedNotification;
import org.apache.cassandra.notifications.SSTableMetadataChanged;
import org.apache.cassandra.notifications.SSTableRepairStatusChanged;
import org.apache.cassandra.notifications.TableDroppedNotification;
import org.apache.cassandra.notifications.TablePreScrubNotification;
import org.apache.cassandra.notifications.TruncationNotification;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
@ -510,31 +513,34 @@ public class Tracker
{
if (repairStatusesChanged.isEmpty())
return;
INotification notification = new SSTableRepairStatusChanged(repairStatusesChanged);
for (INotificationConsumer subscriber : subscribers)
subscriber.handleNotification(notification, this);
notify(new SSTableRepairStatusChanged(repairStatusesChanged));
}
public void notifySSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata)
{
INotification notification = new SSTableMetadataChanged(levelChanged, oldMetadata);
for (INotificationConsumer subscriber : subscribers)
subscriber.handleNotification(notification, this);
notify(new SSTableMetadataChanged(levelChanged, oldMetadata));
}
public void notifyDeleting(SSTableReader deleting)
{
INotification notification = new SSTableDeletingNotification(deleting);
for (INotificationConsumer subscriber : subscribers)
subscriber.handleNotification(notification, this);
notify(new SSTableDeletingNotification(deleting));
}
public void notifyTruncated(long truncatedAt)
public void notifyTruncated(boolean disableSnapshot,
long truncatedAt,
DurationSpec.IntSecondsBound ttl)
{
INotification notification = new TruncationNotification(truncatedAt);
for (INotificationConsumer subscriber : subscribers)
subscriber.handleNotification(notification, this);
notify(new TruncationNotification(cfstore, disableSnapshot, truncatedAt, ttl));
}
public void notifyDropped(DurationSpec.IntSecondsBound ttl)
{
notify(new TableDroppedNotification(cfstore, ttl));
}
public void notifyPreScrubbed()
{
notify(new TablePreScrubNotification(cfstore));
}
public void notifyRenewed(Memtable renewed)

23
src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java
View File

@ -22,8 +22,7 @@ import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import com.google.common.base.Predicate;
import java.util.function.Predicate;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionManager;
@ -36,6 +35,9 @@ import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.repair.TableRepairManager;
import org.apache.cassandra.repair.ValidationPartitionIterator;
import org.apache.cassandra.repair.NoSuchRepairSessionException;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.service.ActiveRepairService;
@ -79,17 +81,14 @@ public class CassandraTableRepairManager implements TableRepairManager
try
{
ActiveRepairService.instance().snapshotExecutor.submit(() -> {
if (force || !cfs.snapshotExists(name))
if (force || !SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), name))
{
cfs.snapshot(name, new Predicate<SSTableReader>()
{
public boolean apply(SSTableReader sstable)
{
return sstable != null &&
!sstable.metadata().isIndex() && // exclude SSTables from 2i
new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges);
}
}, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup
Predicate<SSTableReader> predicate = sstable -> sstable != null &&
!sstable.metadata().isIndex() && // exclude SSTables from 2i
new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges);
SnapshotOptions options = SnapshotOptions.systemSnapshot(name, SnapshotType.REPAIR, predicate, cfs.getKeyspaceTableName()).ephemeral().build();
SnapshotManager.instance.takeSnapshot(options);
}
}).get();
}

10
src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java
View File

@ -57,6 +57,8 @@ import org.apache.cassandra.repair.ValidationPartitionIterator;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.repair.NoSuchRepairSessionException;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Refs;
@ -179,19 +181,19 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
this.cfs = cfs;
this.ctx = ctx;
isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString());
isGlobalSnapshotValidation = SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), parentId.toString());
if (isGlobalSnapshotValidation)
snapshotName = parentId.toString();
else
snapshotName = sessionID.toString();
isSnapshotValidation = cfs.snapshotExists(snapshotName);
isSnapshotValidation = SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), snapshotName);
if (isSnapshotValidation)
{
// If there is a snapshot created for the session then read from there.
// note that we populate the parent repair session when creating the snapshot, meaning the sstables in the snapshot are the ones we
// are supposed to validate.
sstables = cfs.getSnapshotSSTableReaders(snapshotName);
sstables = TableSnapshot.getSnapshotSSTableReaders(cfs, snapshotName);
}
else
{
@ -270,7 +272,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
{
// we can only clear the snapshot if we are not doing a global snapshot validation (we then clear it once anticompaction
// is done).
cfs.clearSnapshot(snapshotName);
SnapshotManager.instance.clearSnapshot(cfs.getKeyspaceName(), cfs.getTableName(), snapshotName);
}
if (sstables != null)

4
src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java
View File

@ -26,7 +26,7 @@ import org.apache.cassandra.db.marshal.TimestampType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
public class SnapshotsTable extends AbstractVirtualTable
@ -62,7 +62,7 @@ public class SnapshotsTable extends AbstractVirtualTable
{
SimpleDataSet result = new SimpleDataSet(metadata());
for (TableSnapshot tableSnapshot : StorageService.instance.snapshotManager.loadSnapshots())
for (TableSnapshot tableSnapshot : SnapshotManager.instance.getSnapshots(false, true))
{
SimpleDataSet row = result.row(tableSnapshot.getTag(),
tableSnapshot.getKeyspaceName(),

34
src/java/org/apache/cassandra/notifications/TableDroppedNotification.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.notifications;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ColumnFamilyStore;
public class TableDroppedNotification implements INotification
{
public final ColumnFamilyStore cfs;
public final DurationSpec.IntSecondsBound ttl;
public TableDroppedNotification(ColumnFamilyStore cfs, DurationSpec.IntSecondsBound ttl)
{
this.cfs = cfs;
this.ttl = ttl;
}
}

31
src/java/org/apache/cassandra/notifications/TablePreScrubNotification.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.notifications;
import org.apache.cassandra.db.ColumnFamilyStore;
public class TablePreScrubNotification implements INotification
{
public final ColumnFamilyStore cfs;
public TablePreScrubNotification(ColumnFamilyStore cfs)
{
this.cfs = cfs;
}
}

14
src/java/org/apache/cassandra/notifications/TruncationNotification.java
View File

@ -17,16 +17,28 @@
*/
package org.apache.cassandra.notifications;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ColumnFamilyStore;
/**
* Fired during truncate, after the memtable has been flushed but before any
* snapshot is taken and SSTables are discarded
*/
public class TruncationNotification implements INotification
{
public final ColumnFamilyStore cfs;
public final boolean disableSnapshot;
public final long truncatedAt;
public final DurationSpec.IntSecondsBound ttl;
public TruncationNotification(long truncatedAt)
public TruncationNotification(ColumnFamilyStore cfs,
boolean disableSnapshot,
long truncatedAt,
DurationSpec.IntSecondsBound ttl)
{
this.cfs = cfs;
this.disableSnapshot = disableSnapshot;
this.truncatedAt = truncatedAt;
this.ttl = ttl;
}
}

3
src/java/org/apache/cassandra/repair/PreviewRepairTask.java
View File

@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.RepairMetrics;
import org.apache.cassandra.repair.consistent.SyncStatSummary;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
import org.apache.cassandra.utils.TimeUUID;
@ -128,7 +129,7 @@ public class PreviewRepairTask extends AbstractRepairTask
for (String table : mismatchingTables)
{
// we can just check snapshot existence locally since the repair coordinator is always a replica (unlike in the read case)
if (!Keyspace.open(keyspace).getColumnFamilyStore(table).snapshotExists(snapshotName))
if (!SnapshotManager.instance.exists(keyspace, table, snapshotName))
{
List<Range<Token>> normalizedRanges = Range.normalize(ranges);
logger.info("{} Snapshotting {}.{} for preview repair mismatch for ranges {} with tag {} on instances {}",

7
src/java/org/apache/cassandra/service/ActiveRepairService.java
View File

@ -106,6 +106,7 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.paxos.PaxosRepair;
import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ExecutorUtils;
@ -884,10 +885,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
.map(cfs -> cfs.metadata().toString()).collect(Collectors.joining(", ")));
long startNanos = ctx.clock().nanoTime();
for (ColumnFamilyStore cfs : session.columnFamilyStores.values())
{
if (cfs.snapshotExists(snapshotName))
cfs.clearSnapshot(snapshotName);
}
SnapshotManager.instance.clearSnapshot(cfs.keyspace.getName(), cfs.name, snapshotName);
logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(ctx.clock().nanoTime() - startNanos));
});
}

13
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -71,6 +71,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.security.ThreadAwareSecurityManager;
import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.tcm.CMSOperations;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -266,8 +267,16 @@ public class CassandraDaemon
DatabaseDescriptor.createAllDirectories();
Keyspace.setInitialized();
CommitLog.instance.start();
runStartupChecks();
SnapshotManager.instance.start(false);
SnapshotManager.instance.clearExpiredSnapshots();
SnapshotManager.instance.clearEphemeralSnapshots();
SnapshotManager.instance.resumeSnapshotCleanup();
SnapshotManager.instance.registerMBean();
// clearing of snapshots above here will in fact clear all ephemeral snapshots
// which were cleared as part of startup checks before CASSANDRA-18111
runStartupChecks();
try
{
@ -291,7 +300,7 @@ public class CassandraDaemon
{
SystemKeyspace.snapshotOnVersionChange();
}
catch (IOException e)
catch (Throwable e)
{
exitOrFail(StartupException.ERR_WRONG_DISK_STATE, e.getMessage(), e.getCause());
}

16
src/java/org/apache/cassandra/service/SnapshotVerbHandler.java
View File

@ -20,11 +20,13 @@ package org.apache.cassandra.service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SnapshotCommand;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.utils.DiagnosticSnapshotService;
@ -38,7 +40,7 @@ public class SnapshotVerbHandler implements IVerbHandler<SnapshotCommand>
SnapshotCommand command = message.payload;
if (command.clear_snapshot)
{
StorageService.instance.clearSnapshot(command.snapshot_name, command.keyspace);
SnapshotManager.instance.clearSnapshots(command.snapshot_name, command.keyspace);
}
else if (DiagnosticSnapshotService.isDiagnosticSnapshotRequest(command))
{
@ -46,7 +48,15 @@ public class SnapshotVerbHandler implements IVerbHandler<SnapshotCommand>
}
else
{
Keyspace.open(command.keyspace).getColumnFamilyStore(command.column_family).snapshot(command.snapshot_name);
try
{
SnapshotOptions options = SnapshotOptions.systemSnapshot(command.snapshot_name, SnapshotType.DIAGNOSTICS, command.keyspace + '.' + command.column_family).build();
SnapshotManager.instance.takeSnapshot(options);
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
}
logger.debug("Enqueuing response to snapshot request {} to {}", command.snapshot_name, message.from());

388
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -22,8 +22,6 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -59,7 +57,6 @@ import javax.management.NotificationListener;
import javax.management.openmbean.CompositeData;
import javax.management.openmbean.OpenDataException;
import javax.management.openmbean.TabularData;
import javax.management.openmbean.TabularDataSupport;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
@ -72,7 +69,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.RateLimiter;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@ -96,21 +92,18 @@ import org.apache.cassandra.config.Config.PaxosStatePurging;
import org.apache.cassandra.config.Converters;
import org.apache.cassandra.config.DataStorageSpec;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.cql3.QueryHandler;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SizeEstimatesRecorder;
import org.apache.cassandra.db.SnapshotDetailsTabularData;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.dht.BootStrapper;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.OwnedRanges;
@ -179,7 +172,6 @@ import org.apache.cassandra.service.paxos.PaxosState;
import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupLocalCoordinator;
import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamState;
@ -208,7 +200,6 @@ import org.apache.cassandra.tcm.transformations.Startup;
import org.apache.cassandra.tcm.transformations.Unregister;
import org.apache.cassandra.transport.ClientResourceLimits;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -263,7 +254,6 @@ import static org.apache.cassandra.tcm.membership.NodeState.MOVING;
import static org.apache.cassandra.tcm.membership.NodeState.REGISTERED;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
import static org.apache.cassandra.utils.FBUtilities.now;
/**
* This abstraction contains the token/identifier of this node
@ -312,8 +302,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
private final List<Runnable> preShutdownHooks = new ArrayList<>();
private final List<Runnable> postShutdownHooks = new ArrayList<>();
public final SnapshotManager snapshotManager = new SnapshotManager();
public static final StorageService instance = new StorageService();
private final SamplingManager samplingManager = new SamplingManager();
@ -884,7 +872,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
DiskUsageBroadcaster.instance.startBroadcasting();
HintsService.instance.startDispatch();
BatchlogManager.instance.start();
startSnapshotManager();
servicesInitialized = true;
}
@ -925,12 +912,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return DatabaseDescriptor.getSeeds().contains(getBroadcastAddressAndPort());
}
@VisibleForTesting
public void startSnapshotManager()
{
snapshotManager.start();
}
public static boolean isReplacingSameAddress()
{
InetAddressAndPort replaceAddress = DatabaseDescriptor.getReplaceAddress();
@ -2742,55 +2723,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return status.statusCode;
}
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag
* the tag given to the snapshot; may not be null or empty
* @param options
* Map of options (skipFlush is the only supported option for now)
* @param entities
* list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
*/
@Override
public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException
{
DurationSpec.IntSecondsBound ttl = options.containsKey("ttl") ? new DurationSpec.IntSecondsBound(options.get("ttl")) : null;
if (ttl != null)
{
int minAllowedTtlSecs = CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.getInt();
if (ttl.toSeconds() < minAllowedTtlSecs)
throw new IllegalArgumentException(String.format("ttl for snapshot must be at least %d seconds", minAllowedTtlSecs));
}
boolean skipFlush = Boolean.parseBoolean(options.getOrDefault("skipFlush", "false"));
if (entities != null && entities.length > 0 && entities[0].contains("."))
{
takeMultipleTableSnapshot(tag, skipFlush, ttl, entities);
}
else
{
takeSnapshot(tag, skipFlush, ttl, entities);
}
}
/**
* Takes the snapshot of a specific table. A snapshot name must be
* specified.
*
* @param keyspaceName
* the keyspace which holds the specified table
* @param tableName
* the table to snapshot
* @param tag
* the tag given to the snapshot; may not be null or empty
*/
public void takeTableSnapshot(String keyspaceName, String tableName, String tag)
throws IOException
{
takeMultipleTableSnapshot(tag, false, null, keyspaceName + "." + tableName);
}
@Override
public void forceKeyspaceCompactionForTokenRange(String keyspaceName, String startToken, String endToken, String... tableNames) throws IOException, ExecutionException, InterruptedException
{
@ -2840,296 +2772,126 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public void forceCompactionKeysIgnoringGcGrace(String keyspaceName,
String tableName, String... partitionKeysIgnoreGcGrace) throws IOException, ExecutionException, InterruptedException
{
ColumnFamilyStore cfStore = getValidKeyspace(keyspaceName).getColumnFamilyStore(tableName);
ColumnFamilyStore cfStore = Keyspace.getValidKeyspace(keyspaceName).getColumnFamilyStore(tableName);
cfStore.forceCompactionKeysIgnoringGcGrace(partitionKeysIgnoreGcGrace);
}
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
* @param options Map of options (skipFlush is the only supported option for now)
* @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
*
* @deprecated See CASSANDRA-18111
*/
@Override
@Deprecated(since = "5.1")
public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException
{
SnapshotManager.instance.takeSnapshot(tag, options, entities);
}
/**
* Takes the snapshot of a specific table. A snapshot name must be
* specified.
*
* @param keyspaceName the keyspace which holds the specified table
* @param tableName the table to snapshot
* @param tag the tag given to the snapshot; may not be null or empty
*
* @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907
*/
@Override
@Deprecated(since = "3.4")
public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException
{
SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), keyspaceName + '.' + tableName);
}
/**
* Takes the snapshot for the given keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
* @param keyspaceNames the names of the keyspaces to snapshot; empty means "all."
*
* @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907
*/
@Override
@Deprecated(since = "3.4")
public void takeSnapshot(String tag, String... keyspaceNames) throws IOException
{
takeSnapshot(tag, false, null, keyspaceNames);
SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), keyspaceNames);
}
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag
* the tag given to the snapshot; may not be null or empty
* @param tableList
* list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
*/
public void takeMultipleTableSnapshot(String tag, String... tableList)
throws IOException
{
takeMultipleTableSnapshot(tag, false, null, tableList);
}
/**
* Takes the snapshot for the given keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
* @param skipFlush Skip blocking flush of memtable
* @param keyspaceNames the names of the keyspaces to snapshot; empty means "all."
*/
private void takeSnapshot(String tag, boolean skipFlush, DurationSpec.IntSecondsBound ttl, String... keyspaceNames) throws IOException
{
if (operationMode() == Mode.JOINING)
throw new IOException("Cannot snapshot until bootstrap completes");
if (tag == null || tag.equals(""))
throw new IOException("You must supply a snapshot name.");
Iterable<Keyspace> keyspaces;
if (keyspaceNames.length == 0)
{
keyspaces = Keyspace.all();
}
else
{
ArrayList<Keyspace> t = new ArrayList<>(keyspaceNames.length);
for (String keyspaceName : keyspaceNames)
t.add(getValidKeyspace(keyspaceName));
keyspaces = t;
}
// Do a check to see if this snapshot exists before we actually snapshot
for (Keyspace keyspace : keyspaces)
if (keyspace.snapshotExists(tag))
throw new IOException("Snapshot " + tag + " already exists.");
RateLimiter snapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
Instant creationTime = now();
for (Keyspace keyspace : keyspaces)
{
keyspace.snapshot(tag, null, skipFlush, ttl, snapshotRateLimiter, creationTime);
}
}
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
* @param tableList list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
*
*
* @param tag
* the tag given to the snapshot; may not be null or empty
* @param skipFlush
* Skip blocking flush of memtable
* @param tableList
* list of tables from different keyspace in the form of ks1.cf1 ks2.cf2
* @deprecated See CASSANDRA-10907
*/
private void takeMultipleTableSnapshot(String tag, boolean skipFlush, DurationSpec.IntSecondsBound ttl, String... tableList)
throws IOException
@Override
@Deprecated(since = "3.4")
public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException
{
Map<Keyspace, List<String>> keyspaceColumnfamily = new HashMap<Keyspace, List<String>>();
for (String table : tableList)
{
String splittedString[] = StringUtils.split(table, '.');
if (splittedString.length == 2)
{
String keyspaceName = splittedString[0];
String tableName = splittedString[1];
if (keyspaceName == null)
throw new IOException("You must supply a keyspace name");
if (operationMode() == Mode.JOINING)
throw new IOException("Cannot snapshot until bootstrap completes");
if (tableName == null)
throw new IOException("You must supply a table name");
if (tag == null || tag.equals(""))
throw new IOException("You must supply a snapshot name.");
Keyspace keyspace = getValidKeyspace(keyspaceName);
ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName);
// As there can be multiple column family from same keyspace check if snapshot exist for that specific
// columnfamily and not for whole keyspace
if (columnFamilyStore.snapshotExists(tag))
throw new IOException("Snapshot " + tag + " already exists.");
if (!keyspaceColumnfamily.containsKey(keyspace))
{
keyspaceColumnfamily.put(keyspace, new ArrayList<String>());
}
// Add Keyspace columnfamily to map in order to support atomicity for snapshot process.
// So no snapshot should happen if any one of the above conditions fail for any keyspace or columnfamily
keyspaceColumnfamily.get(keyspace).add(tableName);
}
else
{
throw new IllegalArgumentException(
"Cannot take a snapshot on secondary index or invalid column family name. You must supply a column family name in the form of keyspace.columnfamily");
}
}
RateLimiter snapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
Instant creationTime = now();
for (Entry<Keyspace, List<String>> entry : keyspaceColumnfamily.entrySet())
{
for (String table : entry.getValue())
entry.getKey().snapshot(tag, table, skipFlush, ttl, snapshotRateLimiter, creationTime);
}
}
private void verifyKeyspaceIsValid(String keyspaceName)
{
if (null != VirtualKeyspaceRegistry.instance.getKeyspaceNullable(keyspaceName))
throw new IllegalArgumentException("Cannot perform any operations against virtual keyspace " + keyspaceName);
if (!Schema.instance.getKeyspaces().contains(keyspaceName))
throw new IllegalArgumentException("Keyspace " + keyspaceName + " does not exist");
}
private Keyspace getValidKeyspace(String keyspaceName)
{
verifyKeyspaceIsValid(keyspaceName);
return Keyspace.open(keyspaceName);
SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), tableList);
}
/**
* Remove the snapshot with the given name from the given keyspaces.
* If no tag is specified we will remove all snapshots.
*
* @deprecated See CASSANDRA-16860
*/
public void clearSnapshot(String tag, String... keyspaceNames)
@Override
@Deprecated(since = "5.0")
public void clearSnapshot(String tag, String... keyspaceNames) throws IOException
{
clearSnapshot(Collections.emptyMap(), tag, keyspaceNames);
SnapshotManager.instance.clearSnapshot(tag, Collections.emptyMap(), keyspaceNames);
}
@Override
@Deprecated(since = "5.1")
public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaceNames)
{
if (tag == null)
tag = "";
if (options == null)
options = Collections.emptyMap();
Set<String> keyspaces = new HashSet<>();
for (String dataDir : DatabaseDescriptor.getAllDataFileLocations())
{
for (String keyspaceDir : new File(dataDir).tryListNames())
{
// Only add a ks if it has been specified as a param, assuming params were actually provided.
if (keyspaceNames.length > 0 && !Arrays.asList(keyspaceNames).contains(keyspaceDir))
continue;
keyspaces.add(keyspaceDir);
}
}
Object olderThan = options.get("older_than");
Object olderThanTimestamp = options.get("older_than_timestamp");
final long clearOlderThanTimestamp;
if (olderThan != null)
{
assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String";
clearOlderThanTimestamp = Clock.Global.currentTimeMillis() - new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds();
}
else if (olderThanTimestamp != null)
{
assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String";
try
{
clearOlderThanTimestamp = Instant.parse((String) olderThanTimestamp).toEpochMilli();
}
catch (DateTimeParseException ex)
{
throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format.");
}
}
else
clearOlderThanTimestamp = 0L;
for (String keyspace : keyspaces)
clearKeyspaceSnapshot(keyspace, tag, clearOlderThanTimestamp);
if (logger.isDebugEnabled())
logger.debug("Cleared out snapshot directories");
}
/**
* Clear snapshots for a given keyspace.
* @param keyspace keyspace to remove snapshots for
* @param tag the user supplied snapshot name. If empty or null, all the snapshots will be cleaned
* @param olderThanTimestamp if a snapshot was created before this timestamp, it will be cleared,
* if its value is 0, this parameter is effectively ignored.
*/
private void clearKeyspaceSnapshot(String keyspace, String tag, long olderThanTimestamp)
{
Set<TableSnapshot> snapshotsToClear = snapshotManager.loadSnapshots(keyspace)
.stream()
.filter(TableSnapshot.shouldClearSnapshot(tag, olderThanTimestamp))
.collect(Collectors.toSet());
for (TableSnapshot snapshot : snapshotsToClear)
snapshotManager.clearSnapshot(snapshot);
SnapshotManager.instance.clearSnapshot(tag, options, keyspaceNames);
}
@Override
@Deprecated(since = "5.1")
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
{
boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false"));
Map<String, TabularData> snapshotMap = new HashMap<>();
for (TableSnapshot snapshot : snapshotManager.loadSnapshots())
{
if (skipExpiring && snapshot.isExpiring())
continue;
if (!includeEphemeral && snapshot.isEphemeral())
continue;
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag());
if (data == null)
{
data = new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE);
snapshotMap.put(snapshot.getTag(), data);
}
SnapshotDetailsTabularData.from(snapshot, data);
}
return snapshotMap;
return SnapshotManager.instance.listSnapshots(options);
}
/** @deprecated See CASSANDRA-16789 */
@Override
@Deprecated(since = "4.1")
public Map<String, TabularData> getSnapshotDetails()
{
return getSnapshotDetails(ImmutableMap.of());
return SnapshotManager.instance.listSnapshots(ImmutableMap.of());
}
@Override
@Deprecated(since = "5.1")
public long trueSnapshotsSize()
{
long total = 0;
for (Keyspace keyspace : Keyspace.all())
{
if (isLocalSystemKeyspace(keyspace.getName()))
continue;
for (ColumnFamilyStore cfStore : keyspace.getColumnFamilyStores())
{
total += cfStore.trueSnapshotsSize();
}
}
return total;
return SnapshotManager.instance.getTrueSnapshotSize();
}
@Override
@Deprecated(since = "5.1")
public void setSnapshotLinksPerSecond(long throttle)
{
logger.info("Setting snapshot throttle to {}", throttle);
DatabaseDescriptor.setSnapshotLinksPerSecond(throttle);
SnapshotManager.instance.setSnapshotLinksPerSecond(throttle);
}
@Override
@Deprecated(since = "5.1")
public long getSnapshotLinksPerSecond()
{
return DatabaseDescriptor.getSnapshotLinksPerSecond();
return SnapshotManager.instance.getSnapshotLinksPerSecond();
}
public void refreshSizeEstimates() throws ExecutionException
@ -3152,7 +2914,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
*/
public Iterable<ColumnFamilyStore> getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames)
{
Keyspace keyspace = getValidKeyspace(keyspaceName);
Keyspace keyspace = Keyspace.getValidKeyspace(keyspaceName);
return keyspace.getValidColumnFamilies(allowIndexes, autoAddIndexes, cfNames);
}
@ -3944,7 +3706,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
logger.error("Batchlog manager timed out shutting down", t);
}
snapshotManager.stop();
SnapshotManager.instance.close();
HintsService.instance.pauseDispatch();
if (daemon != null)
@ -4038,7 +3800,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
FBUtilities.waitOnFutures(flushes);
SnapshotManager.shutdownAndWait(1L, MINUTES);
SnapshotManager.instance.shutdownAndWait(1L, MINUTES);
HintsService.instance.shutdownBlocking();
// Interrupt ongoing compactions and shutdown CM to prevent further compactions.
@ -4167,7 +3929,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public void truncate(String keyspace, String table) throws TimeoutException, IOException
{
verifyKeyspaceIsValid(keyspace);
Keyspace.verifyKeyspaceIsValid(keyspace);
try
{
@ -4537,7 +4299,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
if (!isInitialized())
throw new RuntimeException("Not yet initialized, can't load new sstables");
verifyKeyspaceIsValid(ksName);
Keyspace.verifyKeyspaceIsValid(ksName);
ColumnFamilyStore.loadNewSSTables(ksName, cfName);
}
@ -5240,10 +5002,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return Math.toIntExact(Guardrails.instance.getPartitionTombstonesWarnThreshold());
}
public void addSnapshot(TableSnapshot snapshot) {
snapshotManager.addSnapshot(snapshot);
}
@Override
public boolean getReadThresholdsEnabled()
{

25
src/java/org/apache/cassandra/service/StorageServiceMBean.java
View File

@ -273,7 +273,9 @@ public interface StorageServiceMBean extends NotificationEmitter
public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException;
/**
* @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907
* Use {@link #takeSnapshot(String tag, Map options, String... entities)} instead.
*
* @deprecated See CASSANDRA-10907
*/
@Deprecated(since = "3.4")
public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException;
@ -281,13 +283,12 @@ public interface StorageServiceMBean extends NotificationEmitter
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag
* the tag given to the snapshot; may not be null or empty
* @param options
* Map of options (skipFlush is the only supported option for now)
* @param entities
* list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
* @param tag the tag given to the snapshot; may not be null or empty
* @param options Map of options (skipFlush is the only supported option for now)
* @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException;
/**
@ -308,7 +309,9 @@ public interface StorageServiceMBean extends NotificationEmitter
* @param options map of options for cleanup operation, consult nodetool's ClearSnapshot
* @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared
* @param keyspaceNames name of keyspaces to clear snapshots for
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaceNames) throws IOException;
/**
@ -324,13 +327,17 @@ public interface StorageServiceMBean extends NotificationEmitter
*
* @param options map of options used for filtering of snapshots
* @return A map of snapshotName to all its details in Tabular form.
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options);
/**
* Get the true size taken by all snapshots across all keyspaces.
* @return True size taken by all the snapshots.
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public long trueSnapshotsSize();
/**
@ -338,7 +345,9 @@ public interface StorageServiceMBean extends NotificationEmitter
* A setting of zero indicates no throttling
*
* @param throttle
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public void setSnapshotLinksPerSecond(long throttle);
/**
@ -346,7 +355,9 @@ public interface StorageServiceMBean extends NotificationEmitter
* A setting of zero indicates no throttling.
*
* @return snapshot links-per-second throttle
* @deprecated See CASSANDRA-18111
*/
@Deprecated(since = "5.1")
public long getSnapshotLinksPerSecond();
/**

42
src/java/org/apache/cassandra/service/snapshot/AbstractSnapshotTask.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.util.concurrent.Callable;
public abstract class AbstractSnapshotTask<T> implements Callable<T>
{
public enum SnapshotTaskType
{
SNAPSHOT,
CLEAR,
RELOAD,
LIST,
SIZE
}
protected final SnapshotOptions options;
public AbstractSnapshotTask(SnapshotOptions options)
{
this.options = options;
}
public abstract SnapshotTaskType getTaskType();
}

196
src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java Normal file
View File

@ -0,0 +1,196 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Clock;
public class ClearSnapshotTask extends AbstractSnapshotTask<Void>
{
private static final Logger logger = LoggerFactory.getLogger(ClearSnapshotTask.class);
private final SnapshotManager manager;
private final Predicate<TableSnapshot> predicateForToBeCleanedSnapshots;
private final boolean deleteData;
public ClearSnapshotTask(SnapshotManager manager,
Predicate<TableSnapshot> predicateForToBeCleanedSnapshots,
boolean deleteData)
{
super(null);
this.manager = manager;
this.predicateForToBeCleanedSnapshots = predicateForToBeCleanedSnapshots;
this.deleteData = deleteData;
}
@Override
public SnapshotTaskType getTaskType()
{
return SnapshotTaskType.CLEAR;
}
@Override
public Void call()
{
Set<TableSnapshot> toRemove = new HashSet<>();
for (TableSnapshot snapshot : new GetSnapshotsTask(manager, predicateForToBeCleanedSnapshots, false).call())
{
logger.debug("Removing snapshot {}{}", snapshot, deleteData ? ", deleting data" : "");
toRemove.add(snapshot);
if (deleteData)
{
for (File snapshotDir : snapshot.getDirectories())
{
try
{
removeSnapshotDirectory(snapshotDir);
}
catch (Throwable ex)
{
logger.warn("Unable to remove snapshot directory {}", snapshotDir, ex);
}
}
}
}
manager.getSnapshots().removeAll(toRemove);
return null;
}
/**
* Returns predicate which will pass the test when arguments match.
*
* @param tag name of snapshot
* @param options options for filtering
* @param keyspaceNames names of keyspaces a snapshot is supposed to be from
* @return predicate which will pass the test when arguments match.
*/
static Predicate<TableSnapshot> getPredicateForCleanedSnapshots(String tag, Map<String, Object> options, String... keyspaceNames)
{
if (options == null)
options = Collections.emptyMap();
Object olderThan = options.get("older_than");
Object olderThanTimestamp = options.get("older_than_timestamp");
long maxCreatedAt = Clock.Global.currentTimeMillis();
if (olderThan != null)
{
assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String";
maxCreatedAt -= new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds();
}
else if (olderThanTimestamp != null)
{
assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String";
try
{
maxCreatedAt = Instant.parse((String) olderThanTimestamp).toEpochMilli();
}
catch (DateTimeParseException ex)
{
throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format.");
}
}
return getClearSnapshotPredicate(tag, Set.of(keyspaceNames), maxCreatedAt, false);
}
/**
* Returns a predicate based on which a snapshot will be included for deletion or not.
*
* @param tag name of snapshot to remove
* @param keyspaces keyspaces this snapshot belongs to
* @param olderThanTimestamp clear the snapshot if it is older than given timestamp
* @param includeEphemeral whether to include ephemeral snapshots as well
* @return predicate which filters snapshots on given parameters
*/
static Predicate<TableSnapshot> getClearSnapshotPredicate(String tag,
Set<String> keyspaces,
long olderThanTimestamp,
boolean includeEphemeral)
{
return ts ->
{
// When no tag is supplied, all snapshots must be cleared
boolean clearAll = tag == null || tag.isEmpty();
if (!clearAll && ts.isEphemeral() && !includeEphemeral)
logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " +
"Ephemeral snapshots are not removable by a user.",
tag, ts.getKeyspaceName());
boolean passedEphemeralTest = !ts.isEphemeral() || (ts.isEphemeral() && includeEphemeral);
boolean shouldClearTag = clearAll || ts.getTag().equals(tag);
boolean byTimestamp = true;
if (olderThanTimestamp > 0L)
{
Instant createdAt = ts.getCreatedAt();
if (createdAt != null)
byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp));
}
boolean byKeyspace = (keyspaces.isEmpty() || keyspaces.contains(ts.getKeyspaceName()));
return passedEphemeralTest && shouldClearTag && byTimestamp && byKeyspace;
};
}
private void removeSnapshotDirectory(File snapshotDir)
{
if (snapshotDir.exists())
{
logger.trace("Removing snapshot directory {}", snapshotDir);
try
{
snapshotDir.deleteRecursive(DatabaseDescriptor.getSnapshotRateLimiter());
}
catch (RuntimeException ex)
{
if (!snapshotDir.exists())
return; // ignore
throw ex;
}
}
}
@Override
public String toString()
{
return "ClearSnapshotTask{" +
"deleteData=" + deleteData +
'}';
}
}

99
src/java/org/apache/cassandra/service/snapshot/GetSnapshotsTask.java Normal file
View File

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
public class GetSnapshotsTask implements Callable<List<TableSnapshot>>
{
private final SnapshotManager snapshotManager;
private final Predicate<TableSnapshot> predicate;
private final boolean shouldRemoveIfNotExists;
public GetSnapshotsTask(SnapshotManager snapshotManager,
Predicate<TableSnapshot> predicate,
boolean shouldRemoveIfNotExists)
{
this.snapshotManager = snapshotManager;
this.predicate = predicate;
this.shouldRemoveIfNotExists = shouldRemoveIfNotExists;
}
@Override
public List<TableSnapshot> call()
{
if (shouldRemoveIfNotExists)
return getWithRemoval();
else
return getWithoutRemoval();
}
private List<TableSnapshot> getWithRemoval()
{
List<TableSnapshot> notExistingAnymore = new ArrayList<>();
List<TableSnapshot> snapshots = new ArrayList<>();
for (TableSnapshot snapshot : snapshotManager.getSnapshots())
{
if (snapshot.isCompleted() && predicate.test(snapshot))
{
if (!snapshot.hasManifest())
notExistingAnymore.add(snapshot);
else
snapshots.add(snapshot);
}
}
// we do not want to clear snapshots which do not exist periodically
// because that would beat the purpose of caching (we would need to go to the disk
// to see if manifests still exists every time), hence, we will clean on listing,
// we do not need to have cache clean of non-existing snapshots when nobody is looking
if (!notExistingAnymore.isEmpty())
{
snapshotManager.getSnapshots().removeAll(notExistingAnymore);
}
return snapshots;
}
private List<TableSnapshot> getWithoutRemoval()
{
List<TableSnapshot> snapshots = new ArrayList<>();
for (TableSnapshot snapshot : snapshotManager.getSnapshots())
{
if (snapshot.isCompleted() && predicate.test(snapshot))
{
if (snapshot.hasManifest())
snapshots.add(snapshot);
}
}
return snapshots;
}
@Override
public String toString()
{
return "GetSnapshotsTask{" +
"shouldRemoveIfNotExists=" + shouldRemoveIfNotExists +
'}';
}
}

148
src/java/org/apache/cassandra/service/snapshot/ListSnapshotsTask.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
import javax.management.openmbean.TabularData;
import javax.management.openmbean.TabularDataSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
public class ListSnapshotsTask implements Callable<Map<String, TabularData>>
{
private static final Logger logger = LoggerFactory.getLogger(ListSnapshotsTask.class);
private final SnapshotManager snapshotManager;
private final Predicate<TableSnapshot> predicate;
private final Map<String, String> options;
private final boolean shouldRemoveIfNotExists;
public ListSnapshotsTask(SnapshotManager snapshotManager,
Map<String, String> options,
boolean shouldRemoveIfNotExists)
{
this.snapshotManager = snapshotManager;
this.options = options;
this.predicate = getListingSnapshotsPredicate(options);
this.shouldRemoveIfNotExists = shouldRemoveIfNotExists;
}
private static Predicate<TableSnapshot> getListingSnapshotsPredicate(Map<String, String> options)
{
boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false"));
String selectedKeyspace = options != null ? options.get("keyspace") : null;
String selectedTable = options != null ? options.get("table") : null;
String selectedSnapshotName = options != null ? options.get("snapshot") : null;
return s -> {
if (selectedSnapshotName != null && !s.getTag().equals(selectedSnapshotName))
return false;
if (skipExpiring && s.isExpiring())
return false;
if (!includeEphemeral && s.isEphemeral())
return false;
if (selectedKeyspace != null && !s.getKeyspaceName().equals(selectedKeyspace))
return false;
return selectedTable == null || s.getTableName().equals(selectedTable);
};
}
@Override
public Map<String, TabularData> call()
{
List<TableSnapshot> filteredSnapshots;
try
{
filteredSnapshots = new GetSnapshotsTask(snapshotManager, predicate, shouldRemoveIfNotExists).call();
}
catch (Exception e)
{
logger.trace("Unable to get snapshots for listing purposes", e);
return Map.of();
}
Map<String, TabularData> snapshotMap = new HashMap<>();
Set<String> tags = new HashSet<>();
for (TableSnapshot t : filteredSnapshots)
tags.add(t.getTag());
for (String tag : tags)
snapshotMap.put(tag, new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE));
Map<String, Set<String>> keyspaceTables = new HashMap<>();
for (TableSnapshot s : filteredSnapshots)
{
keyspaceTables.computeIfAbsent(s.getKeyspaceName(), ignore -> new HashSet<>());
keyspaceTables.get(s.getKeyspaceName()).add(s.getTableName());
}
Map<String, Set<String>> cfsFiles = new HashMap<>();
for (Map.Entry<String, Set<String>> entry : keyspaceTables.entrySet())
{
for (String table : entry.getValue())
{
try
{
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(entry.getKey(), table);
if (cfs == null)
continue;
cfsFiles.put(cfs.getKeyspaceName() + '.' + cfs.name, cfs.getFilesOfCfs());
}
catch (Throwable t)
{
logger.debug("Unable to get all files of live SSTables for {}.{}", entry.getKey(), entry.getValue());
}
}
}
for (TableSnapshot snapshot : filteredSnapshots)
{
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag());
SnapshotDetailsTabularData.from(snapshot, data, cfsFiles.get(snapshot.getKeyspaceTable()));
}
return snapshotMap;
}
@Override
public String toString()
{
return "ListSnapshotsTask{" +
"options=" + options +
'}';
}
}

37
src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java → src/java/org/apache/cassandra/service/snapshot/SnapshotDetailsTabularData.java
View File

@ -15,33 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
package org.apache.cassandra.service.snapshot;
import java.util.Set;
import javax.management.openmbean.*;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.snapshot.TableSnapshot;
public class SnapshotDetailsTabularData
{
private static final String[] ITEM_NAMES = new String[]{"Snapshot name",
"Keyspace name",
"Column family name",
"True size",
"Size on disk",
"Creation time",
"Expiration time",
"Ephemeral"};
"Keyspace name",
"Column family name",
"True size",
"Size on disk",
"Creation time",
"Expiration time",
"Ephemeral"};
private static final String[] ITEM_DESCS = new String[]{"snapshot_name",
"keyspace_name",
"columnfamily_name",
"TrueDiskSpaceUsed",
"TotalDiskSpaceUsed",
"created_at",
"expires_at",
"ephemeral"};
"keyspace_name",
"columnfamily_name",
"TrueDiskSpaceUsed",
"TotalDiskSpaceUsed",
"created_at",
"expires_at",
"ephemeral"};
private static final String TYPE_NAME = "SnapshotDetails";
@ -70,17 +70,18 @@ public class SnapshotDetailsTabularData
}
public static void from(TableSnapshot details, TabularDataSupport result)
public static void from(TableSnapshot details, TabularDataSupport result, Set<String> files)
{
try
{
final String totalSize = FileUtils.stringifyFileSize(details.computeSizeOnDiskBytes());
long trueSizeBytes = details.computeTrueSizeBytes(files);
final String liveSize = FileUtils.stringifyFileSize(details.computeTrueSizeBytes());
String createdAt = safeToString(details.getCreatedAt());
String expiresAt = safeToString(details.getExpiresAt());
String ephemeral = Boolean.toString(details.isEphemeral());
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral }));
new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral }));
}
catch (OpenDataException e)
{

32
src/java/org/apache/cassandra/service/snapshot/SnapshotException.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
public class SnapshotException extends RuntimeException
{
public SnapshotException(String message)
{
super(message);
}
public SnapshotException(String message, Throwable cause)
{
super(message, cause);
}
}

9
src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
View File

@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@ -153,7 +154,7 @@ public class SnapshotLoader
String tag = snapshotDirMatcher.group("tag");
String snapshotId = buildSnapshotId(keyspaceName, tableName, tableId, tag);
TableSnapshot.Builder builder = snapshots.computeIfAbsent(snapshotId, k -> new TableSnapshot.Builder(keyspaceName, tableName, tableId, tag));
builder.addSnapshotDir(new File(snapshotDir));
builder.addSnapshotDir(new File(snapshotDir).toAbsolute());
}
}
@ -184,7 +185,11 @@ public class SnapshotLoader
}
}
return snapshots.values().stream().map(TableSnapshot.Builder::build).collect(Collectors.toSet());
Set<TableSnapshot> tableSnapshots = new HashSet<>();
for (TableSnapshot.Builder snapshotBuilder : snapshots.values())
tableSnapshots.add(snapshotBuilder.build());
return tableSnapshots;
}
public Set<TableSnapshot> loadSnapshots()

550
src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java
View File

@ -17,81 +17,213 @@
*/
package org.apache.cassandra.service.snapshot;
import java.util.Collection;
import java.util.PriorityQueue;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Predicate;
import javax.management.openmbean.TabularData;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.notifications.INotification;
import org.apache.cassandra.notifications.INotificationConsumer;
import org.apache.cassandra.notifications.TableDroppedNotification;
import org.apache.cassandra.notifications.TablePreScrubNotification;
import org.apache.cassandra.notifications.TruncationNotification;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.toList;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.utils.FBUtilities.now;
public class SnapshotManager {
private static final ScheduledExecutorPlus executor = executorFactory().scheduled(false, "SnapshotCleanup");
import static org.apache.cassandra.service.snapshot.ClearSnapshotTask.getClearSnapshotPredicate;
import static org.apache.cassandra.service.snapshot.ClearSnapshotTask.getPredicateForCleanedSnapshots;
public class SnapshotManager implements SnapshotManagerMBean, INotificationConsumer, AutoCloseable
{
private static final Logger logger = LoggerFactory.getLogger(SnapshotManager.class);
private ScheduledExecutorPlus snapshotCleanupExecutor;
public static final SnapshotManager instance = new SnapshotManager();
private final long initialDelaySeconds;
private final long cleanupPeriodSeconds;
private final SnapshotLoader snapshotLoader;
@VisibleForTesting
protected volatile ScheduledFuture<?> cleanupTaskFuture;
private volatile ScheduledFuture<?> cleanupTaskFuture;
private final String[] dataDirs;
private volatile boolean started = false;
/**
* Expiring snapshots ordered by expiration date, to allow only iterating over snapshots
* that need to be removed on {@link this#clearExpiredSnapshots()}
* We read / list snapshots way more often than write / create them so COW is ideal to use here.
* This enables us to not submit listing tasks or tasks computing snapshot sizes to any executor's queue as they
* can be just run concurrently which gives way better throughput in case
* of excessive listing from clients (dashboards and similar) where snapshot metrics are gathered.
*/
private final PriorityQueue<TableSnapshot> expiringSnapshots = new PriorityQueue<>(comparing(TableSnapshot::getExpiresAt));
private final List<TableSnapshot> snapshots = new CopyOnWriteArrayList<>();
public SnapshotManager()
private SnapshotManager()
{
this(CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(),
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt());
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt(),
DatabaseDescriptor.getAllDataFileLocations());
}
@VisibleForTesting
protected SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds)
SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds, String[] dataDirs)
{
this.initialDelaySeconds = initialDelaySeconds;
this.cleanupPeriodSeconds = cleanupPeriodSeconds;
snapshotLoader = new SnapshotLoader(DatabaseDescriptor.getAllDataFileLocations());
this.dataDirs = dataDirs;
this.snapshotCleanupExecutor = createSnapshotCleanupExecutor();
}
public Collection<TableSnapshot> getExpiringSnapshots()
public void registerMBean()
{
return expiringSnapshots;
MBeanWrapper.instance.registerMBean(this, MBEAN_NAME);
}
public synchronized void start()
public void unregisterMBean()
{
addSnapshots(loadSnapshots());
resumeSnapshotCleanup();
MBeanWrapper.instance.unregisterMBean(MBEAN_NAME);
}
public synchronized void stop() throws InterruptedException, TimeoutException
public synchronized SnapshotManager start(boolean runPeriodicSnapshotCleaner)
{
if (started)
return this;
if (snapshotCleanupExecutor == null)
snapshotCleanupExecutor = createSnapshotCleanupExecutor();
executeTask(new ReloadSnapshotsTask(dataDirs));
if (runPeriodicSnapshotCleaner)
resumeSnapshotCleanup();
started = true;
return this;
}
@Override
public synchronized void close()
{
if (!started)
return;
pauseSnapshotCleanup();
shutdownAndWait(1, TimeUnit.MINUTES);
snapshots.clear();
started = false;
}
public synchronized void shutdownAndWait(long timeout, TimeUnit unit)
{
try
{
ExecutorUtils.shutdownNowAndWait(timeout, unit, snapshotCleanupExecutor);
}
catch (InterruptedException | TimeoutException ex)
{
throw new RuntimeException(ex);
}
finally
{
snapshotCleanupExecutor = null;
}
}
public synchronized void restart(boolean runPeriodicSnapshotCleaner)
{
if (!started)
return;
logger.debug("Restarting SnapshotManager");
close();
start(runPeriodicSnapshotCleaner);
logger.debug("SnapshotManager restarted");
}
public synchronized void restart()
{
restart(true);
}
private static class ReloadSnapshotsTask extends AbstractSnapshotTask<Set<TableSnapshot>>
{
private final String[] dataDirs;
public ReloadSnapshotsTask(String[] dataDirs)
{
super(null);
this.dataDirs = dataDirs;
}
@Override
public Set<TableSnapshot> call()
{
Set<TableSnapshot> tableSnapshots = new SnapshotLoader(dataDirs).loadSnapshots();
new ClearSnapshotTask(SnapshotManager.instance, snapshot -> true, false).call();
for (TableSnapshot snapshot : tableSnapshots)
SnapshotManager.instance.addSnapshot(snapshot);
return tableSnapshots;
}
@Override
public SnapshotTaskType getTaskType()
{
return SnapshotTaskType.RELOAD;
}
}
void addSnapshot(TableSnapshot snapshot)
{
logger.debug("Adding snapshot {}", snapshot);
snapshots.add(snapshot);
}
List<TableSnapshot> getSnapshots()
{
return snapshots;
}
public void resumeSnapshotCleanup()
{
if (cleanupTaskFuture == null)
{
logger.info("Scheduling expired snapshots cleanup with initialDelaySeconds={} and cleanupPeriodSeconds={}",
initialDelaySeconds, cleanupPeriodSeconds);
cleanupTaskFuture = snapshotCleanupExecutor.scheduleWithFixedDelay(SnapshotManager.instance::clearExpiredSnapshots,
initialDelaySeconds,
cleanupPeriodSeconds,
SECONDS);
}
}
private void pauseSnapshotCleanup()
{
expiringSnapshots.clear();
if (cleanupTaskFuture != null)
{
cleanupTaskFuture.cancel(false);
@ -99,74 +231,350 @@ public class SnapshotManager {
}
}
public synchronized void addSnapshot(TableSnapshot snapshot)
/**
* Deletes snapshot and removes it from manager.
*
* @param snapshot snapshot to clear
*/
void clearSnapshot(TableSnapshot snapshot)
{
// We currently only care about expiring snapshots
if (snapshot.isExpiring())
executeTask(new ClearSnapshotTask(this, s -> s.equals(snapshot), true));
}
/**
* Returns list of snapshots of given keyspace
*
* @param keyspace keyspace of a snapshot
* @return list of snapshots of given keyspace.
*/
public List<TableSnapshot> getSnapshots(String keyspace)
{
return getSnapshots(snapshot -> snapshot.getKeyspaceName().equals(keyspace));
}
/**
* Return snapshots based on given parameters.
*
* @param skipExpiring if expiring snapshots should be skipped
* @param includeEphemeral if ephemeral snapshots should be included
* @return snapshots based on given parameters
*/
public List<TableSnapshot> getSnapshots(boolean skipExpiring, boolean includeEphemeral)
{
return getSnapshots(s -> (!skipExpiring || !s.isExpiring()) && (includeEphemeral || !s.isEphemeral()));
}
/**
* Returns all snapshots passing the given predicate.
*
* @param predicate predicate to filter all snapshots of
* @return list of snapshots passing the predicate
*/
public List<TableSnapshot> getSnapshots(Predicate<TableSnapshot> predicate)
{
return new GetSnapshotsTask(this, predicate, true).call();
}
/**
* Returns a snapshot or empty optional based on the given parameters.
*
* @param keyspace keyspace of a snapshot
* @param table table of a snapshot
* @param tag name of a snapshot
* @return empty optional if there is not such snapshot, non-empty otherwise
*/
public Optional<TableSnapshot> getSnapshot(String keyspace, String table, String tag)
{
List<TableSnapshot> foundSnapshots = new GetSnapshotsTask(this,
snapshot -> snapshot.getKeyspaceName().equals(keyspace) &&
snapshot.getTableName().equals(table) &&
snapshot.getTag().equals(tag) || (tag != null && tag.isEmpty()),
true).call();
if (foundSnapshots.isEmpty())
return Optional.empty();
else
return Optional.of(foundSnapshots.get(0));
}
/**
* Checks whether a snapshot for given keyspace and table exists of a given name exists.
*
* @param keyspace keyspace to get a snapshot of
* @param table table to get a snapshot of
* @param tag name of a snapshot
* @return true of a snapshot of given properties exist, false otherwise
*/
public boolean exists(String keyspace, String table, String tag)
{
return getSnapshot(keyspace, table, tag).isPresent();
}
/**
* Checks whether a snapshot which satisfies given predicate exists.
*
* @param predicate predicate to check the existence of a snapshot
* @return true if a snapshot which satisfies a predicate exists, false otherwise
*/
public boolean exists(Predicate<TableSnapshot> predicate)
{
return !getSnapshots(predicate).isEmpty();
}
/**
* Clear snapshots of given tag from given keyspace. Does not remove ephemeral snapshots.
* <p>
*
* @param tag snapshot name
* @param keyspace keyspace to clear all snapshots of a given tag of
*/
public void clearSnapshots(String tag, String keyspace)
{
clearSnapshots(tag, Set.of(keyspace), Clock.Global.currentTimeMillis());
}
/**
* Removes a snapshot.
* <p>
*
* @param keyspace keyspace of a snapshot to remove
* @param table table of a snapshot to remove
* @param tag name of a snapshot to remove.
*/
public void clearSnapshot(String keyspace, String table, String tag)
{
executeTask(new ClearSnapshotTask(this,
snapshot -> snapshot.getKeyspaceName().equals(keyspace)
&& snapshot.getTableName().equals(table)
&& snapshot.getTag().equals(tag),
true));
}
/**
* Removes all snapshots for given keyspace and table.
*
* @param keyspace keyspace to remove snapshots for
* @param table table in a given keyspace to remove snapshots for
*/
public void clearAllSnapshots(String keyspace, String table)
{
executeTask(new ClearSnapshotTask(this,
snapshot -> snapshot.getKeyspaceName().equals(keyspace)
&& snapshot.getTableName().equals(table),
true));
}
/**
* Clears all snapshots, expiring and ephemeral as well.
*/
public void clearAllSnapshots()
{
executeTask(new ClearSnapshotTask(this, snapshot -> true, true));
}
/**
* Clear snapshots based on a given predicate
*
* @param predicate predicate to filter snapshots on
*/
public void clearSnapshot(Predicate<TableSnapshot> predicate)
{
executeTask(new ClearSnapshotTask(this, predicate, true));
}
/**
* Clears all ephemeral snapshots in a node.
*/
public void clearEphemeralSnapshots()
{
executeTask(new ClearSnapshotTask(this, TableSnapshot::isEphemeral, true));
}
/**
* Clears all expired snapshots in a node.
*/
public void clearExpiredSnapshots()
{
Instant now = FBUtilities.now();
executeTask(new ClearSnapshotTask(this, s -> s.isExpired(now), true));
}
/**
* Clear snapshots of given tag from given keyspaces.
* <p>
* If tag is not present / is empty, all snapshots are considered to be cleared.
* If keyspaces are empty, all snapshots of given tag and older than maxCreatedAt are removed.
*
* @param tag optional tag of snapshot to clear
* @param keyspaces keyspaces to remove snapshots for
* @param maxCreatedAt clear all such snapshots which were created before this timestamp
*/
private void clearSnapshots(String tag, Set<String> keyspaces, long maxCreatedAt)
{
executeTask(new ClearSnapshotTask(this, getClearSnapshotPredicate(tag, keyspaces, maxCreatedAt, false), true));
}
public List<TableSnapshot> takeSnapshot(SnapshotOptions options)
{
return executeTask(new TakeSnapshotTask(this, options));
}
// Super methods
@Override
public void takeSnapshot(String tag, String... entities)
{
takeSnapshot(SnapshotOptions.userSnapshot(tag, Map.of(), entities));
}
@Override
public void takeSnapshot(String tag, Map<String, String> optMap, String... entities) throws IOException
{
try
{
logger.debug("Adding expiring snapshot {}", snapshot);
expiringSnapshots.add(snapshot);
takeSnapshot(SnapshotOptions.userSnapshot(tag, optMap, entities));
}
catch (SnapshotException ex)
{
// to be compatible with deprecated methods in StorageService
throw new IOException(ex);
}
}
public synchronized Set<TableSnapshot> loadSnapshots(String keyspace)
@Override
public void clearSnapshot(String tag, Map<String, Object> options, String... keyspaceNames)
{
return snapshotLoader.loadSnapshots(keyspace);
executeTask(new ClearSnapshotTask(this, getPredicateForCleanedSnapshots(tag, options, keyspaceNames), true));
}
public synchronized Set<TableSnapshot> loadSnapshots()
@Override
public Map<String, TabularData> listSnapshots(Map<String, String> options)
{
return snapshotLoader.loadSnapshots();
return new ListSnapshotsTask(this, options, true).call();
}
@VisibleForTesting
protected synchronized void addSnapshots(Collection<TableSnapshot> snapshots)
@Override
public long getTrueSnapshotSize()
{
logger.debug("Adding snapshots: {}.", Joiner.on(", ").join(snapshots.stream().map(TableSnapshot::getId).collect(toList())));
snapshots.forEach(this::addSnapshot);
return new TrueSnapshotSizeTask(this, s -> true).call();
}
// TODO: Support pausing snapshot cleanup
@VisibleForTesting
synchronized void resumeSnapshotCleanup()
@Override
public long getTrueSnapshotsSize(String keyspace)
{
if (cleanupTaskFuture == null)
return new TrueSnapshotSizeTask(this, s -> s.getKeyspaceName().equals(keyspace)).call();
}
@Override
public long getTrueSnapshotsSize(String keyspace, String table)
{
return new TrueSnapshotSizeTask(this, s -> s.getKeyspaceName().equals(keyspace) && s.getTableName().equals(table)).call();
}
@Override
public void setSnapshotLinksPerSecond(long throttle)
{
logger.info("Setting snapshot throttle to {}", throttle);
DatabaseDescriptor.setSnapshotLinksPerSecond(throttle);
}
@Override
public long getSnapshotLinksPerSecond()
{
return DatabaseDescriptor.getSnapshotLinksPerSecond();
}
@Override
public void handleNotification(INotification notification, Object sender)
{
if (notification instanceof TruncationNotification)
{
logger.info("Scheduling expired snapshot cleanup with initialDelaySeconds={} and cleanupPeriodSeconds={}",
initialDelaySeconds, cleanupPeriodSeconds);
cleanupTaskFuture = executor.scheduleWithFixedDelay(this::clearExpiredSnapshots, initialDelaySeconds,
cleanupPeriodSeconds, TimeUnit.SECONDS);
TruncationNotification truncationNotification = (TruncationNotification) notification;
ColumnFamilyStore cfs = truncationNotification.cfs;
if (!truncationNotification.disableSnapshot && cfs.isAutoSnapshotEnabled())
{
SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.TRUNCATE, cfs.getKeyspaceTableName())
.ttl(truncationNotification.ttl).build();
takeSnapshot(opts);
}
}
else if (notification instanceof TableDroppedNotification)
{
TableDroppedNotification tableDroppedNotification = (TableDroppedNotification) notification;
ColumnFamilyStore cfs = tableDroppedNotification.cfs;
if (cfs.isAutoSnapshotEnabled())
{
SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.DROP, cfs.getKeyspaceTableName())
.cfs(cfs).ttl(tableDroppedNotification.ttl).build();
takeSnapshot(opts);
}
}
else if (notification instanceof TablePreScrubNotification)
{
ColumnFamilyStore cfs = ((TablePreScrubNotification) notification).cfs;
SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.PRE_SCRUB, cfs.getKeyspaceTableName()).build();
takeSnapshot(opts);
}
}
@VisibleForTesting
protected synchronized void clearExpiredSnapshots()
List<TableSnapshot> executeTask(TakeSnapshotTask task)
{
TableSnapshot expiredSnapshot;
while ((expiredSnapshot = expiringSnapshots.peek()) != null)
try
{
if (!expiredSnapshot.isExpired(now()))
break; // the earliest expiring snapshot is not expired yet, so there is no more expired snapshots to remove
prePopulateSnapshots(task);
return task.call();
}
catch (Throwable t)
{
throw new SnapshotException(String.format("Exception occured while executing %s: %s", task.toString(), t.getMessage()), t);
}
}
logger.debug("Removing expired snapshot {}.", expiredSnapshot);
clearSnapshot(expiredSnapshot);
@VisibleForTesting
<T> T executeTask(AbstractSnapshotTask<T> task)
{
try
{
return task.call();
}
catch (Throwable t)
{
throw new SnapshotException(String.format("Exception occured while executing %s", task.toString()), t);
}
}
/**
* Deletes snapshot and remove it from manager
* Add table snapshots to snaphots cow list in advance in order to be sure that the snapshots to be created
* are not existing already. "executeTask" method can be invoked by multiple threads, and they hit
* this synchronized method which populates snapshots so next thread will fail to advance (exception is thrown)
* if table snapshots of some other tasks are already present in snapshots cow list.
*
* Added snapshots to snapshot list are at this point "incomplete", they will not appear in listing output
* until they are indeed taken and exist on disk, otherwise we would see "phantom" snapshots in listings,
* they would be present but in fact they are just going through the process of being taken.
*
* @param task task to process
*/
public synchronized void clearSnapshot(TableSnapshot snapshot)
private synchronized void prePopulateSnapshots(TakeSnapshotTask task)
{
for (File snapshotDir : snapshot.getDirectories())
Directories.removeSnapshotDirectory(DatabaseDescriptor.getSnapshotRateLimiter(), snapshotDir);
Map<ColumnFamilyStore, TableSnapshot> snapshotsToCreate = task.getSnapshotsToCreate();
for (Map.Entry<ColumnFamilyStore, TableSnapshot> toCreateEntry : snapshotsToCreate.entrySet())
{
if (snapshots.contains(toCreateEntry.getValue()))
{
throw new RuntimeException(format("Snapshot %s for %s.%s already exists.",
toCreateEntry.getValue().getTag(),
toCreateEntry.getValue().getKeyspaceName(),
toCreateEntry.getValue().getTableName()));
}
}
expiringSnapshots.remove(snapshot);
snapshots.addAll(snapshotsToCreate.values());
}
@VisibleForTesting
public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
private static ScheduledExecutorPlus createSnapshotCleanupExecutor()
{
ExecutorUtils.shutdownNowAndWait(timeout, unit, executor);
return executorFactory().scheduled(false, "SnapshotCleanup");
}
}

109
src/java/org/apache/cassandra/service/snapshot/SnapshotManagerMBean.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.io.IOException;
import java.util.Map;
import javax.management.openmbean.TabularData;
public interface SnapshotManagerMBean
{
String MBEAN_NAME = "org.apache.cassandra.service.snapshot:type=SnapshotManager";
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
* @param options Map of options (skipFlush is the only supported option for now)
* @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
*/
void takeSnapshot(String tag, Map<String, String> options, String... entities) throws IOException;
/**
* Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified.
*
* @param tag the tag given to the snapshot; may not be null or empty
* @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,...
*/
void takeSnapshot(String tag, String... entities) throws IOException;
/**
* Remove the snapshot with the given name from the given keyspaces.
* If no tag is specified we will remove all snapshots.
*
* @param tag name of snapshot to clear, if null or empty string,
* all snapshots of given keyspace will be cleared
* @param options map of options for cleanup operation, consult nodetool's ClearSnapshot
* @param keyspaceNames name of keyspaces to clear snapshots for
*/
void clearSnapshot(String tag, Map<String, Object> options, String... keyspaceNames) throws IOException;
/**
* Get the details of all the snapshots
*
* @param options map of options used for filtering of snapshots
* @return A map of snapshotName to all its details in Tabular form.
*/
Map<String, TabularData> listSnapshots(Map<String, String> options);
/**
* Get the true size taken by all snapshots across all keyspaces.
*
* @return True size taken by all the snapshots.
*/
long getTrueSnapshotSize();
/**
* Get the true size take by all snapshots in given keyspace.
*
* @param keyspace keyspace to get true size of all snapshots of
* @return true size of all snapshots in given keyspace
*/
long getTrueSnapshotsSize(String keyspace);
/**
* Get the true size take by all snapshots in given keyspace and table.
*
* @param keyspace keyspace to get true size of all snapshots of
* @param table table in a keyspace to get true size of all snapshots of
* @return true size of all snapshots in given keyspace and table
*/
long getTrueSnapshotsSize(String keyspace, String table);
/**
* Set the current hardlink-per-second throttle for snapshots
* A setting of zero indicates no throttling
*
* @param throttle hard-links-per-second
*/
void setSnapshotLinksPerSecond(long throttle);
/**
* Get the current hardlink-per-second throttle for snapshots
* A setting of zero indicates no throttling.
*
* @return snapshot links-per-second throttle
*/
long getSnapshotLinksPerSecond();
/**
* Restarting means that snapshots will be reloaded from disk.
*/
void restart();
}

7
src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java
View File

@ -63,7 +63,7 @@ public class SnapshotManifest
{
this.files = files;
this.createdAt = creationTime;
this.expiresAt = ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds());
this.expiresAt = computeExpiration(ttl, creationTime);
this.ephemeral = ephemeral;
}
@ -97,6 +97,11 @@ public class SnapshotManifest
return JsonUtils.deserializeFromJsonFile(SnapshotManifest.class, file);
}
public static Instant computeExpiration(DurationSpec.IntSecondsBound ttl, Instant createdAt)
{
return ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds());
}
@Override
public boolean equals(Object o)
{

235
src/java/org/apache/cassandra/service/snapshot/SnapshotOptions.java Normal file
View File

@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.function.Predicate;
import com.google.common.util.concurrent.RateLimiter;
import org.apache.commons.lang3.StringUtils;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import static java.lang.String.format;
public class SnapshotOptions
{
public static final String SKIP_FLUSH = "skipFlush";
public static final String TTL = "ttl";
public final SnapshotType type;
public final String tag;
public final DurationSpec.IntSecondsBound ttl;
public final Instant creationTime;
public final boolean skipFlush;
public final boolean ephemeral;
public final String[] entities;
public final RateLimiter rateLimiter;
public final Predicate<SSTableReader> sstableFilter;
public final ColumnFamilyStore cfs;
private SnapshotOptions(SnapshotType type,
String tag,
DurationSpec.IntSecondsBound ttl,
Instant creationTime,
boolean skipFlush,
boolean ephemeral,
String[] entities,
RateLimiter rateLimiter,
Predicate<SSTableReader> sstableFilter,
ColumnFamilyStore cfs)
{
this.type = type;
this.tag = tag;
this.ttl = ttl;
this.creationTime = creationTime;
this.skipFlush = skipFlush;
this.ephemeral = ephemeral;
this.entities = entities;
this.rateLimiter = rateLimiter;
this.sstableFilter = sstableFilter;
this.cfs = cfs;
}
public static Builder systemSnapshot(String tag, SnapshotType type, String... entities)
{
return new Builder(tag, type, ssTableReader -> true, entities);
}
public static Builder systemSnapshot(String tag, SnapshotType type, Predicate<SSTableReader> sstableFilter, String... entities)
{
return new Builder(tag, type, sstableFilter, entities);
}
public static SnapshotOptions userSnapshot(String tag, String... entities)
{
return userSnapshot(tag, Collections.emptyMap(), entities);
}
public static SnapshotOptions userSnapshot(String tag, Map<String, String> options, String... entities)
{
Builder builder = new Builder(tag, SnapshotType.USER, ssTableReader -> true, entities).ttl(options.get(TTL));
if (Boolean.parseBoolean(options.getOrDefault(SKIP_FLUSH, Boolean.FALSE.toString())))
builder.skipFlush();
return builder.build();
}
public String getSnapshotName(Instant creationTime)
{
// Diagnostic snapshots have very specific naming convention hence we are keeping it.
// Repair snapshots rely on snapshots having name of their repair session ids
if (type == SnapshotType.USER || type == SnapshotType.DIAGNOSTICS || type == SnapshotType.REPAIR)
return tag;
// System snapshots have the creation timestamp on the name
String snapshotName = format("%d-%s", creationTime.toEpochMilli(), type.label);
if (StringUtils.isNotBlank(tag))
snapshotName = snapshotName + '-' + tag;
return snapshotName;
}
public static class Builder
{
private final String tag;
private final String[] entities;
private DurationSpec.IntSecondsBound ttl;
private Instant creationTime;
private boolean skipFlush = false;
private boolean ephemeral = false;
private ColumnFamilyStore cfs;
private final Predicate<SSTableReader> sstableFilter;
private final SnapshotType type;
private RateLimiter rateLimiter;
public Builder(String tag, SnapshotType type, Predicate<SSTableReader> sstableFilter, String... entities)
{
this.tag = tag;
this.type = type;
this.entities = entities;
this.sstableFilter = sstableFilter;
}
public Builder ttl(String ttl)
{
if (ttl != null)
this.ttl = new DurationSpec.IntSecondsBound(ttl);
return this;
}
public Builder ttl(DurationSpec.IntSecondsBound ttl)
{
this.ttl = ttl;
return this;
}
public Builder creationTime(String creationTime)
{
if (creationTime != null)
{
try
{
return creationTime(Long.parseLong(creationTime));
}
catch (Exception ex)
{
throw new RuntimeException("Unable to parse creation time from " + creationTime);
}
}
return this;
}
public Builder creationTime(Instant creationTime)
{
this.creationTime = creationTime;
return this;
}
public Builder creationTime(long creationTime)
{
return creationTime(Instant.ofEpochMilli(creationTime));
}
public Builder skipFlush()
{
skipFlush = true;
return this;
}
public Builder ephemeral()
{
ephemeral = true;
return this;
}
public Builder cfs(ColumnFamilyStore cfs)
{
this.cfs = cfs;
return this;
}
public Builder rateLimiter(RateLimiter rateLimiter)
{
this.rateLimiter = rateLimiter;
return this;
}
public SnapshotOptions build()
{
if (tag == null || tag.isEmpty())
throw new RuntimeException("You must supply a snapshot name.");
if (ttl != null)
{
int minAllowedTtlSecs = CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.getInt();
if (ttl.toSeconds() < minAllowedTtlSecs)
throw new IllegalArgumentException(format("ttl for snapshot must be at least %d seconds", minAllowedTtlSecs));
}
if (ephemeral && ttl != null)
throw new IllegalStateException(format("can not take ephemeral snapshot (%s) while ttl is specified too", tag));
if (rateLimiter == null)
rateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
return new SnapshotOptions(type, tag, ttl, creationTime, skipFlush, ephemeral, entities, rateLimiter,
sstableFilter, cfs);
}
}
@Override
public String toString()
{
return "CreateSnapshotOptions{" +
"type=" + type +
", tag='" + tag + '\'' +
", ttl=" + ttl +
", creationTime=" + creationTime +
", skipFlush=" + skipFlush +
", ephemeral=" + ephemeral +
", entities=" + Arrays.toString(entities) +
'}';
}
}

42
src/java/org/apache/cassandra/service/snapshot/SnapshotType.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
public enum SnapshotType
{
USER("user"),
TRUNCATE("truncated"),
DROP("dropped"),
PRE_SCRUB("pre-scrub"),
COMPACT("compact"),
UPGRADE("upgrade"),
DIAGNOSTICS("diagnostics"),
REPAIR("repair"),
MISC("misc");
public final String label;
SnapshotType(String label)
{
this.label = label;
}
}

378
src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
View File

@ -21,21 +21,33 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableId;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.DirectorySizeCalculator;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Refs;
public class TableSnapshot
{
@ -43,6 +55,7 @@ public class TableSnapshot
private final String keyspaceName;
private final String tableName;
private final String keyspaceTable;
private final UUID tableId;
private final String tag;
private final boolean ephemeral;
@ -52,25 +65,51 @@ public class TableSnapshot
private final Set<File> snapshotDirs;
private volatile long sizeOnDisk = 0;
private volatile long manifestsSize;
private volatile long schemasSize;
private volatile boolean inProgress = false;
public TableSnapshot(String keyspaceName, String tableName, UUID tableId,
String tag, Instant createdAt, Instant expiresAt,
Set<File> snapshotDirs, boolean ephemeral)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
this.keyspaceTable = keyspaceName + '.' + tableName;
this.tableId = tableId;
this.tag = tag;
this.createdAt = createdAt;
this.expiresAt = expiresAt;
this.snapshotDirs = snapshotDirs;
this.ephemeral = ephemeral;
manifestsSize = getManifestsSize();
schemasSize = getSchemasSize();
}
public boolean isCompleted()
{
return !inProgress;
}
public void incomplete()
{
inProgress = true;
}
public void complete()
{
inProgress = false;
}
/**
* Unique identifier of a snapshot. Used
* only to deduplicate snapshots internally,
* not exposed externally.
*
* <p>
* Format: "$ks:$table_name:$table_id:$tag"
*/
public String getId()
@ -88,6 +127,11 @@ public class TableSnapshot
return tableName;
}
public String getKeyspaceTable()
{
return keyspaceTable;
}
public String getTag()
{
return tag;
@ -95,13 +139,22 @@ public class TableSnapshot
public Instant getCreatedAt()
{
if (createdAt == null)
{
long minCreation = snapshotDirs.stream().mapToLong(File::lastModified).min().orElse(0);
if (minCreation != 0)
long minCreation = 0;
for (File snapshotDir : snapshotDirs)
{
return Instant.ofEpochMilli(minCreation);
long lastModified = snapshotDir.lastModified();
if (lastModified == 0)
continue;
if (minCreation == 0 || minCreation > lastModified)
minCreation = lastModified;
}
if (minCreation != 0)
return Instant.ofEpochMilli(minCreation);
}
return createdAt;
}
@ -123,7 +176,11 @@ public class TableSnapshot
public boolean exists()
{
return snapshotDirs.stream().anyMatch(File::exists);
for (File snapshotDir : snapshotDirs)
if (snapshotDir.exists())
return true;
return false;
}
public boolean isEphemeral()
@ -138,26 +195,106 @@ public class TableSnapshot
public long computeSizeOnDiskBytes()
{
return snapshotDirs.stream().mapToLong(FileUtils::folderSize).sum();
long sum = sizeOnDisk;
if (sum == 0)
{
for (File snapshotDir : snapshotDirs)
sum += FileUtils.folderSize(snapshotDir);
sizeOnDisk = sum;
}
return sum;
}
public long computeTrueSizeBytes()
{
DirectorySizeCalculator visitor = new SnapshotTrueSizeCalculator();
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName);
if (cfs == null)
return 0;
for (File snapshotDir : snapshotDirs)
return computeTrueSizeBytes(cfs.getFilesOfCfs());
}
public long computeTrueSizeBytes(Set<String> files)
{
long size = manifestsSize + schemasSize;
for (File dataPath : getDirectories())
{
try
List<Path> snapshotFiles = listDir(dataPath.toPath());
for (Path snapshotFile : snapshotFiles)
{
Files.walkFileTree(snapshotDir.toPath(), visitor);
}
catch (IOException e)
{
logger.error("Could not calculate the size of {}.", snapshotDir, e);
if (!snapshotFile.endsWith("manifest.json") && !snapshotFile.endsWith("schema.cql"))
{
// files == null means that the underlying table was most probably dropped
// so in that case we indeed go to count snapshots file in for true size
if (files == null || (!files.contains(getLiveFileFromSnapshotFile(snapshotFile))))
size += getFileSize(snapshotFile);
}
}
}
return visitor.getAllocatedSize();
return size;
}
private List<Path> listDir(Path dir)
{
List<Path> paths = new ArrayList<>();
try (Stream<Path> stream = Files.list(dir))
{
stream.forEach(p -> {
if (p.getFileName().toString().startsWith("."))
{
paths.addAll(listDir(p));
}
else
{
paths.add(p);
}
});
}
catch (Throwable t)
{
logger.error("Could not list directory content {}", dir, t);
}
return paths;
}
private long getFileSize(Path file)
{
try
{
return Files.size(file);
}
catch (Throwable t)
{
return 0;
}
}
/**
* Returns the corresponding live file for a given snapshot file.
* <p>
* Example:
* - Base table:
* - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db
* - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db
* - Secondary index:
* - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db
* - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db
*/
static String getLiveFileFromSnapshotFile(Path snapshotFilePath)
{
// Snapshot directory structure format is {data_dir}/snapshots/{snapshot_name}/{snapshot_file}
Path liveDir = snapshotFilePath.getParent().getParent().getParent();
if (Directories.isSecondaryIndexFolder(snapshotFilePath.getParent()))
{
// Snapshot file structure format is {data_dir}/snapshots/{snapshot_name}/.{index}/{sstable-component}.db
liveDir = File.getPath(liveDir.getParent().toString(), snapshotFilePath.getParent().getFileName().toString());
}
return liveDir.resolve(snapshotFilePath.getFileName().toString()).toAbsolutePath().toString();
}
public Collection<File> getDirectories()
@ -165,48 +302,88 @@ public class TableSnapshot
return snapshotDirs;
}
public Optional<File> getManifestFile()
/**
* Returns all manifest files of a snapshot.
* <p>
* In practice, there might be multiple manifest files, as many as we have snapshot dirs.
* Each snapshot dir will hold its view of a snapshot, containing only sstables located in such snapshot dir.
*
* @return all manifest files
*/
public Set<File> getManifestFiles()
{
Set<File> manifestFiles = new HashSet<>();
for (File snapshotDir : snapshotDirs)
{
File manifestFile = Directories.getSnapshotManifestFile(snapshotDir);
if (manifestFile.exists())
{
return Optional.of(manifestFile);
}
manifestFiles.add(manifestFile);
}
return Optional.empty();
return manifestFiles;
}
public Optional<File> getSchemaFile()
public boolean hasManifest()
{
for (File snapshotDir : snapshotDirs)
{
if (Directories.getSnapshotManifestFile(snapshotDir).exists())
return true;
}
return false;
}
/**
* Returns all schemas files of a snapshot.
*
* @return all schema files
*/
public Set<File> getSchemaFiles()
{
Set<File> schemaFiles = new HashSet<>();
for (File snapshotDir : snapshotDirs)
{
File schemaFile = Directories.getSnapshotSchemaFile(snapshotDir);
if (schemaFile.exists())
{
return Optional.of(schemaFile);
}
schemaFiles.add(schemaFile);
}
return Optional.empty();
return schemaFiles;
}
public long getManifestsSize()
{
long size = 0;
for (File manifestFile : getManifestFiles())
size += manifestFile.length();
return size;
}
public long getSchemasSize()
{
long size = 0;
for (File schemaFile : getSchemaFiles())
size += schemaFile.length();
return size;
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TableSnapshot snapshot = (TableSnapshot) o;
return Objects.equals(keyspaceName, snapshot.keyspaceName) && Objects.equals(tableName, snapshot.tableName) &&
Objects.equals(tableId, snapshot.tableId) && Objects.equals(tag, snapshot.tag) &&
Objects.equals(createdAt, snapshot.createdAt) && Objects.equals(expiresAt, snapshot.expiresAt) &&
Objects.equals(snapshotDirs, snapshot.snapshotDirs) && Objects.equals(ephemeral, snapshot.ephemeral);
return Objects.equals(keyspaceName, snapshot.keyspaceName) &&
Objects.equals(tableName, snapshot.tableName) &&
Objects.equals(tableId, snapshot.tableId) &&
Objects.equals(tag, snapshot.tag);
}
@Override
public int hashCode()
{
return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral);
return Objects.hash(keyspaceName, tableName, tableId, tag);
}
@Override
@ -224,7 +401,16 @@ public class TableSnapshot
'}';
}
static class Builder {
public void updateMetadataSize()
{
if (manifestsSize == 0)
manifestsSize = getManifestsSize();
if (schemasSize == 0)
schemasSize = getSchemasSize();
}
static class Builder
{
private final String keyspaceName;
private final String tableName;
private final UUID tableId;
@ -287,66 +473,82 @@ public class TableSnapshot
return String.format("%s:%s:%s:%s", keyspaceName, tableName, tableId, tag);
}
public static class SnapshotTrueSizeCalculator extends DirectorySizeCalculator
public static Set<Descriptor> getSnapshotDescriptors(String keyspace, String table, String tag)
{
/**
* Snapshots are composed of hard-linked sstables. The true snapshot size should only include
* snapshot files which do not contain a corresponding "live" sstable file.
*/
@Override
public boolean isAcceptable(Path snapshotFilePath)
try
{
return !getLiveFileFromSnapshotFile(snapshotFilePath).exists();
}
}
Refs<SSTableReader> snapshotSSTableReaders = getSnapshotSSTableReaders(keyspace, table, tag);
/**
* Returns the corresponding live file for a given snapshot file.
*
* Example:
* - Base table:
* - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db
* - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db
* - Secondary index:
* - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db
* - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db
*
*/
static File getLiveFileFromSnapshotFile(Path snapshotFilePath)
{
// Snapshot directory structure format is {data_dir}/snapshots/{snapshot_name}/{snapshot_file}
Path liveDir = snapshotFilePath.getParent().getParent().getParent();
if (Directories.isSecondaryIndexFolder(snapshotFilePath.getParent()))
{
// Snapshot file structure format is {data_dir}/snapshots/{snapshot_name}/.{index}/{sstable-component}.db
liveDir = File.getPath(liveDir.getParent().toString(), snapshotFilePath.getParent().getFileName().toString());
}
return new File(liveDir.toString(), snapshotFilePath.getFileName().toString());
}
public static Predicate<TableSnapshot> shouldClearSnapshot(String tag, long olderThanTimestamp)
{
return ts ->
{
// When no tag is supplied, all snapshots must be cleared
boolean clearAll = tag == null || tag.isEmpty();
if (!clearAll && ts.isEphemeral())
logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " +
"Ephemeral snapshots are not removable by a user.",
tag, ts.keyspaceName);
boolean notEphemeral = !ts.isEphemeral();
boolean shouldClearTag = clearAll || ts.tag.equals(tag);
boolean byTimestamp = true;
if (olderThanTimestamp > 0L)
Set<Descriptor> descriptors = new HashSet<>();
for (SSTableReader ssTableReader : snapshotSSTableReaders)
{
Instant createdAt = ts.getCreatedAt();
if (createdAt != null)
byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp));
descriptors.add(ssTableReader.descriptor);
}
return notEphemeral && shouldClearTag && byTimestamp;
};
return descriptors;
}
catch (IOException e)
{
throw Throwables.unchecked(e);
}
}
public static Refs<SSTableReader> getSnapshotSSTableReaders(String keyspace, String table, String tag) throws IOException
{
return getSnapshotSSTableReaders(Keyspace.open(keyspace).getColumnFamilyStore(table), tag);
}
public static Refs<SSTableReader> getSnapshotSSTableReaders(ColumnFamilyStore cfs, String tag)
{
Map<SSTableId, SSTableReader> active = new HashMap<>();
for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
active.put(sstable.descriptor.id, sstable);
Map<Descriptor, Set<Component>> snapshots = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list();
Refs<SSTableReader> refs = new Refs<>();
try
{
for (Map.Entry<Descriptor, Set<Component>> entries : snapshots.entrySet())
{
// Try to acquire reference to an active sstable instead of snapshot if it exists,
// to avoid opening new sstables. If it fails, use the snapshot reference instead.
SSTableReader sstable = active.get(entries.getKey().id);
if (sstable == null || !refs.tryRef(sstable))
{
if (logger.isTraceEnabled())
logger.trace("using snapshot sstable {}", entries.getKey());
// open offline so we don't modify components or track hotness.
sstable = SSTableReader.open(cfs, entries.getKey(), entries.getValue(), cfs.metadata, true, true);
refs.tryRef(sstable);
// release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released
sstable.selfRef().release();
}
else if (logger.isTraceEnabled())
{
logger.trace("using active sstable {}", entries.getKey());
}
}
}
catch (FSReadError | RuntimeException e)
{
// In case one of the snapshot sstables fails to open,
// we must release the references to the ones we opened so far
refs.release();
throw e;
}
return refs;
}
public static String getTimestampedSnapshotName(String clientSuppliedName, long timestamp)
{
String snapshotName = Long.toString(timestamp);
if (clientSuppliedName != null && !clientSuppliedName.isEmpty())
snapshotName = snapshotName + '-' + clientSuppliedName;
return snapshotName;
}
public static String getTimestampedSnapshotNameWithPrefix(String clientSuppliedName, long timestamp, String prefix)
{
return prefix + '-' + getTimestampedSnapshotName(clientSuppliedName, timestamp);
}
}

310
src/java/org/apache/cassandra/service/snapshot/TakeSnapshotTask.java Normal file
View File

@ -0,0 +1,310 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.io.IOException;
import java.io.PrintStream;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SchemaCQLHelper;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.FBUtilities;
public class TakeSnapshotTask extends AbstractSnapshotTask<List<TableSnapshot>>
{
private static final Logger logger = LoggerFactory.getLogger(TakeSnapshotTask.class);
private final SnapshotManager manager;
private Instant creationTime;
private String snapshotName;
private final Map<ColumnFamilyStore, TableSnapshot> snapshotsToCreate = new HashMap<>();
public TakeSnapshotTask(SnapshotManager manager, SnapshotOptions options)
{
super(options);
this.manager = manager;
}
@Override
public SnapshotTaskType getTaskType()
{
return SnapshotTaskType.SNAPSHOT;
}
public Map<ColumnFamilyStore, TableSnapshot> getSnapshotsToCreate()
{
if (StorageService.instance.operationMode() == StorageService.Mode.JOINING)
throw new RuntimeException("Cannot snapshot until bootstrap completes");
// This is not in builder's build method on purpose in order to postpone the timestamp for as long as possible
// until the actual snapshot is taken. If we constructed a task and have not done anything with it for 5 minutes
// then by the time a snapshot would be taken the creation time would be quite off
Instant creationTimeInOptions = options.creationTime;
if (creationTimeInOptions == null)
creationTime = Instant.ofEpochMilli(Clock.Global.currentTimeMillis());
else
creationTime = options.creationTime;
snapshotName = options.getSnapshotName(creationTime);
Set<ColumnFamilyStore> entitiesForSnapshot = options.cfs == null ? parseEntitiesForSnapshot(options.entities) : Set.of(options.cfs);
for (ColumnFamilyStore cfs : entitiesForSnapshot)
{
Set<File> snapshotDirs = cfs.getDirectories().getSnapshotDirs(snapshotName);
TableSnapshot tableSnapshot = new TableSnapshot(cfs.metadata.keyspace,
cfs.metadata.name,
cfs.metadata.id.asUUID(),
snapshotName,
creationTime,
SnapshotManifest.computeExpiration(options.ttl, creationTime),
snapshotDirs,
options.ephemeral);
// this snapshot does not have any actual representation on disk
// because that snapshot was technically not taken yet
tableSnapshot.incomplete();
snapshotsToCreate.put(cfs, tableSnapshot);
}
return snapshotsToCreate;
}
@Override
public List<TableSnapshot> call()
{
assert snapshotName != null : "You need to call getSnapshotsToCreate() first";
List<TableSnapshot> createdSnapshots = new ArrayList<>();
for (Map.Entry<ColumnFamilyStore, TableSnapshot> entry : snapshotsToCreate.entrySet())
{
try
{
ColumnFamilyStore cfs = entry.getKey();
if (!options.skipFlush)
{
Memtable current = cfs.getTracker().getView().getCurrentMemtable();
if (!current.isClean())
{
if (current.shouldSwitch(ColumnFamilyStore.FlushReason.SNAPSHOT))
FBUtilities.waitOnFuture(cfs.switchMemtableIfCurrent(current, ColumnFamilyStore.FlushReason.SNAPSHOT));
else
current.performSnapshot(snapshotName);
}
}
createSnapshot(cfs, entry.getValue(), snapshotName, creationTime);
createdSnapshots.add(entry.getValue());
}
catch (Throwable t)
{
logger.warn(String.format("Unable to create snapshot %s for %s", entry.getValue().getTag(), entry.getKey().getKeyspaceTableName()), t);
// if we fail to take a snapshot, there is its phantom / in-progress representation among manager's taken snapshots,
// so we need to remove it from there to not appear in the outputs / not leak
manager.getSnapshots().remove(entry.getValue());
}
}
return createdSnapshots;
}
private void createSnapshot(ColumnFamilyStore cfs, TableSnapshot snapshotToCreate, String snapshotName, Instant creationTime)
{
Predicate<SSTableReader> predicate = options.sstableFilter;
Set<SSTableReader> sstables = new LinkedHashSet<>();
for (ColumnFamilyStore aCfs : cfs.concatWithIndexes())
{
try (ColumnFamilyStore.RefViewFragment currentView = aCfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.test(x))))
{
for (SSTableReader ssTable : currentView.sstables)
{
File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName);
ssTable.createLinks(snapshotDirectory.path(), options.rateLimiter); // hard links
if (logger.isTraceEnabled())
logger.trace("Snapshot for {} keyspace data file {} created in {}", cfs.keyspace, ssTable.getFilename(), snapshotDirectory);
sstables.add(ssTable);
}
}
}
List<String> dataComponents = new ArrayList<>();
for (SSTableReader sstable : sstables)
dataComponents.add(sstable.descriptor.relativeFilenameFor(SSTableFormat.Components.DATA));
SnapshotManifest manifest = new SnapshotManifest(dataComponents, options.ttl, creationTime, options.ephemeral);
Set<File> snapshotDirs = cfs.getDirectories().getSnapshotDirs(snapshotName);
for (File snapshotDir : snapshotDirs)
{
writeSnapshotManifest(manifest, Directories.getSnapshotManifestFile(snapshotDir));
if (!SchemaConstants.isLocalSystemKeyspace(cfs.metadata.keyspace)
&& !SchemaConstants.isReplicatedSystemKeyspace(cfs.metadata.keyspace))
{
writeSnapshotSchema(Directories.getSnapshotSchemaFile(snapshotDir), cfs);
}
}
snapshotToCreate.updateMetadataSize();
snapshotToCreate.complete();
}
private Set<ColumnFamilyStore> parseEntitiesForSnapshot(String... entities)
{
Set<ColumnFamilyStore> entitiesForSnapshot = new HashSet<>();
if (entities != null && entities.length > 0 && entities[0].contains("."))
{
for (String entity : entities)
{
String[] splitted = StringUtils.split(entity, '.');
if (splitted.length == 2)
{
String keyspaceName = splitted[0];
String tableName = splitted[1];
if (keyspaceName == null)
throw new RuntimeException("You must supply a keyspace name");
if (tableName == null)
throw new RuntimeException("You must supply a table name");
Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspaceName);
ColumnFamilyStore existingTable = validKeyspace.getColumnFamilyStore(tableName);
entitiesForSnapshot.add(existingTable);
}
// special case for index which we can not normally create a snapshot for
// but a snapshot is apparently taken before a secondary index is scrubbed,
// so we preserve this behavior
else if (splitted.length == 3)
{
String keyspaceName = splitted[0];
String tableName = splitted[1];
Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspaceName);
ColumnFamilyStore existingTable = validKeyspace.getColumnFamilyStore(tableName);
Index indexByName = existingTable.indexManager.getIndexByName(splitted[2]);
if (indexByName instanceof CassandraIndex)
{
ColumnFamilyStore indexCfs = ((CassandraIndex) indexByName).getIndexCfs();
entitiesForSnapshot.add(indexCfs);
}
else
{
throw new IllegalArgumentException("Unknown index " + entity);
}
}
else
{
throw new IllegalArgumentException("Cannot take a snapshot on secondary index or invalid column " +
"family name. You must supply a column family name in the " +
"form of keyspace.columnfamily");
}
}
}
else
{
if (entities != null && entities.length == 0)
{
for (Keyspace keyspace : Keyspace.all())
{
entitiesForSnapshot.addAll(keyspace.getColumnFamilyStores());
}
}
else if (entities != null)
{
for (String keyspace : entities)
{
Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspace);
entitiesForSnapshot.addAll(validKeyspace.getColumnFamilyStores());
}
}
}
return entitiesForSnapshot;
}
private void writeSnapshotManifest(SnapshotManifest manifest, File manifestFile)
{
try
{
manifestFile.parent().tryCreateDirectories();
manifest.serializeToJsonFile(manifestFile);
}
catch (IOException e)
{
throw new FSWriteError(e, manifestFile);
}
}
private void writeSnapshotSchema(File schemaFile, ColumnFamilyStore cfs)
{
try
{
if (!schemaFile.parent().exists())
schemaFile.parent().tryCreateDirectories();
try (PrintStream out = new PrintStream(new FileOutputStreamPlus(schemaFile)))
{
SchemaCQLHelper.reCreateStatementsForSchemaCql(cfs.metadata(), cfs.keyspace.getMetadata())
.forEach(out::println);
}
}
catch (IOException e)
{
throw new FSWriteError(e, schemaFile);
}
}
@Override
public String toString()
{
return "TakeSnapshotTask{" +
"options=" + options +
'}';
}
}

76
src/java/org/apache/cassandra/service/snapshot/TrueSnapshotSizeTask.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.function.Predicate;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
public class TrueSnapshotSizeTask implements Callable<Long>
{
private final SnapshotManager snapshotManager;
private final Predicate<TableSnapshot> predicate;
public TrueSnapshotSizeTask(SnapshotManager snapshotManager, Predicate<TableSnapshot> predicate)
{
this.snapshotManager = snapshotManager;
this.predicate = predicate;
}
@Override
public Long call()
{
long size = 0;
for (TableSnapshot snapshot : snapshotManager.getSnapshots())
{
if (predicate.test(snapshot))
try
{
size += snapshot.computeTrueSizeBytes(getTablesFiles(snapshot.getKeyspaceName(), snapshot.getTableName()));
}
catch (Throwable ex)
{
// if any error happens while computing size, we don't include
return 0L;
}
}
return size;
}
private Set<String> getTablesFiles(String keyspaceName, String tableName)
{
try
{
Keyspace keyspace = Keyspace.getValidKeyspace(keyspaceName);
ColumnFamilyStore table = keyspace.getColumnFamilyStore(tableName);
return table.getFilesOfCfs();
}
catch (IllegalArgumentException ex)
{
// keyspace / table not found
return null;
}
}
}

30
src/java/org/apache/cassandra/tools/NodeProbe.java
View File

@ -102,6 +102,7 @@ import org.apache.cassandra.net.MessagingServiceMBean;
import org.apache.cassandra.service.ActiveRepairServiceMBean;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.service.CacheServiceMBean;
import org.apache.cassandra.service.snapshot.SnapshotManagerMBean;
import org.apache.cassandra.tcm.CMSOperationsMBean;
import org.apache.cassandra.service.GCInspector;
import org.apache.cassandra.service.GCInspectorMXBean;
@ -150,6 +151,7 @@ public class NodeProbe implements AutoCloseable
protected MBeanServerConnection mbeanServerConn;
protected CompactionManagerMBean compactionProxy;
protected StorageServiceMBean ssProxy;
protected SnapshotManagerMBean snapshotProxy;
protected CMSOperationsMBean cmsProxy;
protected GossiperMBean gossProxy;
protected MemoryMXBean memProxy;
@ -263,6 +265,8 @@ public class NodeProbe implements AutoCloseable
{
ObjectName name = new ObjectName(ssObjName);
ssProxy = JMX.newMBeanProxy(mbeanServerConn, name, StorageServiceMBean.class);
name = new ObjectName(SnapshotManagerMBean.MBEAN_NAME);
snapshotProxy = JMX.newMBeanProxy(mbeanServerConn, name, SnapshotManagerMBean.class);
name = new ObjectName(CMSOperations.MBEAN_OBJECT_NAME);
cmsProxy = JMX.newMBeanProxy(mbeanServerConn, name, CMSOperationsMBean.class);
name = new ObjectName(MessagingService.MBEAN_NAME);
@ -875,12 +879,12 @@ public class NodeProbe implements AutoCloseable
public long getSnapshotLinksPerSecond()
{
return ssProxy.getSnapshotLinksPerSecond();
return snapshotProxy.getSnapshotLinksPerSecond();
}
public void setSnapshotLinksPerSecond(long throttle)
{
ssProxy.setSnapshotLinksPerSecond(throttle);
snapshotProxy.setSnapshotLinksPerSecond(throttle);
}
/**
@ -900,10 +904,10 @@ public class NodeProbe implements AutoCloseable
throw new IOException("When specifying the table for a snapshot, you must specify one and only one keyspace");
}
ssProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table);
snapshotProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table);
}
else
ssProxy.takeSnapshot(snapshotName, options, keyspaces);
snapshotProxy.takeSnapshot(snapshotName, options, keyspaces);
}
/**
@ -921,11 +925,11 @@ public class NodeProbe implements AutoCloseable
{
if (null != tableList && tableList.length != 0)
{
ssProxy.takeSnapshot(snapshotName, options, tableList);
snapshotProxy.takeSnapshot(snapshotName, options, tableList);
}
else
{
throw new IOException("The column family List for a snapshot should not be empty or null");
throw new IOException("The column family list for a snapshot should not be empty or null");
}
}
@ -955,12 +959,20 @@ public class NodeProbe implements AutoCloseable
*/
public void clearSnapshot(Map<String, Object> options, String tag, String... keyspaces) throws IOException
{
ssProxy.clearSnapshot(options, tag, keyspaces);
snapshotProxy.clearSnapshot(tag, options, keyspaces);
}
/**
* Gets all snapshots' details.
*
* @param options options to use upon listing of snapshots
* @return details of snapshots
*/
/** @deprecated See CASSANDRA-18111 */
@Deprecated(since = "5.1")
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
{
return ssProxy.getSnapshotDetails(options);
return snapshotProxy.listSnapshots(options);
}
/** @deprecated See CASSANDRA-16789 */
@ -972,7 +984,7 @@ public class NodeProbe implements AutoCloseable
public long trueSnapshotsSize()
{
return ssProxy.trueSnapshotsSize();
return snapshotProxy.getTrueSnapshotSize();
}
public boolean isJoined()

21
src/java/org/apache/cassandra/tools/nodetool/Snapshot.java
View File

@ -29,6 +29,7 @@ import io.airlift.airline.Command;
import io.airlift.airline.Option;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
@ -68,10 +69,10 @@ public class Snapshot extends NodeToolCmd
sb.append("Requested creating snapshot(s) for ");
Map<String, String> options = new HashMap<String,String>();
options.put("skipFlush", Boolean.toString(skipFlush));
options.put(SnapshotOptions.SKIP_FLUSH, Boolean.toString(skipFlush));
if (null != ttl) {
DurationSpec.LongNanosecondsBound d = new DurationSpec.LongNanosecondsBound(ttl);
options.put("ttl", d.toString());
options.put(SnapshotOptions.TTL, d.toString());
}
if (!snapshotName.isEmpty() && snapshotName.contains(File.pathSeparator()))
@ -83,16 +84,16 @@ public class Snapshot extends NodeToolCmd
{
ktList = ktList.replace(" ", "");
if (keyspaces.isEmpty() && null == table)
sb.append("[").append(ktList).append("]");
sb.append('[').append(ktList).append(']');
else
{
throw new IOException(
"When specifying the Keyspace table list (using -kt,--kt-list,-kc,--kc.list), you must not also specify keyspaces to snapshot");
}
if (!snapshotName.isEmpty())
sb.append(" with snapshot name [").append(snapshotName).append("]");
sb.append(" and options ").append(options.toString());
out.println(sb.toString());
sb.append(" with snapshot name [").append(snapshotName).append(']');
sb.append(" and options ").append(options);
out.println(sb);
probe.takeMultipleTableSnapshot(snapshotName, options, ktList.split(","));
out.println("Snapshot directory: " + snapshotName);
}
@ -101,12 +102,12 @@ public class Snapshot extends NodeToolCmd
if (keyspaces.isEmpty())
sb.append("[all keyspaces]");
else
sb.append("[").append(join(keyspaces, ", ")).append("]");
sb.append('[').append(join(keyspaces, ", ")).append(']');
if (!snapshotName.isEmpty())
sb.append(" with snapshot name [").append(snapshotName).append("]");
sb.append(" and options ").append(options.toString());
out.println(sb.toString());
sb.append(" with snapshot name [").append(snapshotName).append(']');
sb.append(" and options ").append(options);
out.println(sb);
probe.takeSnapshot(snapshotName, table, options, toArray(keyspaces, String.class));
out.println("Snapshot directory: " + snapshotName);

26
src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java
View File

@ -27,6 +27,7 @@ import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -39,12 +40,16 @@ import org.apache.cassandra.db.SnapshotCommand;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotType;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS;
@ -194,7 +199,8 @@ public class DiagnosticSnapshotService
}
ColumnFamilyStore cfs = ks.getColumnFamilyStore(command.column_family);
if (cfs.snapshotExists(command.snapshot_name))
if (SnapshotManager.instance.exists(command.keyspace, command.column_family, command.snapshot_name))
{
logger.info("Received diagnostic snapshot request from {} for {}.{}, " +
"but snapshot with tag {} already exists",
@ -210,16 +216,14 @@ public class DiagnosticSnapshotService
command.column_family,
command.snapshot_name);
if (ranges.isEmpty())
cfs.snapshot(command.snapshot_name);
else
{
cfs.snapshot(command.snapshot_name,
(sstable) -> checkIntersection(ranges,
sstable.getFirst().getToken(),
sstable.getLast().getToken()),
false, false);
}
Predicate<SSTableReader> predicate = null;
if (!ranges.isEmpty())
predicate = (sstable) -> checkIntersection(ranges,
sstable.getFirst().getToken(),
sstable.getLast().getToken());
SnapshotOptions options = SnapshotOptions.systemSnapshot(command.snapshot_name, SnapshotType.DIAGNOSTICS, predicate, cfs.getKeyspaceTableName()).build();
SnapshotManager.instance.takeSnapshot(options);
}
catch (IllegalArgumentException e)
{

4
src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java
View File

@ -42,7 +42,7 @@ public class DirectorySizeCalculator extends SimpleFileVisitor<Path>
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
{
if (isAcceptable(file))
size += attrs.size();
@ -63,7 +63,7 @@ public class DirectorySizeCalculator extends SimpleFileVisitor<Path>
/**
* Reset the size to 0 in case that the size calculator is used multiple times
*/
protected void resetSize()
public void resetSize()
{
size = 0;
}

11
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -632,7 +632,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
{
partialStartup(cluster);
}
StorageService.instance.startSnapshotManager();
}
catch (Throwable t)
{
@ -719,6 +718,12 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
CassandraDaemon.getInstanceForTesting().migrateSystemDataIfNeeded();
CommitLog.instance.start();
SnapshotManager.instance.start(false);
SnapshotManager.instance.clearExpiredSnapshots();
SnapshotManager.instance.clearEphemeralSnapshots();
SnapshotManager.instance.resumeSnapshotCleanup();
CassandraDaemon.getInstanceForTesting().runStartupChecks();
Keyspace.setInitialized(); // TODO: this seems to be superfluous by now
@ -891,6 +896,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
Future<?> future = async((ExecutorService executor) -> {
Throwable error = null;
error = parallelRun(error, executor, SnapshotManager.instance::close);
CompactionManager.instance.forceShutdown();
error = parallelRun(error, executor,
@ -945,7 +952,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
() -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())),
() -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES),
() -> EpochAwareDebounce.instance.close(),
() -> SnapshotManager.shutdownAndWait(1L, MINUTES)
SnapshotManager.instance::close
);
internodeMessagingStarted = false;

2
test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java
View File

@ -44,6 +44,7 @@ import org.apache.cassandra.service.CacheServiceMBean;
import org.apache.cassandra.service.GCInspector;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.streaming.StreamManager;
import org.apache.cassandra.tcm.CMSOperations;
import org.apache.cassandra.tools.NodeProbe;
@ -69,6 +70,7 @@ public class InternalNodeProbe extends NodeProbe
StorageService.instance.skipNotificationListeners = !withNotifications;
ssProxy = StorageService.instance;
snapshotProxy = SnapshotManager.instance;
cmsProxy = CMSOperations.instance;
msProxy = MessagingService.instance();
streamProxy = StreamManager.instance;

6
test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java
View File

@ -27,7 +27,7 @@ import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import static org.apache.cassandra.distributed.Cluster.build;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
@ -147,8 +147,8 @@ public class AllowAutoSnapshotTest extends TestBaseImpl
{
final int node = i; // has to be effectively final for the usage in "until" method
await().until(() -> cluster.get(node).appliesOnInstance((IIsolatedExecutor.SerializableTriFunction<Boolean, String, String, Boolean>) (shouldContainSnapshot, tableName, prefix) -> {
Stream<String> stream = StorageService.instance.getSnapshotDetails(Collections.emptyMap()).keySet().stream();
Predicate<String> predicate = tag -> tag.startsWith(prefix + '-') && tag.endsWith('-' + tableName);
Stream<String> stream = SnapshotManager.instance.listSnapshots(Collections.emptyMap()).keySet().stream();
Predicate<String> predicate = tag -> tag.contains(prefix + '-' + table);
return shouldContainSnapshot ? stream.anyMatch(predicate) : stream.noneMatch(predicate);
}).apply(shouldContain, table, snapshotPrefix));
}

24
test/distributed/org/apache/cassandra/distributed/test/AutoSnapshotTtlTest.java
View File

@ -25,16 +25,14 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.service.snapshot.SnapshotType;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_DROP_PREFIX;
import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_TRUNCATE_PREFIX;
import static org.apache.cassandra.distributed.Cluster.build;
import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
import static org.awaitility.Awaitility.await;
@ -81,12 +79,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl
cluster.schemaChange(withKeyspace("TRUNCATE %s.tbl;"));
// Check snapshot is listed after table is truncated
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_TRUNCATE_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label);
// Check snapshot is removed after 10s
await().timeout(10, SECONDS)
.pollInterval(1, SECONDS)
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX));
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label));
}
}
@ -111,12 +109,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl
cluster.schemaChange(withKeyspace("DROP TABLE %s.tbl;"));
// Check snapshot is listed after table is dropped
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label);
// Check snapshot is removed after 10s
await().timeout(10, SECONDS)
.pollInterval(1, SECONDS)
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX));
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label));
}
}
@ -146,12 +144,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl
instance.startup();
// Check snapshot is listed after restart
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label);
// Check snapshot is removed after at most auto_snapshot_ttl + 1s
await().timeout(ONE_MINUTE + 1, SECONDS)
.pollInterval(1, SECONDS)
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX));
.until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label));
}
}
@ -178,13 +176,13 @@ public class AutoSnapshotTtlTest extends TestBaseImpl
cluster.schemaChange(withKeyspace("DROP TABLE %s.tbl;"));
// Check snapshots are created after table is truncated and dropped
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_TRUNCATE_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label);
// Check snapshot are *NOT* expired after 10s
Thread.sleep(2 * FIVE_SECONDS * 1000L);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(ColumnFamilyStore.SNAPSHOT_TRUNCATE_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(ColumnFamilyStore.SNAPSHOT_DROP_PREFIX);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label);
instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label);
}
}

9
test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java
View File

@ -30,6 +30,7 @@ import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.utils.Pair;
@ -61,6 +62,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl
rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true));
verify(c.get(1));
}
}
@ -88,6 +91,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl
Files.createFile(ephemeralMarkerFile);
c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true));
verify(c.get(1));
}
}
@ -104,6 +109,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl
Pair<String, String[]> initialisationData = initialise(c);
rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true));
assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
instance.nodetoolResult("clearsnapshot", "-t", snapshotName).asserts().success();
// ephemeral snapshot was not removed as it can not be (from nodetool / user operation)
@ -127,6 +134,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl
rewriteManifestToEphemeral(initialisationData.left, initialisationData.right);
c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true));
instance.nodetoolResult("clearsnapshot", "--all").asserts().success();
assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
assertFalse(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName2));

5
test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java
View File

@ -36,6 +36,7 @@ import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.concurrent.Refs;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@ -126,7 +127,7 @@ public class PreviewRepairSnapshotTest extends TestBaseImpl
String snapshotTag = await().atMost(1, MINUTES)
.pollInterval(100, MILLISECONDS)
.until(() -> {
for (String tag : cfs.listSnapshots().keySet())
for (String tag : Util.listSnapshots(cfs).keySet())
{
// we create the snapshot schema file last, so when this exists we know the snapshot is complete;
if (cfs.getDirectories().getSnapshotSchemaFile(tag).exists())
@ -138,7 +139,7 @@ public class PreviewRepairSnapshotTest extends TestBaseImpl
Set<SSTableReader> inSnapshot = new HashSet<>();
try (Refs<SSTableReader> sstables = cfs.getSnapshotSSTableReaders(snapshotTag))
try (Refs<SSTableReader> sstables = TableSnapshot.getSnapshotSSTableReaders(cfs.getKeyspaceName(), cfs.name, snapshotTag))
{
inSnapshot.addAll(sstables);
}

5
test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
View File

@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.Util;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.concurrent.Condition;
import org.junit.BeforeClass;
@ -434,11 +435,11 @@ public class PreviewRepairTest extends TestBaseImpl
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
if(shouldBeEmpty)
{
assertTrue(cfs.listSnapshots().isEmpty());
assertTrue(Util.listSnapshots(cfs).isEmpty());
}
else
{
while (cfs.listSnapshots().isEmpty())
while (Util.listSnapshots(cfs).isEmpty())
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
}
}));

11
test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
View File

@ -30,6 +30,8 @@ import java.util.stream.Stream;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.Assert;
import org.apache.cassandra.Util;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.dht.Token;
@ -39,6 +41,7 @@ import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaLayout;
import org.apache.cassandra.locator.ReplicaUtils;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.Throwables;
import org.junit.Test;
@ -591,7 +594,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl
int attempts = 100;
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
while (cfs.listSnapshots().isEmpty())
while (Util.listSnapshots(cfs).isEmpty())
{
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
if (attempts-- < 0)
@ -602,11 +605,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl
private IInvokableInstance.SerializableRunnable assertSnapshotNotPresent(String snapshotName)
{
return () ->
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
Assert.assertFalse(cfs.snapshotExists(snapshotName));
};
return () -> Assert.assertFalse(SnapshotManager.instance.exists(KEYSPACE, TABLE, snapshotName));
}
private long getConfirmedInconsistencies(IInvokableInstance instance)

27
test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java
View File

@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
@ -51,6 +52,9 @@ import org.apache.cassandra.io.sstable.SequenceBasedSSTableId;
import org.apache.cassandra.io.sstable.UUIDBasedSSTableId;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.metrics.RestorableMeter;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.tools.SystemExitException;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
@ -60,7 +64,6 @@ import static java.lang.String.format;
import static org.apache.cassandra.Util.bulkLoadSSTables;
import static org.apache.cassandra.Util.getBackups;
import static org.apache.cassandra.Util.getSSTables;
import static org.apache.cassandra.Util.getSnapshots;
import static org.apache.cassandra.Util.relativizePath;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.db.SystemKeyspace.LEGACY_SSTABLE_ACTIVITY;
@ -404,12 +407,20 @@ public class SSTableIdGenerationTest extends TestBaseImpl
private static Set<String> snapshot(IInvokableInstance instance, String ks, String tableName)
{
Set<String> snapshotDirs = instance.callOnInstance(() -> ColumnFamilyStore.getIfExists(ks, tableName)
.snapshot(SNAPSHOT_TAG)
.getDirectories()
.stream()
.map(File::toString)
.collect(Collectors.toSet()));
Set<String> snapshotDirs = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ks, tableName);
if (cfs == null)
return Set.of();
TableSnapshot tableSnapshot = SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(SNAPSHOT_TAG, cfs.getKeyspaceTableName())).iterator().next();
Set<String> dirs = new HashSet<>();
for (File dir : tableSnapshot.getDirectories())
dirs.add(dir.toString());
return dirs;
});
assertThat(snapshotDirs).isNotEmpty();
return snapshotDirs;
}
@ -458,7 +469,7 @@ public class SSTableIdGenerationTest extends TestBaseImpl
private static void assertSnapshotSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames)
{
instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getSnapshots(ks, tableName, SNAPSHOT_TAG), tableName, expectedSeqGenIds, expectedUUIDGenIds))));
instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(TableSnapshot.getSnapshotDescriptors(ks, tableName, SNAPSHOT_TAG), tableName, expectedSeqGenIds, expectedUUIDGenIds))));
}
private static void assertBackupSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames)

109
test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java
View File

@ -19,6 +19,9 @@
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
@ -29,9 +32,11 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableCallable;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.utils.Clock;
@ -40,8 +45,10 @@ import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class SnapshotsTest extends TestBaseImpl
{
@ -51,9 +58,9 @@ public class SnapshotsTest extends TestBaseImpl
private static final WithProperties properties = new WithProperties();
private static Cluster cluster;
private final String[] exoticSnapshotNames = new String[] { "snapshot", "snapshots", "backup", "backups",
"Snapshot", "Snapshots", "Backups", "Backup",
"snapshot.with.dots-and-dashes"};
private final String[] exoticSnapshotNames = new String[]{ "snapshot", "snapshots", "backup", "backups",
"Snapshot", "Snapshots", "Backups", "Backup",
"snapshot.with.dots-and-dashes" };
@BeforeClass
public static void before() throws IOException
@ -61,15 +68,18 @@ public class SnapshotsTest extends TestBaseImpl
properties.set(CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS, 0);
properties.set(CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS, SNAPSHOT_CLEANUP_PERIOD_SECONDS);
properties.set(CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS, FIVE_SECONDS);
cluster = init(Cluster.build(1).start());
cluster = init(Cluster.build(1)
.withDataDirCount(3)
.start());
}
@After
public void clearAllSnapshots()
{
cluster.schemaChange(withKeyspace("DROP TABLE IF EXISTS %s.tbl;"));
cluster.schemaChange(withKeyspace("DROP TABLE IF EXISTS %s.tbl2;"));
cluster.get(1).nodetoolResult("clearsnapshot", "--all").asserts().success();
for (String tag : new String[] {"basic", "first", "second", "tag1"})
for (String tag : new String[]{ "basic", "first", "second", "tag1" })
waitForSnapshotCleared(tag);
for (String tag : exoticSnapshotNames)
waitForSnapshot(tag, false, true);
@ -83,6 +93,34 @@ public class SnapshotsTest extends TestBaseImpl
cluster.close();
}
@Test
public void testEverySnapshotDirHasManifestAndSchema()
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))"));
String[] dataDirs = (String[]) cluster.get(1).config().get("data_file_directories");
String tableId = cluster.get(1).callOnInstance((SerializableCallable<String>) () -> {
return ColumnFamilyStore.getIfExists("distributed_test_keyspace", "tbl").metadata().id.toHexString();
});
cluster.get(1)
.nodetoolResult("snapshot", "-t", "mysnapshot", "-kt", format("%s.tbl", KEYSPACE))
.asserts()
.success();
for (String dataDir : dataDirs)
{
Path snapshotDir = Paths.get(dataDir)
.resolve(KEYSPACE)
.resolve("tbl-" + tableId)
.resolve("snapshots")
.resolve("mysnapshot");
assertTrue(snapshotDir.toFile().exists());
assertTrue(snapshotDir.resolve("manifest.json").toFile().exists());
assertTrue(snapshotDir.resolve("schema.cql").toFile().exists());
}
}
@Test
public void testSnapshotsCleanupByTTL()
{
@ -161,7 +199,8 @@ public class SnapshotsTest extends TestBaseImpl
}
@Test
public void testManualSnapshotCleanup() {
public void testManualSnapshotCleanup()
{
// take snapshots with ttl
cluster.get(1).nodetoolResult("snapshot", "--ttl",
format("%ds", TEN_SECONDS),
@ -210,8 +249,8 @@ public class SnapshotsTest extends TestBaseImpl
populate(cluster);
instance.nodetoolResult("snapshot",
"-t", "tag1",
"-kt", withKeyspace("%s.tbl")).asserts().success();
"-t", "tag1",
"-kt", withKeyspace("%s.tbl")).asserts().success();
// Check snapshot is listed when table is not dropped
waitForSnapshotPresent("tag1");
@ -314,21 +353,47 @@ public class SnapshotsTest extends TestBaseImpl
Pattern COMPILE = Pattern.compile(" +");
long distinctTimestamps = Arrays.stream(result.getStdout().split("\n"))
.filter(line -> line.startsWith("sametimestamp"))
.map(line -> COMPILE.matcher(line).replaceAll(" ").split(" ")[7])
.distinct()
.count();
.filter(line -> line.startsWith("sametimestamp"))
.map(line -> COMPILE.matcher(line).replaceAll(" ").split(" ")[7])
.distinct()
.count();
// assert all dates are same so there is just one value accross all individual tables
assertEquals(1, distinctTimestamps);
}
@Test
public void testFailureToSnapshotTwiceOnSameEntityWithSameSnapshotName()
{
cluster.get(1).nodetoolResult("snapshot", "-t", "somename").asserts().success();
NodeToolResult failedSnapshotResult = cluster.get(1).nodetoolResult("snapshot", "-t", "somename");
failedSnapshotResult.asserts().failure();
Throwable error = failedSnapshotResult.getError();
assertThat(error.getMessage()).contains("already exists");
}
@Test
public void testTakingSnapshoWithSameNameOnDifferentTablesDoesNotFail()
{
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))"));
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl2 (key int, value text, PRIMARY KEY (key))"));
cluster.get(1).nodetoolResult("snapshot", "-t", "somename", "-kt", String.format("%s.tbl", KEYSPACE)).asserts().success();
cluster.get(1).nodetoolResult("snapshot", "-t", "somename", "-kt", String.format("%s.tbl2", KEYSPACE)).asserts().success();
}
private void populate(Cluster cluster)
{
for (int i = 0; i < 100; i++)
cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (key, value) VALUES (?, 'txt')"), ConsistencyLevel.ONE, i);
}
private void populate(Cluster cluster, String keyspace, String table)
{
for (int i = 0; i < 100; i++)
cluster.coordinator(1).execute(format("INSERT INTO %s.%s (key, value) VALUES (?, 'txt')", keyspace, table), ConsistencyLevel.ONE, i);
}
private void waitForSnapshotPresent(String snapshotName)
{
waitForSnapshot(snapshotName, true, false);
@ -339,20 +404,28 @@ public class SnapshotsTest extends TestBaseImpl
waitForSnapshot(snapshotName, false, false);
}
private void waitForSnapshot(String snapshotName, boolean expectPresent, boolean noTTL)
private void waitForSnapshot(String keyspaceName, String tableName, String snapshotName, boolean expectPresent, boolean noTTL)
{
await().timeout(20, SECONDS)
.pollDelay(0, SECONDS)
.pollInterval(1, SECONDS)
.until(() -> waitForSnapshotInternal(snapshotName, expectPresent, noTTL));
.until(() -> waitForSnapshotInternal(keyspaceName, tableName, snapshotName, expectPresent, noTTL));
}
private boolean waitForSnapshotInternal(String snapshotName, boolean expectPresent, boolean noTTL) {
private void waitForSnapshot(String snapshotName, boolean expectPresent, boolean noTTL)
{
waitForSnapshot(null, null, snapshotName, expectPresent, noTTL);
}
private boolean waitForSnapshotInternal(String keyspaceName, String tableName, String snapshotName, boolean expectPresent, boolean noTTL)
{
List<String> args = new ArrayList<>();
args.add("listsnapshots");
NodeToolResult listsnapshots;
if (noTTL)
listsnapshots = cluster.get(1).nodetoolResult("listsnapshots", "-nt");
else
listsnapshots = cluster.get(1).nodetoolResult("listsnapshots");
args.add("-nt");
listsnapshots = cluster.get(1).nodetoolResult(args.toArray(new String[0]));
List<String> lines = Arrays.stream(listsnapshots.getStdout().split("\n"))
.filter(line -> !line.isEmpty())

1251
test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java Normal file
View File

File diff suppressed because it is too large Load Diff

167
test/microbench/org/apache/cassandra/test/microbench/AbstractSnapshotManagerBase.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.RowUpdateBuilder;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.tcm.ClusterMetadataService;
import static org.apache.cassandra.service.snapshot.SnapshotOptions.userSnapshot;
import static org.junit.Assert.assertEquals;
public abstract class AbstractSnapshotManagerBase
{
static final String KEYSPACE = "KEYSPACE";
private final Random random = new Random();
static int NUM_SSTABLES = 10;
static int NUM_KEYSPACES = 5;
static int NUM_TABLES_PER_KEYSPACE = 10;
static int NUM_SNAPSHOTS_PER_TABLE = 10;
public void teardown()
{
SnapshotManager.instance.clearAllSnapshots();
CQLTester.tearDownClass();
CommitLog.instance.stopUnsafe(true);
// TODO CASSANDRA-20119
ClusterMetadataService.instance().log().close();
CQLTester.cleanup();
}
public void setup(boolean takeSnapshotAfterTablePopulation)
{
CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(1000);
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(60);
DatabaseDescriptor.daemonInitialization(() -> {
Config config = DatabaseDescriptor.loadConfig();
config.dump_heap_on_uncaught_exception = false;
return config;
});
SchemaLoader.prepareServer();
SnapshotManager.instance.start(true);
populateTables(takeSnapshotAfterTablePopulation);
}
public void populateTables(boolean takeSnapshotAfterTablePopulation)
{
for (int i = 0; i < NUM_KEYSPACES; i++)
{
String keyspaceName = KEYSPACE + '_' + i;
// Create Schema
TableMetadata[] tables = new TableMetadata[NUM_TABLES_PER_KEYSPACE];
for (int j = 0; j < NUM_TABLES_PER_KEYSPACE; j++)
{
tables[j] = SchemaLoader.standardCFMD(keyspaceName, String.format("table%d", i + '_' + j)).build();
}
SchemaLoader.createKeyspace(keyspaceName,
KeyspaceParams.simple(1),
tables);
for (int j = 0; j < NUM_TABLES_PER_KEYSPACE; j++)
{
String tableName = String.format("table%d", i + '_' + j);
ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName);
cfs.disableAutoCompaction();
for (int k = 0; k < NUM_SSTABLES; k++)
{
new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
Util.flush(cfs);
}
if (takeSnapshotAfterTablePopulation)
{
for (int snapId = 0; snapId < NUM_SNAPSHOTS_PER_TABLE; snapId++)
SnapshotManager.instance.takeSnapshot(userSnapshot(String.format("snap_%d_%d_%d", i, j, snapId), cfs.getKeyspaceTableName()));
}
}
}
if (takeSnapshotAfterTablePopulation)
{
List<TableSnapshot> snapshots = SnapshotManager.instance.getSnapshots(t -> true);
assertEquals(NUM_KEYSPACES * NUM_TABLES_PER_KEYSPACE * NUM_SNAPSHOTS_PER_TABLE, snapshots.size());
}
}
public String getRandomKeyspaceTable()
{
List<Keyspace> keyspaces = new ArrayList<>();
Keyspace.nonSystem().forEach(keyspaces::add);
Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size()));
List<ColumnFamilyStore> cfss = new ArrayList<>(randomKeyspace.getColumnFamilyStores());
ColumnFamilyStore cfs = cfss.get(random.nextInt(cfss.size()));
return cfs.getKeyspaceTableName();
}
public String getRandomKeyspace()
{
List<Keyspace> keyspaces = new ArrayList<>();
Keyspace.nonSystem().forEach(keyspaces::add);
Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size()));
return randomKeyspace.getName();
}
public Pair<String, String> getRandomKeyspaceTablePair()
{
List<Keyspace> keyspaces = new ArrayList<>();
Keyspace.nonSystem().forEach(keyspaces::add);
Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size()));
List<ColumnFamilyStore> cfss = new ArrayList<>(randomKeyspace.getColumnFamilyStores());
ColumnFamilyStore cfs = cfss.get(random.nextInt(cfss.size()));
return Pair.of(cfs.getKeyspaceName(), cfs.getTableName());
}
public void takeSnapshotOnRandomTable()
{
SnapshotManager.instance.takeSnapshot(userSnapshot("snap_" + UUID.randomUUID(), getRandomKeyspaceTable()));
}
}

4
test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
View File

@ -30,6 +30,7 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.openjdk.jmh.annotations.*;
@BenchmarkMode(Mode.AverageTime)
@ -78,8 +79,7 @@ public class CompactionBench extends CQLTester
cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
cfs.snapshot("originals");
SnapshotManager.instance.takeSnapshot("originals", cfs.getKeyspaceTableName());
snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles();
}

63
test/microbench/org/apache/cassandra/test/microbench/SnapshotListingBench.java Normal file
View File

@ -0,0 +1,63 @@
package org.apache.cassandra.test.microbench;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Fork(value = 1)
@State(Scope.Benchmark)
public class SnapshotListingBench extends AbstractSnapshotManagerBase
{
@Setup(Level.Trial)
public void setup()
{
setup(true);
}
@TearDown(Level.Trial)
public void teardown()
{
super.teardown();
}
@Benchmark
@BenchmarkMode(Mode.All)
@Warmup(iterations = 0)
@Threads(4)
@Measurement(iterations = 1, time = 60)
public void listingTest()
{
SnapshotManager.instance.listSnapshots(null);
}
}

64
test/microbench/org/apache/cassandra/test/microbench/SnapshotTakingBench.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench;
import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Fork(value = 1)
@State(Scope.Benchmark)
public class SnapshotTakingBench extends AbstractSnapshotManagerBase
{
@Setup(Level.Trial)
public void setup()
{
setup(false);
}
@TearDown(Level.Trial)
public void teardown()
{
super.teardown();
}
@Benchmark
@BenchmarkMode(Mode.All)
@Warmup(iterations = 0)
@Threads(4)
@Measurement(iterations = 1, time = 60)
public void takingSnapshotTest()
{
takeSnapshotOnRandomTable();
}
}

65
test/microbench/org/apache/cassandra/test/microbench/SnapshotTrueSizeBench.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.test.microbench;
import java.util.concurrent.TimeUnit;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Threads;
import org.openjdk.jmh.annotations.Warmup;
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Fork(value = 1)
@State(Scope.Benchmark)
public class SnapshotTrueSizeBench extends AbstractSnapshotManagerBase
{
@Setup(Level.Trial)
public void setup()
{
setup(true);
}
@TearDown(Level.Trial)
public void teardown()
{
super.teardown();
}
@Benchmark
@BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime})
@Warmup(iterations = 0)
@Threads(16)
@Measurement(iterations = 1, time = 60)
public void getTrueSizeOfSnapshots()
{
SnapshotManager.instance.getTrueSnapshotSize();
}
}

34
test/unit/org/apache/cassandra/Util.java
View File

@ -35,8 +35,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
@ -133,6 +135,8 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.pager.PagingState;
import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -147,7 +151,6 @@ import org.apache.cassandra.utils.CounterId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.FilterFactory;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Throwables;
import org.awaitility.Awaitility;
import org.hamcrest.Matcher;
import org.mockito.Mockito;
@ -1170,23 +1173,6 @@ public class Util
.collect(Collectors.toSet());
}
public static Set<Descriptor> getSnapshots(String ks, String tableName, String snapshotTag)
{
try
{
return Keyspace.open(ks)
.getColumnFamilyStore(tableName)
.getSnapshotSSTableReaders(snapshotTag)
.stream()
.map(sstr -> sstr.descriptor)
.collect(Collectors.toSet());
}
catch (IOException e)
{
throw Throwables.unchecked(e);
}
}
public static Set<Descriptor> getBackups(String ks, String tableName)
{
return Keyspace.open(ks)
@ -1283,4 +1269,16 @@ public class Util
{
return new UnsupportedOperationException("Test must be implemented for sstable format " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName());
}
public static Map<String, TableSnapshot> listSnapshots(ColumnFamilyStore cfs)
{
Set<TableSnapshot> snapshots = new SnapshotLoader(cfs.getDirectories()).loadSnapshots();
Map<String, TableSnapshot> tagSnapshotsMap = new HashMap<>();
for (TableSnapshot snapshot : snapshots)
tagSnapshotsMap.put(snapshot.getTag(), snapshot);
return tagSnapshotsMap;
}
}

2
test/unit/org/apache/cassandra/cql3/CQLTester.java
View File

@ -172,6 +172,7 @@ import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.QueryState;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.transport.Event;
import org.apache.cassandra.transport.Message;
@ -445,6 +446,7 @@ public abstract class CQLTester
DatabaseDescriptor.setRowCacheSizeInMiB(ROW_CACHE_SIZE_IN_MIB);
StorageService.instance.registerMBeans();
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
SnapshotManager.instance.registerMBean();
}
@AfterClass

29
test/unit/org/apache/cassandra/cql3/validation/operations/AutoSnapshotTest.java
View File

@ -30,21 +30,28 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.assertj.core.api.Condition;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.lang.String.format;
import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_DROP_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
@RunWith(Parameterized.class)
public class AutoSnapshotTest extends CQLTester
{
static
{
CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.setInt(1);
}
static int TTL_SECS = 1;
public static Boolean enabledBefore;
@ -97,7 +104,7 @@ public class AutoSnapshotTest extends CQLTester
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
// Check there are no snapshots
ColumnFamilyStore tableDir = getCurrentColumnFamilyStore();
assertThat(tableDir.listSnapshots()).isEmpty();
assertThat(Util.listSnapshots(tableDir)).isEmpty();
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
@ -106,7 +113,7 @@ public class AutoSnapshotTest extends CQLTester
execute("DROP TABLE %s");
verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableDir, currentTable());
verifyAutoSnapshot(SnapshotType.DROP.label, tableDir, currentTable());
}
@Test
@ -115,7 +122,7 @@ public class AutoSnapshotTest extends CQLTester
createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))");
// Check there are no snapshots
ColumnFamilyStore tableDir = getCurrentColumnFamilyStore();
assertThat(tableDir.listSnapshots()).isEmpty();
assertThat(Util.listSnapshots(tableDir)).isEmpty();
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0);
execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1);
@ -124,7 +131,7 @@ public class AutoSnapshotTest extends CQLTester
execute("DROP TABLE %s");
verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableDir, currentTable());
verifyAutoSnapshot(SnapshotType.DROP.label, tableDir, currentTable());
}
@Test
@ -136,13 +143,13 @@ public class AutoSnapshotTest extends CQLTester
flush();
// Check no snapshots
assertThat(tableA.listSnapshots()).isEmpty();
assertThat(tableB.listSnapshots()).isEmpty();
assertThat(Util.listSnapshots(tableA)).isEmpty();
assertThat(Util.listSnapshots(tableB)).isEmpty();
// Drop keyspace, should have snapshot for table A and B
execute(format("DROP KEYSPACE %s", keyspace()));
verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableA, tableA.name);
verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableB, tableB.name);
verifyAutoSnapshot(SnapshotType.DROP.label, tableA, tableA.name);
verifyAutoSnapshot(SnapshotType.DROP.label, tableB, tableB.name);
}
private ColumnFamilyStore createAndPopulateTable() throws Throwable
@ -163,11 +170,11 @@ public class AutoSnapshotTest extends CQLTester
*/
private void verifyAutoSnapshot(String snapshotPrefix, ColumnFamilyStore tableDir, String expectedTableName)
{
Map<String, TableSnapshot> snapshots = tableDir.listSnapshots();
Map<String, TableSnapshot> snapshots = Util.listSnapshots(tableDir);
if (autoSnapshotEnabled)
{
assertThat(snapshots).hasSize(1);
assertThat(snapshots).hasKeySatisfying(new Condition<>(k -> k.startsWith(snapshotPrefix), "is dropped snapshot"));
assertThat(snapshots).hasKeySatisfying(new Condition<>(k -> k.contains(snapshotPrefix), "is dropped snapshot"));
TableSnapshot snapshot = snapshots.values().iterator().next();
assertThat(snapshot.getTableName()).isEqualTo(expectedTableName);
if (autoSnapshotTTl == null)

287
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
View File

@ -1,21 +1,21 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cassandra.db;
import java.io.IOException;
@ -23,18 +23,19 @@ import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@ -44,7 +45,9 @@ import com.googlecode.concurrenttrees.common.Iterables;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.ColumnFamilyStore.FlushReason;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.filter.ColumnFilter;
@ -59,7 +62,9 @@ import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
@ -71,11 +76,17 @@ import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.ClearableHistogram;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.schema.Indexes;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.SnapshotType;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@ -83,6 +94,10 @@ import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.cassandra.utils.concurrent.OpOrder.Barrier;
import org.apache.cassandra.utils.concurrent.OpOrder.Group;
import static org.apache.cassandra.cql3.statements.schema.IndexTarget.Type.VALUES;
import static org.apache.cassandra.schema.IndexMetadata.Kind.COMPOSITES;
import static org.apache.cassandra.schema.IndexMetadata.Kind.CUSTOM;
import static org.apache.cassandra.schema.IndexMetadata.fromIndexTargets;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -121,14 +136,27 @@ public class ColumnFamilyStoreTest
@Before
public void truncateCFS()
{
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlocking();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1).truncateBlocking();
Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlocking();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlockingWithoutSnapshot();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlockingWithoutSnapshot();
Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1).truncateBlockingWithoutSnapshot();
Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlockingWithoutSnapshot();
Keyspace.open(KEYSPACE3).getColumnFamilyStore(CF_SPEC_RETRY1).truncateBlockingWithoutSnapshot();
SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_STANDARD1);
SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_STANDARD2);
SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_INDEX1);
SnapshotManager.instance.clearAllSnapshots(KEYSPACE2, CF_STANDARD1);
SnapshotManager.instance.clearAllSnapshots(KEYSPACE3, CF_SPEC_RETRY1);
}
@After
public void afterTest()
{
SnapshotManager.instance.clearAllSnapshots();
}
@Test
public void testMemtableTimestamp() throws Throwable
public void testMemtableTimestamp()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
assertEquals(Memtable.NO_MIN_TIMESTAMP, fakeMemTableWithMinTS(cfs, EncodingStats.NO_STATS.minTimestamp).getMinTimestamp());
@ -142,20 +170,20 @@ public class ColumnFamilyStoreTest
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
Util.flush(cfs);
new RowUpdateBuilder(cfs.metadata(), 1, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
Util.flush(cfs);
((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts
((ClearableHistogram) cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts
Util.getAll(Util.cmd(cfs, "key1").includeRow("c1").build());
assertEquals(1, cfs.metric.sstablesPerReadHistogram.cf.getCount());
}
@ -165,7 +193,7 @@ public class ColumnFamilyStoreTest
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1);
keyspace.getColumnFamilyStores().forEach(ColumnFamilyStore::truncateBlocking);
keyspace.getColumnFamilyStores().forEach(ColumnFamilyStore::truncateBlockingWithoutSnapshot);
List<Mutation> rms = new LinkedList<>();
rms.add(new RowUpdateBuilder(cfs.metadata(), 0, "key1")
@ -195,7 +223,7 @@ public class ColumnFamilyStoreTest
{
Row toCheck = Util.getOnlyRowUnfiltered(Util.cmd(cfs, "key1").build());
Iterator<Cell<?>> iter = toCheck.cells().iterator();
assert(Iterators.size(iter) == 0);
assert (Iterators.size(iter) == 0);
}
};
@ -265,42 +293,34 @@ public class ColumnFamilyStoreTest
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1);
//cleanup any previous test gargbage
cfs.clearSnapshot("");
int numRows = 1000;
long[] colValues = new long [numRows * 2]; // each row has two columns
for (int i = 0; i < colValues.length; i+=2)
long[] colValues = new long[numRows * 2]; // each row has two columns
for (int i = 0; i < colValues.length; i += 2)
{
colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column
colValues[i+1] = 3L; //other column
colValues[i + 1] = 3L; //other column
}
ScrubTest.fillIndexCF(cfs, false, colValues);
cfs.snapshot("nonEphemeralSnapshot", null, false, false);
cfs.snapshot("ephemeralSnapshot", null, true, false);
SnapshotManager.instance.takeSnapshot(SnapshotOptions.systemSnapshot("nonEphemeralSnapshot", SnapshotType.MISC, cfs.getKeyspaceTableName()).build());
SnapshotManager.instance.takeSnapshot(SnapshotOptions.systemSnapshot("ephemeralSnapshot", SnapshotType.REPAIR, cfs.getKeyspaceTableName()).ephemeral().build());
Map<String, TableSnapshot> snapshotDetails = cfs.listSnapshots();
assertEquals(2, snapshotDetails.size());
assertTrue(snapshotDetails.containsKey("ephemeralSnapshot"));
assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot"));
assertTrue(SnapshotManager.instance.exists(p -> p.getTag().endsWith("ephemeralSnapshot")));
assertTrue(SnapshotManager.instance.exists(p -> p.getTag().endsWith("nonEphemeralSnapshot")));
ColumnFamilyStore.clearEphemeralSnapshots(cfs.getDirectories());
SnapshotManager.instance.clearEphemeralSnapshots();
snapshotDetails = cfs.listSnapshots();
assertEquals(1, snapshotDetails.size());
assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot"));
List<TableSnapshot> snapshots = SnapshotManager.instance.getSnapshots(p -> true);
assertEquals(1, snapshots.size());
//test cleanup
cfs.clearSnapshot("");
assertTrue(snapshots.get(0).getTag().endsWith("nonEphemeralSnapshot"));
}
@Test
public void testSnapshotSize() throws IOException
{
// cleanup any previous test gargbage
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
cfs.clearSnapshot("");
// Add row
new RowUpdateBuilder(cfs.metadata(), 0, "key1")
@ -311,10 +331,10 @@ public class ColumnFamilyStoreTest
Util.flush(cfs);
// snapshot
cfs.snapshot("basic", null, false, false);
SnapshotManager.instance.takeSnapshot("basic", cfs.getKeyspaceTableName());
// check snapshot was created
Map<String, TableSnapshot> snapshotDetails = cfs.listSnapshots();
Map<String, TableSnapshot> snapshotDetails = Util.listSnapshots(cfs);
assertThat(snapshotDetails).hasSize(1);
assertThat(snapshotDetails).containsKey("basic");
@ -329,13 +349,121 @@ public class ColumnFamilyStoreTest
// sizeOnDisk > trueSize because trueSize does not include manifest.json
// Check that truesize now is > 0
snapshotDetails = cfs.listSnapshots();
snapshotDetails = Util.listSnapshots(cfs);
details = snapshotDetails.get("basic");
assertThat(details.computeSizeOnDiskBytes()).isEqualTo(details.computeTrueSizeBytes());
}
@Test
public void testBackupAfterFlush() throws Throwable
public void testTrueSnapshotSizeWithLegacyIndex() throws Throwable
{
testTrueSnapshotSizeInternal("snapshot_sizes_legacy", "true_snapshot_size_with_legacy_index", false);
}
@Test
public void testTrueSnapshotSizeWithSaiIndex() throws Throwable
{
testTrueSnapshotSizeInternal("snapshot_sizes_sai","true_snapshot_size_with_sai_index", true);
}
private void testTrueSnapshotSizeInternal(String keyspace, String table, boolean forSai) throws Throwable
{
TableMetadata tableMetadata = SchemaLoader.standardCFMD(keyspace, table).partitioner(Murmur3Partitioner.instance).build();
SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), tableMetadata);
ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "value1")
.build()
.applyUnsafe();
Util.flush(cfs);
assertThat(cfs.trueSnapshotsSize()).isZero();
SnapshotManager.instance.takeSnapshot("snapshot_without_index", cfs.getKeyspaceTableName());
long firstSnapshotsSize = cfs.trueSnapshotsSize();
Map<String, TableSnapshot> listedSnapshots = Util.listSnapshots(cfs);
assertThat(firstSnapshotsSize).isPositive();
assertThat(listedSnapshots.size()).isEqualTo(1);
assertThat(listedSnapshots.get("snapshot_without_index")).isNotNull();
long withoutIndexSize = listedSnapshots.get("snapshot_without_index").computeSizeOnDiskBytes();
long withoutIndexTrueSize = listedSnapshots.get("snapshot_without_index").computeTrueSizeBytes();
assertThat(withoutIndexSize).isGreaterThan(withoutIndexTrueSize);
assertEquals(firstSnapshotsSize, withoutIndexTrueSize);
// add index, trueSnapshotSize should reflect that
ColumnIdentifier col = ColumnIdentifier.getInterned("val", true);
IndexMetadata indexMetadata;
if (forSai)
indexMetadata = fromIndexTargets(List.of(new IndexTarget(col, VALUES)), "idx", CUSTOM, Map.of("class_name", "sai"));
else
indexMetadata = fromIndexTargets(List.of(new IndexTarget(col, VALUES)), "idx", COMPOSITES, Map.of());
TableMetadata tableMetadataWithIndex = SchemaLoader.standardCFMD(keyspace, table)
.partitioner(Murmur3Partitioner.instance)
.id(tableMetadata.id)
.indexes(Indexes.of(indexMetadata))
.build();
SchemaTestUtil.announceTableUpdate(tableMetadataWithIndex);
rebuildIndices(cfs);
SnapshotManager.instance.takeSnapshot("snapshot_with_index", new HashMap<>(), cfs.getKeyspaceTableName());
long secondSnapshotSize = cfs.trueSnapshotsSize();
Map<String, TableSnapshot> secondListedSnapshots = Util.listSnapshots(cfs);
assertThat(secondSnapshotSize).isPositive();
assertThat(secondSnapshotSize).isGreaterThan(firstSnapshotsSize);
assertThat(secondListedSnapshots.size()).isEqualTo(2);
assertThat(secondListedSnapshots.get("snapshot_with_index")).isNotNull();
assertThat(secondListedSnapshots.get("snapshot_without_index")).isNotNull();
long withIndexSize = secondListedSnapshots.get("snapshot_with_index").computeSizeOnDiskBytes();
long withIndexTrueSize = secondListedSnapshots.get("snapshot_with_index").computeTrueSizeBytes();
assertThat(withIndexSize).isGreaterThan(withIndexTrueSize);
assertEquals(secondSnapshotSize, withIndexTrueSize + withoutIndexTrueSize);
// taking another one is basically a copy of the previous
SnapshotManager.instance.takeSnapshot("another_snapshot_with_index", new HashMap<>(), cfs.getKeyspaceTableName());
long thirdSnapshotSize = cfs.trueSnapshotsSize();
Map<String, TableSnapshot> thirdListedSnapshots = Util.listSnapshots(cfs);
assertThat(thirdSnapshotSize).isPositive();
assertThat(thirdSnapshotSize).isGreaterThan(secondSnapshotSize);
long anotherWithIndexSize = thirdListedSnapshots.get("another_snapshot_with_index").computeSizeOnDiskBytes();
long anotherWithIndexTrueSize = thirdListedSnapshots.get("another_snapshot_with_index").computeTrueSizeBytes();
assertEquals(withIndexSize, anotherWithIndexSize);
assertEquals(withIndexTrueSize, anotherWithIndexTrueSize);
}
private void rebuildIndices(ColumnFamilyStore cfs)
{
// be sure we build that index
for (Index index : cfs.indexManager.listIndexes())
{
try
{
cfs.indexManager.buildIndex(index).get();
}
catch (Throwable t)
{
throw new RuntimeException("Unable to build index", t);
}
}
}
@Test
public void testBackupAfterFlush()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe();
@ -381,8 +509,8 @@ public class ColumnFamilyStoreTest
// remember, latencies are only an estimate - off by up to 20% by the 1.2 factor between buckets.
assertThat(cfs.metric.coordinatorReadLatency.getCount()).isEqualTo(count);
assertThat(cfs.metric.coordinatorReadLatency.getSnapshot().getValue(0.5))
.isBetween((double) TimeUnit.MILLISECONDS.toMicros(5839),
(double) TimeUnit.MILLISECONDS.toMicros(5840));
.isBetween((double) TimeUnit.MILLISECONDS.toMicros(5839),
(double) TimeUnit.MILLISECONDS.toMicros(5840));
// Sanity check the metrics - 75th percentileof linear 0-10000ms
assertThat(cfs.metric.coordinatorWriteLatency.getCount()).isEqualTo(count);
assertThat(cfs.metric.coordinatorWriteLatency.getSnapshot().getValue(0.75))
@ -555,7 +683,7 @@ public class ColumnFamilyStoreTest
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1);
cfs.truncateBlocking();
cfs.truncateBlockingWithoutSnapshot();
UpdateBuilder builder = UpdateBuilder.create(cfs.metadata.get(), "key")
.newRow()
@ -565,7 +693,7 @@ public class ColumnFamilyStoreTest
Util.flush(cfs);
String snapshotName = "newSnapshot";
cfs.snapshotWithoutMemtable(snapshotName);
SnapshotManager.instance.takeSnapshot(snapshotName, Map.of(SnapshotOptions.SKIP_FLUSH, "true"), cfs.getKeyspaceTableName());
File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName);
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(snapshotManifestFile);
@ -595,17 +723,18 @@ public class ColumnFamilyStoreTest
writeData(cfs);
}
TableSnapshot snapshot = cfs.snapshot("basic");
SnapshotManager.instance.takeSnapshot("basic", cfs.getKeyspaceTableName());
TableSnapshot snapshot = SnapshotManager.instance.getSnapshot(cfs.metadata.keyspace, cfs.metadata.name, "basic").get();
assertNotNull(snapshot);
assertThat(snapshot.exists()).isTrue();
assertThat(cfs.listSnapshots().containsKey("basic")).isTrue();
assertThat(cfs.listSnapshots().get("basic")).isEqualTo(snapshot);
assertThat(Util.listSnapshots(cfs).containsKey("basic")).isTrue();
assertThat(Util.listSnapshots(cfs).get("basic")).isEqualTo(snapshot);
snapshot.getDirectories().forEach(FileUtils::deleteRecursive);
assertThat(snapshot.exists()).isFalse();
assertFalse(cfs.listSnapshots().containsKey("basic"));
assertFalse(Util.listSnapshots(cfs).containsKey("basic"));
}
private void writeData(ColumnFamilyStore cfs)
@ -623,7 +752,8 @@ public class ColumnFamilyStoreTest
}
@Test
public void testSnapshotCreationAndDeleteEmptyTable() {
public void testSnapshotCreationAndDeleteEmptyTable()
{
createSnapshotAndDelete(KEYSPACE1, CF_INDEX1, false);
createSnapshotAndDelete(KEYSPACE1, CF_STANDARD1, false);
createSnapshotAndDelete(KEYSPACE1, CF_STANDARD2, false);
@ -632,7 +762,8 @@ public class ColumnFamilyStoreTest
}
@Test
public void testSnapshotCreationAndDeletePopulatedTable() {
public void testSnapshotCreationAndDeletePopulatedTable()
{
createSnapshotAndDelete(KEYSPACE1, CF_INDEX1, true);
createSnapshotAndDelete(KEYSPACE1, CF_STANDARD1, true);
createSnapshotAndDelete(KEYSPACE1, CF_STANDARD2, true);
@ -651,8 +782,8 @@ public class ColumnFamilyStoreTest
String keyspace = path.getParent().getFileName().toString();
String table = path.getFileName().toString().split("-")[0];
Assert.assertEquals(cfs.getTableName(), table);
Assert.assertEquals(KEYSPACE1, keyspace);
assertEquals(cfs.getTableName(), table);
assertEquals(KEYSPACE1, keyspace);
}
@Test
@ -685,15 +816,15 @@ public class ColumnFamilyStoreTest
}
@VisibleForTesting
public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot) throws IOException
public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot)
{
Optional<File> schemaFile = snapshot.getSchemaFile();
Optional<File> manifestFile = snapshot.getManifestFile();
long schemaAndManifestFileSizes = 0;
schemaAndManifestFileSizes += schemaFile.isPresent() ? schemaFile.get().length() : 0;
schemaAndManifestFileSizes += manifestFile.isPresent() ? manifestFile.get().length() : 0;
for (File schemaFile : snapshot.getSchemaFiles())
schemaAndManifestFileSizes += schemaFile.length();
for (File manifestFile : snapshot.getManifestFiles())
schemaAndManifestFileSizes += manifestFile.length();
return schemaAndManifestFileSizes;
}
@ -827,7 +958,7 @@ public class ColumnFamilyStoreTest
@Override
public UnfilteredPartitionIterator
partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener)
partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener)
{
return null;
}

96
test/unit/org/apache/cassandra/db/DirectoriesTest.java
View File

@ -19,8 +19,11 @@ package org.apache.cassandra.db;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.attribute.FileStoreAttributeView;
import java.time.Instant;
@ -93,6 +96,7 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspaceTables;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.DefaultFSErrorHandler;
import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -326,12 +330,24 @@ public class DirectoriesTest
}
}
private Map<String, TableSnapshot> listSnapshots(Directories directories)
{
Set<TableSnapshot> snapshots = new SnapshotLoader(directories).loadSnapshots();
Map<String, TableSnapshot> tagSnapshotsMap = new HashMap<>();
for (TableSnapshot snapshot : snapshots)
tagSnapshotsMap.put(snapshot.getTag(), snapshot);
return tagSnapshotsMap;
}
@Test
public void testListSnapshots() throws Exception {
// Initial state
TableMetadata fakeTable = createFakeTable(TABLE_NAME);
Directories directories = new Directories(fakeTable, toDataDirectories(tempDataDir));
assertThat(directories.listSnapshots()).isEmpty();
assertThat(listSnapshots(directories)).isEmpty();
// Create snapshot with and without manifest
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
@ -340,7 +356,7 @@ public class DirectoriesTest
FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, TableSnapshot> snapshots = directories.listSnapshots();
Map<String, TableSnapshot> snapshots = listSnapshots(directories);
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT1)).isEqualTo(snapshot1.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
@ -350,40 +366,13 @@ public class DirectoriesTest
snapshot1.snapshotDir.deleteRecursive();
// Only snapshot 2 and 3 should be present
snapshots = directories.listSnapshots();
snapshots = listSnapshots(directories);
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT3).isEphemeral()).isTrue();
}
@Test
public void testListSnapshotDirsByTag() throws Exception {
// Initial state
TableMetadata fakeTable = createFakeTable("FakeTable");
Directories directories = new Directories(fakeTable, toDataDirectories(tempDataDir));
assertThat(directories.listSnapshotDirsByTag()).isEmpty();
// Create snapshot with and without manifest
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false);
FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, Set<File>> snapshotDirs = directories.listSnapshotDirsByTag();
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshotDirs.get(SNAPSHOT1)).allMatch(snapshotDir -> snapshotDir.equals(snapshot1.snapshotDir));
assertThat(snapshotDirs.get(SNAPSHOT2)).allMatch(snapshotDir -> snapshotDir.equals(snapshot2.snapshotDir));
assertThat(snapshotDirs.get(SNAPSHOT3)).allMatch(snapshotDir -> snapshotDir.equals(snapshot3.snapshotDir));
// Now remove snapshot1
snapshot1.snapshotDir.deleteRecursive();
// Only snapshot 2 and 3 should be present
snapshotDirs = directories.listSnapshotDirsByTag();
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
}
@Test
public void testMaybeManifestLoading() throws Exception {
for (TableMetadata cfm : CFM)
@ -412,7 +401,7 @@ public class DirectoriesTest
}
@Test
public void testSecondaryIndexDirectories()
public void testSecondaryIndexDirectories() throws IOException
{
TableMetadata.Builder builder =
TableMetadata.builder(KS, "cf")
@ -447,8 +436,8 @@ public class DirectoriesTest
// check if snapshot directory exists
parentSnapshotDirectory.tryCreateDirectories();
assertTrue(parentDirectories.snapshotExists("test"));
assertTrue(indexDirectories.snapshotExists("test"));
assertTrue(snapshotExists(parentDirectories, "test"));
assertTrue(snapshotExists(indexDirectories, "test"));
// check true snapshot size
Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
@ -456,14 +445,21 @@ public class DirectoriesTest
Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat());
createFile(indexSnapshot.fileFor(Components.DATA), 40);
assertEquals(30, parentDirectories.trueSnapshotsSize());
assertEquals(40, indexDirectories.trueSnapshotsSize());
// check snapshot details
Map<String, TableSnapshot> parentSnapshotDetail = parentDirectories.listSnapshots();
Map<String, TableSnapshot> parentSnapshotDetail = listSnapshots(parentDirectories);
assertTrue(parentSnapshotDetail.containsKey("test"));
Set<String> files = new HashSet<>();
Files.walkFileTree(parentSnapshotDirectory.toPath(), new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
{
files.add(file.toAbsolutePath().toString());
return FileVisitResult.CONTINUE;
}
});
// CASSANDRA-17357: include indexes when computing true size of parent table
assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes());
assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes(files));
// check backup directory
File parentBackupDirectory = Directories.getBackupsDirectory(parentDesc);
@ -471,6 +467,30 @@ public class DirectoriesTest
assertEquals(parentBackupDirectory, indexBackupDirectory.parent());
}
private boolean snapshotExists(Directories directories, String snapshotName)
{
for (File dir : directories.getDataPaths())
{
File snapshotDir;
if (Directories.isSecondaryIndexFolder(dir))
{
snapshotDir = new File(dir.parent(), join(Directories.SNAPSHOT_SUBDIR, snapshotName, dir.name()));
}
else
{
snapshotDir = new File(dir, join(Directories.SNAPSHOT_SUBDIR, snapshotName));
}
if (snapshotDir.exists())
return true;
}
return false;
}
private static String join(String... s)
{
return StringUtils.join(s, File.pathSeparator());
}
private File createFile(File file, int size)
{
try (FileOutputStreamPlus writer = new FileOutputStreamPlus(file);)

34
test/unit/org/apache/cassandra/db/KeyspaceTest.java
View File

@ -19,34 +19,49 @@
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.Collection;
import java.util.List;
import org.assertj.core.api.Assertions;
import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.Util;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.big.BigTableReader;
import org.apache.cassandra.metrics.ClearableHistogram;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.assertj.core.api.Assertions;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
public class KeyspaceTest extends CQLTester
{
// Test needs synchronous table drop to avoid flushes causing flaky failures of testLimitSSTables
@Before
public void cleanupSnapshots()
{
SnapshotManager.instance.clearAllSnapshots();
}
@Override
protected String createTable(String query)
{
@ -425,10 +440,11 @@ public class KeyspaceTest extends CQLTester
Keyspace ks = Keyspace.open(KEYSPACE_PER_TEST);
String table = getCurrentColumnFamilyStore().name;
ks.snapshot("test", table);
SnapshotManager.instance.takeSnapshot("test", ks.getName() + '.' + table);
assertTrue(ks.snapshotExists("test"));
assertEquals(1, ks.getAllSnapshots().count());
List<TableSnapshot> snapshots = SnapshotManager.instance.getSnapshots(ks.getName());
assertEquals(1, snapshots.size());
assertEquals(snapshots.get(0).getTag(), "test");
}
@Test

11
test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java
View File

@ -35,6 +35,8 @@ import org.apache.cassandra.index.internal.CassandraIndex;
import org.apache.cassandra.index.sasi.SASIIndex;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.service.reads.SpeculativeRetryPolicy;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JsonUtils;
@ -431,7 +433,7 @@ public class SchemaCQLHelperTest extends CQLTester
execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1, reg2) VALUES (?, ?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null, i + 5);
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
cfs.snapshot(SNAPSHOT);
SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(SNAPSHOT, cfs.getKeyspaceTableName()));
String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset());
assertThat(schema,
@ -480,6 +482,7 @@ public class SchemaCQLHelperTest extends CQLTester
"INDEX IF NOT EXISTS " + tableName + "_reg2_idx ON " + keyspace() + '.' + tableName + " (reg2)" +
(" USING '" + (isIndexLegacy ? CassandraIndex.NAME : DatabaseDescriptor.getDefaultSecondaryIndex()) + "'") + ";"));
// TODO: construct manifest from SnapshotManager
JsonNode manifest = JsonUtils.JSON_OBJECT_MAPPER.readTree(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).toJavaIOFile());
JsonNode files = manifest.get("files");
// two files, the second is index
@ -508,7 +511,7 @@ public class SchemaCQLHelperTest extends CQLTester
execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1) VALUES (?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null);
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
cfs.snapshot(SNAPSHOT);
SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName());
String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset());
schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order
@ -551,7 +554,7 @@ public class SchemaCQLHelperTest extends CQLTester
execute("INSERT INTO %s (pk1, reg1) VALUES (?, ?)", i, i + 1);
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName);
cfs.snapshot(SNAPSHOT);
SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName());
String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset());
schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order
@ -577,7 +580,7 @@ public class SchemaCQLHelperTest extends CQLTester
public void testSystemKsSnapshot()
{
ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("peers");
cfs.snapshot(SNAPSHOT);
SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName());
Assert.assertTrue(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).exists());
Assert.assertFalse(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).exists());

4
test/unit/org/apache/cassandra/db/SnapshotTest.java
View File

@ -27,6 +27,7 @@ import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.snapshot.SnapshotManager;
public class SnapshotTest extends CQLTester
{
@ -41,6 +42,7 @@ public class SnapshotTest extends CQLTester
File toc = sstable.descriptor.fileFor(Components.TOC);
Files.write(toc.toPath(), new byte[0], StandardOpenOption.TRUNCATE_EXISTING);
}
getCurrentColumnFamilyStore().snapshot("hello");
SnapshotManager.instance.takeSnapshot("hello", getCurrentColumnFamilyStore().getKeyspaceTableName());
}
}

13
test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
View File

@ -25,6 +25,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@ -34,7 +35,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
@ -100,14 +101,14 @@ public class SystemKeyspaceTest
// First, check that in the absence of any previous installed version, we don't create snapshots
for (ColumnFamilyStore cfs : Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStores())
cfs.clearUnsafe();
StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME);
SystemKeyspace.snapshotOnVersionChange();
assertDeleted();
// now setup system.local as if we're upgrading from a previous version
setupReleaseVersion(getOlderVersionString());
StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME);
assertDeleted();
// Compare versions again & verify that snapshots were created for all tables in the system ks
@ -120,7 +121,7 @@ public class SystemKeyspaceTest
// clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't
// see any new snapshots created this time.
StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME);
setupReleaseVersion(FBUtilities.getReleaseVersionString());
SystemKeyspace.snapshotOnVersionChange();
@ -129,7 +130,7 @@ public class SystemKeyspaceTest
// 10 files expected.
assertDeleted();
StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME);
SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME);
}
@Test
@ -170,7 +171,7 @@ public class SystemKeyspaceTest
Set<String> snapshottedTableNames = new HashSet<>();
for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores())
{
if (!cfs.listSnapshots().isEmpty())
if (!Util.listSnapshots(cfs).isEmpty())
snapshottedTableNames.add(cfs.getTableName());
}
return snapshottedTableNames;

59
test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java
View File

@ -19,27 +19,30 @@
package org.apache.cassandra.db.virtual;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import com.google.common.collect.ImmutableList;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.TableSnapshot;
public class SnapshotsTableTest extends CQLTester
{
private static final String KS_NAME = "vts";
private static final String SNAPSHOT_TTL = "snapshotTtl";
private static final String SNAPSHOT_NO_TTL = "snapshotNoTtl";
private static final DurationSpec.IntSecondsBound ttl = new DurationSpec.IntSecondsBound("4h");
private static final String TTL = "4h";
@Before
public void before() throws Throwable
@ -58,48 +61,56 @@ public class SnapshotsTableTest extends CQLTester
@After
public void after()
{
StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE);
StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_TTL, KEYSPACE);
SnapshotManager.instance.clearSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), KEYSPACE);
SnapshotManager.instance.clearSnapshot(SNAPSHOT_TTL, Collections.emptyMap(), KEYSPACE);
schemaChange(String.format("DROP TABLE %s", KEYSPACE + '.' + currentTable()));
}
schemaChange(String.format("DROP TABLE %s", KEYSPACE + "." + currentTable()));
private static Date toDate(Instant instant)
{
return new Date(instant.toEpochMilli());
}
private static TableSnapshot createSnapshot(String snapshotTtl, Map<String, String> options, ColumnFamilyStore cfs)
{
List<TableSnapshot> snapshots = SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(snapshotTtl, options, cfs.getKeyspaceTableName()));
Assert.assertEquals(1, snapshots.size());
return snapshots.iterator().next();
}
@Test
public void testSnapshots() throws Throwable
public void testSnapshots()
{
Instant now = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()).truncatedTo(ChronoUnit.MILLIS);
Date createdAt = new Date(now.toEpochMilli());
Date expiresAt = new Date(now.plusSeconds(ttl.toSeconds()).toEpochMilli());
getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_NO_TTL, null, false, false, null, null, now);
getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_TTL, null, false, false, ttl, null, now);
ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE);
TableSnapshot snapshotWithTtl = createSnapshot(SNAPSHOT_TTL, Map.of(SnapshotOptions.TTL, TTL), cfs);
TableSnapshot snapshotWithoutTtl = createSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), cfs);
// query all from snapshots virtual table
UntypedResultSet result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots");
assertRowsIgnoringOrder(result,
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false));
// query with conditions
result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where ephemeral = false");
assertRows(result,
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false));
result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where size_on_disk > 1000");
assertRows(result,
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false),
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false));
result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where name = ?", SNAPSHOT_TTL);
assertRows(result,
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false));
// clear some snapshots
StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE);
SnapshotManager.instance.clearSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), KEYSPACE);
result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots");
assertRowsIgnoringOrder(result,
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false));
row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false));
}
}

6
test/unit/org/apache/cassandra/index/sai/SAITester.java
View File

@ -102,7 +102,8 @@ import org.apache.cassandra.schema.MockSchema;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotOptions;
import org.apache.cassandra.utils.ConfigGenBuilder;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Throwables;
@ -683,8 +684,7 @@ public abstract class SAITester extends CQLTester.Fuzzed
protected int snapshot(String snapshotName)
{
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
TableSnapshot snapshot = cfs.snapshot(snapshotName);
return snapshot.getDirectories().size();
return SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(snapshotName, cfs.getKeyspaceTableName())).iterator().next().getDirectories().size();
}
protected void restoreSnapshot(String snapshot)

8
test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
View File

@ -126,6 +126,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.serializers.TypeSerializer;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -199,7 +200,7 @@ public class SASIIndexTest
try
{
store.snapshot(snapshotName);
SnapshotManager.instance.takeSnapshot(snapshotName, store.getKeyspaceTableName());
// Compact to make true snapshot size != 0
store.forceMajorCompaction();
@ -208,6 +209,7 @@ public class SASIIndexTest
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName));
Assert.assertFalse(ssTableReaders.isEmpty());
Assert.assertNotNull(manifest.files);
Assert.assertFalse(manifest.files.isEmpty());
Assert.assertEquals(ssTableReaders.size(), manifest.files.size());
@ -243,7 +245,7 @@ public class SASIIndexTest
}
}
TableSnapshot details = store.listSnapshots().get(snapshotName);
TableSnapshot details = Util.listSnapshots(store).get(snapshotName);
// check that SASI components are included in the computation of snapshot size
long snapshotSize = tableSize + indexSize + getSnapshotManifestAndSchemaFileSizes(details);
@ -251,7 +253,7 @@ public class SASIIndexTest
}
finally
{
store.clearSnapshot(snapshotName);
SnapshotManager.instance.clearSnapshot(store.getKeyspaceName(), store.getTableName(), snapshotName);
}
}

5
test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java
View File

@ -31,6 +31,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
@ -129,7 +130,7 @@ public class SchemaKeyspaceTest
SchemaTestUtil.announceTableDrop(keyspaceName, tableName);
assertFalse(cfs.listSnapshots().isEmpty());
assertFalse(Util.listSnapshots(cfs).isEmpty());
}
@Test
@ -147,7 +148,7 @@ public class SchemaKeyspaceTest
SchemaTestUtil.announceTableDrop(keyspaceName, tableName);
assertTrue(cfs.listSnapshots().isEmpty());
assertTrue(Util.listSnapshots(cfs).isEmpty());
}
private static void updateTable(String keyspace, TableMetadata oldTable, TableMetadata newTable)

3
test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
View File

@ -61,6 +61,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.repair.messages.RepairOption;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeAddresses;
@ -293,7 +294,7 @@ public class ActiveRepairServiceTest
true, PreviewKind.NONE);
createSSTables(store, 2);
store.getRepairManager().snapshot(prsId.toString(), ranges, false);
try (Refs<SSTableReader> refs = store.getSnapshotSSTableReaders(prsId.toString()))
try (Refs<SSTableReader> refs = TableSnapshot.getSnapshotSSTableReaders(store, prsId.toString()))
{
assertEquals(original, Sets.newHashSet(refs.iterator()));
}

14
test/unit/org/apache/cassandra/service/StorageServiceServerTest.java
View File

@ -55,6 +55,7 @@ import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaKeyspaceTables;
import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.membership.Location;
@ -144,6 +145,7 @@ public class StorageServiceServerTest
@Before
public void resetCMS()
{
SnapshotManager.instance.clearAllSnapshots();
ServerTestUtils.resetCMS();
}
@ -158,21 +160,27 @@ public class StorageServiceServerTest
public void testSnapshotWithFlush() throws IOException
{
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
StorageService.instance.takeSnapshot(UUID.randomUUID().toString());
String snapshotName = UUID.randomUUID().toString();
StorageService.instance.takeSnapshot(snapshotName);
Assert.assertTrue(SnapshotManager.instance.exists(p -> p.getTag().equals(snapshotName)));
}
@Test
public void testTableSnapshot() throws IOException
{
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
StorageService.instance.takeTableSnapshot(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, UUID.randomUUID().toString());
String snapshotName = UUID.randomUUID().toString();
StorageService.instance.takeTableSnapshot(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, snapshotName);
Assert.assertTrue(SnapshotManager.instance.exists(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, snapshotName));
}
@Test
public void testSnapshot() throws IOException
{
// no need to insert extra data, even an "empty" database will have a little information in the system keyspace
StorageService.instance.takeSnapshot(UUID.randomUUID().toString(), SchemaConstants.SCHEMA_KEYSPACE_NAME);
String snapshotName = UUID.randomUUID().toString();
StorageService.instance.takeSnapshot(snapshotName, SchemaConstants.SCHEMA_KEYSPACE_NAME);
Assert.assertTrue(SnapshotManager.instance.exists(p -> p.getTag().equals(snapshotName) && p.getKeyspaceName().equals(SchemaConstants.SCHEMA_KEYSPACE_NAME)));
}
@Test

162
test/unit/org/apache/cassandra/service/snapshot/MetadataSnapshotsTest.java
View File

@ -22,26 +22,22 @@ import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import com.google.common.util.concurrent.Uninterruptibles;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.DefaultFSErrorHandler;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.service.snapshot.TableSnapshotTest.createFolders;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertTrue;
public class MetadataSnapshotsTest
{
@ -50,6 +46,9 @@ public class MetadataSnapshotsTest
@BeforeClass
public static void beforeClass()
{
CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.setInt(3);
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(3);
DatabaseDescriptor.daemonInitialization();
FileUtils.setFSErrorHandler(new DefaultFSErrorHandler());
}
@ -57,6 +56,21 @@ public class MetadataSnapshotsTest
@ClassRule
public static TemporaryFolder temporaryFolder = new TemporaryFolder();
private SnapshotManager manager;
@Before
public void beforeTest()
{
manager = SnapshotManager.instance;
}
@After
public void afterTest() throws Exception
{
SnapshotManager.instance.clearAllSnapshots();
SnapshotManager.instance.close();
}
private TableSnapshot generateSnapshotDetails(String tag, Instant expiration, boolean ephemeral)
{
try
@ -77,25 +91,28 @@ public class MetadataSnapshotsTest
}
@Test
public void testLoadSnapshots() throws Exception {
public void testExpiringSnapshots()
{
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS), false);
TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false);
List<TableSnapshot> snapshots = Arrays.asList(expired, nonExpired, nonExpiring);
// Create SnapshotManager with 3 snapshots: expired, non-expired and non-expiring
SnapshotManager manager = new SnapshotManager(3, 3);
manager.addSnapshots(snapshots);
manager.start(false);
for (TableSnapshot snapshot : snapshots)
manager.addSnapshot(snapshot);
// Only expiring snapshots should be loaded
assertThat(manager.getExpiringSnapshots()).hasSize(2);
assertThat(manager.getExpiringSnapshots()).contains(expired);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expired);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired);
}
@Test
public void testClearExpiredSnapshots() throws Exception {
SnapshotManager manager = new SnapshotManager(3, 3);
public void testClearExpiredSnapshots()
{
manager.start(false);
// Add 3 snapshots: expired, non-expired and non-expiring
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
@ -106,119 +123,66 @@ public class MetadataSnapshotsTest
manager.addSnapshot(nonExpiring);
// Only expiring snapshot should be indexed and all should exist
assertThat(manager.getExpiringSnapshots()).hasSize(2);
assertThat(manager.getExpiringSnapshots()).contains(expired);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expired);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired);
assertThat(expired.exists()).isTrue();
assertThat(nonExpired.exists()).isTrue();
assertThat(nonExpiring.exists()).isTrue();
// After clearing expired snapshots, expired snapshot should be removed while the others should remain
manager.clearExpiredSnapshots();
assertThat(manager.getExpiringSnapshots()).hasSize(1);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(1);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired);
assertThat(expired.exists()).isFalse();
assertThat(nonExpired.exists()).isTrue();
assertThat(nonExpiring.exists()).isTrue();
}
@Test
public void testScheduledCleanup() throws Exception {
SnapshotManager manager = new SnapshotManager(0, 1);
try
{
// Start snapshot manager which should start expired snapshot cleanup thread
manager.start();
public void testScheduledCleanup() throws Exception
{
manager.start(true);
// Add 2 expiring snapshots: one to expire in 2 seconds, another in 1 day
int TTL_SECS = 2;
TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
manager.addSnapshot(toExpire);
manager.addSnapshot(nonExpired);
// Add 2 expiring snapshots: one to expire in 6 seconds, another in 1 day
int TTL_SECS = 6;
TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
manager.addSnapshot(toExpire);
manager.addSnapshot(nonExpired);
// Check both snapshots still exist
assertThat(toExpire.exists()).isTrue();
assertThat(nonExpired.exists()).isTrue();
assertThat(manager.getExpiringSnapshots()).hasSize(2);
assertThat(manager.getExpiringSnapshots()).contains(toExpire);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
// Check both snapshots still exist
assertThat(toExpire.exists()).isTrue();
assertThat(nonExpired.exists()).isTrue();
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(toExpire);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired);
// Sleep 4 seconds
Thread.sleep((TTL_SECS + 2) * 1000L);
// Sleep 10 seconds
Thread.sleep((TTL_SECS + 4) * 1000L);
// Snapshot with ttl=2s should be gone, while other should remain
assertThat(manager.getExpiringSnapshots()).hasSize(1);
assertThat(manager.getExpiringSnapshots()).contains(nonExpired);
assertThat(toExpire.exists()).isFalse();
assertThat(nonExpired.exists()).isTrue();
}
finally
{
manager.stop();
}
// Snapshot with ttl=6s should be gone, while other should remain
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(1);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired);
assertThat(toExpire.exists()).isFalse();
assertThat(nonExpired.exists()).isTrue();
}
@Test
public void testClearSnapshot() throws Exception
public void testClearSnapshot()
{
// Given
SnapshotManager manager = new SnapshotManager(1, 3);
manager.start(false);
TableSnapshot expiringSnapshot = generateSnapshotDetails("snapshot", now().plusMillis(50000), false);
manager.addSnapshot(expiringSnapshot);
assertThat(manager.getExpiringSnapshots()).contains(expiringSnapshot);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expiringSnapshot);
assertThat(expiringSnapshot.exists()).isTrue();
// When
manager.clearSnapshot(expiringSnapshot);
// Then
assertThat(manager.getExpiringSnapshots()).doesNotContain(expiringSnapshot);
assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).doesNotContain(expiringSnapshot);
assertThat(expiringSnapshot.exists()).isFalse();
}
@Test // see CASSANDRA-18211
public void testConcurrentClearingOfSnapshots() throws Exception
{
AtomicReference<Long> firstInvocationTime = new AtomicReference<>(0L);
AtomicReference<Long> secondInvocationTime = new AtomicReference<>(0L);
SnapshotManager manager = new SnapshotManager(0, 5)
{
@Override
public synchronized void clearSnapshot(TableSnapshot snapshot)
{
if (snapshot.getTag().equals("mysnapshot"))
{
firstInvocationTime.set(currentTimeMillis());
Uninterruptibles.sleepUninterruptibly(10, SECONDS);
}
else if (snapshot.getTag().equals("mysnapshot2"))
{
secondInvocationTime.set(currentTimeMillis());
}
super.clearSnapshot(snapshot);
}
};
TableSnapshot expiringSnapshot = generateSnapshotDetails("mysnapshot", Instant.now().plusSeconds(15), false);
manager.addSnapshot(expiringSnapshot);
manager.resumeSnapshotCleanup();
Thread nonExpiringSnapshotCleanupThred = new Thread(() -> manager.clearSnapshot(generateSnapshotDetails("mysnapshot2", null, false)));
// wait until the first snapshot expires
await().pollInterval(1, SECONDS)
.pollDelay(0, SECONDS)
.timeout(1, MINUTES)
.until(() -> firstInvocationTime.get() > 0);
// this will block until the first snapshot is cleaned up
nonExpiringSnapshotCleanupThred.start();
nonExpiringSnapshotCleanupThred.join();
assertTrue(secondInvocationTime.get() - firstInvocationTime.get() > 10_000);
}
}

8
test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
View File

@ -28,10 +28,12 @@ import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.util.File;
@ -67,6 +69,12 @@ public class SnapshotLoaderTest
@ClassRule
public static TemporaryFolder tmpDir = new TemporaryFolder();
@BeforeClass
public static void setup()
{
DatabaseDescriptor.daemonInitialization();
}
@Test
public void testMatcher()
{

274
test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java Normal file
View File

@ -0,0 +1,274 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.function.Consumer;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.PathUtils;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import static java.lang.String.format;
import static org.apache.cassandra.config.CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class SnapshotManagerTest
{
@ClassRule
public static TemporaryFolder temporaryFolder = new TemporaryFolder();
@ClassRule
public static TemporaryFolder temporaryFolder2 = new TemporaryFolder();
private static File rootDir1;
private static File rootDir2;
private static String[] dataDirs;
@BeforeClass
public static void beforeClass() throws Exception
{
CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(10);
SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.setInt(5);
rootDir1 = new File(temporaryFolder.getRoot());
rootDir2 = new File(temporaryFolder2.getRoot());
dataDirs = new String[]{
rootDir1.toPath().toAbsolutePath().toString(),
rootDir2.toPath().toAbsolutePath().toString()
};
}
private static class ThrowingTask extends AbstractSnapshotTask<Void>
{
public ThrowingTask()
{
super(null);
}
@Override
public Void call()
{
throw new RuntimeException("an exception");
}
@Override
public SnapshotTaskType getTaskType()
{
return SnapshotTaskType.SNAPSHOT;
}
}
@Test
public void testTaskThrowingException()
{
doWithManager(manager -> assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class)
.hasRootCauseMessage("an exception")
.hasMessageContaining("Exception occured while executing"));
doWithManager(manager -> {
manager.close();
assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class)
.hasRootCauseMessage("an exception")
.hasMessageContaining("Exception occured while executing");
manager.start(false);
assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class)
.hasRootCauseMessage("an exception")
.hasMessageContaining("Exception occured while executing");
});
}
/**
* Tests that if we remove all manifests files, that equals to stopping manager to track
* such snapshot, however no data will be removed.
*/
@Test
public void testRemovingManifestsLogicallyRemovesSnapshot()
{
doWithManager(manager -> {
List<TableSnapshot> tableSnapshots = generateTableSnapshots(10, 100);
for (TableSnapshot snapshot : tableSnapshots)
manager.addSnapshot(snapshot);
// we still have 1000 snapshots because we removed just one manifest
removeManifestOfSnapshot(tableSnapshots.get(0));
assertEquals(1000, manager.getSnapshots((t) -> true).size());
// remove the second manifest, that will render snapshot to be logically removed
removeManifestOfSnapshot(tableSnapshots.get(0));
assertEquals(999, manager.getSnapshots((t) -> true).size());
// check that data are still there
assertFalse(tableSnapshots.get(0).hasManifest());
assertTrue(tableSnapshots.get(0).exists());
for (File snapshotDir : tableSnapshots.get(0).getDirectories())
assertTrue(snapshotDir.exists());
});
}
private void doWithManager(Consumer<SnapshotManager> action)
{
try (MockedStatic<DatabaseDescriptor> mockedDD = Mockito.mockStatic(DatabaseDescriptor.class))
{
mockedDD.when(() -> DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations()).thenReturn(dataDirs);
mockedDD.when(() -> DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations()).thenReturn(new String[]{});
mockedDD.when(() -> DatabaseDescriptor.getAllDataFileLocations()).thenReturn(dataDirs);
mockedDD.when(() -> DatabaseDescriptor.getDumpHeapOnUncaughtException()).thenReturn(false);
try (SnapshotManager snapshotManager = new SnapshotManager(SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(10),
SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(5),
dataDirs).start(true))
{
action.accept(snapshotManager);
}
finally
{
clearDirectory(rootDir1.toPath());
clearDirectory(rootDir2.toPath());
}
}
}
private List<TableSnapshot> generateTableSnapshots(int keyspaces, int tables) throws RuntimeException
{
List<TableSnapshot> tableSnapshots = new ArrayList<>();
for (int i = 0; i < keyspaces; i++)
{
for (int j = 0; j < tables; j++)
{
String snapshotName = format("mysnapshot_%s_%s", i, j);
File dir1 = new File(Paths.get(rootDir1.absolutePath(), "ks", "tb-1b255f4def2540a60000000000000005", "snapshots", snapshotName));
File dir2 = new File(Paths.get(rootDir2.absolutePath(), "ks", "tb-1b255f4def2540a60000000000000005", "snapshots", snapshotName));
dir1.tryCreateDirectories();
dir2.tryCreateDirectories();
TableSnapshot snapshot = generateSnapshotDetails(Set.of(dir1, dir2), snapshotName, "ks", "tb-1b255f4def2540a60000000000000005", null, false);
SnapshotManifest manifest = new SnapshotManifest(List.of(), null, snapshot.getCreatedAt(), snapshot.isEphemeral());
try
{
manifest.serializeToJsonFile(new File(dir1.toPath().resolve("manifest.json")));
manifest.serializeToJsonFile(new File(dir2.toPath().resolve("manifest.json")));
generateFileInSnapshot(snapshot);
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
tableSnapshots.add(snapshot);
}
}
return tableSnapshots;
}
private void generateFileInSnapshot(TableSnapshot tableSnapshot) throws IOException
{
for (File snapshotDir : tableSnapshot.getDirectories())
Files.createFile(snapshotDir.toPath().resolve("schema.cql"));
}
private void removeManifestOfSnapshot(TableSnapshot tableSnapshot)
{
for (File snapshotDir : tableSnapshot.getDirectories())
{
if (snapshotDir.exists())
{
File manifest = new File(snapshotDir, "manifest.json");
if (!manifest.exists())
continue;
manifest.delete();
return;
}
}
}
private TableSnapshot generateSnapshotDetails(Set<File> roots,
String tag,
String keyspace,
String table,
Instant expiration,
boolean ephemeral)
{
try
{
Set<File> snapshotDirs = new HashSet<>();
for (File root : roots)
{
root.tryCreateDirectories();
snapshotDirs.add(root);
}
return new TableSnapshot(keyspace,
table,
UUID.randomUUID(),
tag,
Instant.EPOCH,
expiration,
snapshotDirs,
ephemeral);
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
}
/**
* Empties everything in directory of "path" but keeps the directory itself.
*
* @param path directory to be emptied
* @throws FSWriteError if any part of the tree cannot be deleted
*/
public static void clearDirectory(Path path)
{
if (PathUtils.isDirectory(path))
PathUtils.forEach(path, PathUtils::deleteRecursive);
}
}

64
test/unit/org/apache/cassandra/service/snapshot/SnapshotOptionsTest.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.snapshot;
import java.time.Instant;
import java.util.EnumSet;
import java.util.List;
import com.google.common.util.concurrent.RateLimiter;
import org.junit.Test;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
public class SnapshotOptionsTest
{
@Test
public void testSnapshotName()
{
List<SnapshotType> sameNameTypes = List.of(SnapshotType.DIAGNOSTICS, SnapshotType.REPAIR, SnapshotType.USER);
for (SnapshotType type : sameNameTypes)
{
SnapshotOptions options = SnapshotOptions.systemSnapshot("a_name", type, "ks.tb")
.rateLimiter(RateLimiter.create(5))
.build();
String snapshotName = options.getSnapshotName(Instant.now());
assertEquals("a_name", snapshotName);
}
EnumSet<SnapshotType> snapshotTypes = EnumSet.allOf(SnapshotType.class);
snapshotTypes.removeAll(sameNameTypes);
for (SnapshotType type : snapshotTypes)
{
SnapshotOptions options = SnapshotOptions.systemSnapshot("a_name", type, "ks.tb")
.rateLimiter(RateLimiter.create(5))
.build();
Instant now = Instant.now();
String snapshotName = options.getSnapshotName(now);
assertEquals(format("%d-%s-%s", now.toEpochMilli(), type.label, "a_name"), snapshotName);
}
}
}

118
test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
View File

@ -19,7 +19,7 @@
package org.apache.cassandra.service.snapshot;
import java.io.IOException;
import java.nio.file.Paths;
import java.nio.file.Files;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@ -38,10 +38,16 @@ import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.utils.Pair;
import org.mockito.MockedStatic;
import static org.apache.cassandra.utils.FBUtilities.now;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.times;
public class TableSnapshotTest
{
@ -62,13 +68,68 @@ public class TableSnapshotTest
{
File subfolder = new File(folder, folderName);
subfolder.tryCreateDirectories();
assertThat(subfolder.exists());
File manifest = new File(subfolder.toPath().resolve("manifest.json"));
File schema = new File(subfolder.toPath().resolve("schema.cql"));
manifest.createFileIfNotExists();
schema.createFileIfNotExists();
Files.write(manifest.toPath(), "{}".getBytes());
Files.write(schema.toPath(), "cql schema".getBytes());
folders.add(subfolder);
}
;
return folders;
}
@Test
public void testTableSnapshotEquality() throws IOException
{
TableSnapshot snapshot1 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, createFolders(tempFolder), false);
TableSnapshot snapshot2 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, createFolders(tempFolder), false);
// they are not equal, because table id is random for each
assertNotEquals(snapshot1, snapshot2);
UUID tableId = UUID.randomUUID();
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, createFolders(tempFolder), false);
snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, createFolders(tempFolder), false);
// they are equal, even their directories differ
assertEquals(snapshot1, snapshot2);
Set<File> folders = createFolders(tempFolder);
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", Instant.now(), null, folders, false);
snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", Instant.now().plusSeconds(1), null, folders, false);
// they are equal, even their creation times differ
assertEquals(snapshot1, snapshot2);
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, Instant.now(), folders, false);
snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", null, Instant.now().plusSeconds(1), folders, false);
// they are equal, even their expiration times differ
assertEquals(snapshot1, snapshot2);
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, folders, false);
snapshot2 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, folders, false);
// they are not equal, because their tableId differs
assertNotEquals(snapshot1, snapshot2);
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some1", null, null, folders, false);
snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some2", null, null, folders, false);
// they are not equal, because their tag differs
assertNotEquals(snapshot1, snapshot2);
snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some1", null, null, folders, false);
snapshot2 = new TableSnapshot("ks", "tbl2", tableId, "some1", null, null, folders, false);
// they are not equal, because their table differs
assertNotEquals(snapshot1, snapshot2);
}
@Test
public void testSnapshotExists() throws IOException
{
@ -186,8 +247,18 @@ public class TableSnapshotTest
res += FileUtils.folderSize(dir);
}
assertThat(tableDetails.computeSizeOnDiskBytes()).isGreaterThan(0L);
assertThat(tableDetails.computeSizeOnDiskBytes()).isEqualTo(res);
try (MockedStatic<FileUtils> fileUtilsMock = mockStatic(FileUtils.class))
{
fileUtilsMock.when(() -> FileUtils.folderSize(any())).thenCallRealMethod();
assertThat(tableDetails.computeSizeOnDiskBytes()).isGreaterThan(0L);
assertThat(tableDetails.computeSizeOnDiskBytes()).isEqualTo(res);
// when we invoke computeSizeOnDiskBytes for the second time, it will use cached value
// 3 invocations for folderSize are from the first invocation of computeSizeOnDiskBytes because
// we have 3 data dirs, if we have not cached it, the number of invocations would be 6.
fileUtilsMock.verify(() -> FileUtils.folderSize(any()), times(3));
}
}
@Test
@ -208,15 +279,27 @@ public class TableSnapshotTest
Long res = 0L;
Set<String> files = new HashSet<>();
for (File dir : folders)
{
File file = new File(dir, "tmp");
files.add(file.toAbsolute().toString());
writeBatchToFile(file);
res += file.length();
res += new File(dir, "manifest.json").length();
res += new File(dir, "schema.cql").length();
}
assertThat(tableDetails.computeTrueSizeBytes()).isGreaterThan(0L);
assertThat(tableDetails.computeTrueSizeBytes()).isEqualTo(res);
try (MockedStatic<TableSnapshot> tableSnapshotMock = mockStatic(TableSnapshot.class))
{
tableSnapshotMock.when(() -> TableSnapshot.getLiveFileFromSnapshotFile(any())).thenCallRealMethod();
assertThat(tableDetails.computeTrueSizeBytes(files)).isGreaterThan(0L);
assertThat(tableDetails.computeTrueSizeBytes(files)).isEqualTo(res);
// 6 because we avoided to call it for manifest and schema because they are cached
tableSnapshotMock.verify(() -> TableSnapshot.getLiveFileFromSnapshotFile(any()), times(6));
}
}
@Test
@ -298,7 +381,7 @@ public class TableSnapshotTest
// 1. snapshot to clear is not ephemeral
// 2. tag to clear is null, empty, or it is equal to snapshot tag
// 3. byTimestamp is true
if (TableSnapshot.shouldClearSnapshot(testingTag, olderThanTimestamp).test(snapshot))
if (ClearSnapshotTask.getClearSnapshotPredicate(testingTag, Set.of(keyspace), olderThanTimestamp, false).test(snapshot))
{
// shouldClearTag = true
boolean shouldClearTag = (testingTag == null || testingTag.isEmpty()) || snapshot.getTag().equals(testingTag);
@ -321,23 +404,4 @@ public class TableSnapshotTest
}
}
}
@Test
public void testGetLiveFileFromSnapshotFile()
{
testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db",
"~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db");
}
@Test
public void testGetLiveFileFromSnapshotIndexFile()
{
testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db",
"~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db");
}
public void testGetLiveFileFromSnapshotFile(String snapshotFile, String expectedLiveFile)
{
assertThat(TableSnapshot.getLiveFileFromSnapshotFile(Paths.get(snapshotFile)).toString()).isEqualTo(expectedLiveFile);
}
}

7
test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java
View File

@ -18,7 +18,6 @@
package org.apache.cassandra.tools;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -34,7 +33,7 @@ import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.io.sstable.LegacySSTableTest;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
@ -93,9 +92,7 @@ public class StandaloneUpgraderOnSStablesTest
{
LegacySSTableTest.truncateLegacyTables(legacyId);
LegacySSTableTest.loadLegacyTables(legacyId);
StorageService.instance.takeSnapshot("testsnapshot",
Collections.emptyMap(),
"legacy_tables.legacy_" + legacyId + "_simple");
SnapshotManager.instance.takeSnapshot("testsnapshot", "legacy_tables.legacy_" + legacyId + "_simple");
ToolResult tool = ToolRunner.invokeClass(StandaloneUpgrader.class,
"-k",

12
test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java
View File

@ -30,6 +30,7 @@ import java.util.regex.Pattern;
import javax.management.openmbean.TabularData;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@ -37,6 +38,7 @@ import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.snapshot.SnapshotManager;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.tools.NodeProbe;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
@ -65,6 +67,12 @@ public class ClearSnapshotTest extends CQLTester
probe = new NodeProbe(jmxHost, jmxPort);
}
@Before
public void clearAllSnapshots()
{
SnapshotManager.instance.clearAllSnapshots();
}
@AfterClass
public static void teardown() throws IOException
{
@ -315,11 +323,15 @@ public class ClearSnapshotTest extends CQLTester
String tableId2 = DASH_PATTERN.matcher(tableMetadata2.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", keyspace2, tableName2)))
.id.asUUID().toString()).replaceAll("");
SnapshotManager.instance.close();
rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "snapshot-to-clear-ks1-tb1", start.minus(5, HOURS));
rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "some-other-snapshot-ks1-tb1", start.minus(2, HOURS));
rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "last-snapshot-ks1-tb1", start.minus(1, SECONDS));
rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "snapshot-to-clear-ks2-tb2", start.minus(5, HOURS));
rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "some-other-snapshot-ks2-tb2", start.minus(2, HOURS));
rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "last-snapshot-ks2-tb2", start.minus(1, SECONDS));
SnapshotManager.instance.start(true);
}
}

4
tools/stress/src/org/apache/cassandra/stress/CompactionStress.java
View File

@ -340,7 +340,7 @@ public abstract class CompactionStress implements Runnable
}
double currentSizeGiB;
while ((currentSizeGiB = directories.getRawDiretoriesSize() / BYTES_IN_GIB) < totalSizeGiB)
while ((currentSizeGiB = directories.getRawDirectoriesSize() / BYTES_IN_GIB) < totalSizeGiB)
{
if (finished.getCount() == 0)
break;
@ -353,7 +353,7 @@ public abstract class CompactionStress implements Runnable
workManager.stop();
Uninterruptibles.awaitUninterruptibly(finished);
currentSizeGiB = directories.getRawDiretoriesSize() / BYTES_IN_GIB;
currentSizeGiB = directories.getRawDirectoriesSize() / BYTES_IN_GIB;
System.out.println(String.format("Finished writing %.2fGB", currentSizeGiB));
}
}