From f410b0fa0bc5adbb674654a0e27b02282971cfec Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Thu, 20 Jun 2024 10:29:51 +0200 Subject: [PATCH] Consolidate all snapshot management to SnapshotManager patch by Stefan Miklosovic; reviewed by Francisco Guerrero for CASSANDRA-18111 --- CHANGES.txt | 1 + NEWS.txt | 3 + .../cassandra/db/ColumnFamilyStore.java | 316 +---- .../org/apache/cassandra/db/Directories.java | 184 +-- .../org/apache/cassandra/db/Keyspace.java | 89 +- .../apache/cassandra/db/SystemKeyspace.java | 27 +- .../db/compaction/CompactionTask.java | 9 +- .../cassandra/db/lifecycle/Tracker.java | 34 +- .../repair/CassandraTableRepairManager.java | 23 +- .../repair/CassandraValidationIterator.java | 10 +- .../cassandra/db/virtual/SnapshotsTable.java | 4 +- .../TableDroppedNotification.java | 34 + .../TablePreScrubNotification.java | 31 + .../notifications/TruncationNotification.java | 14 +- .../cassandra/repair/PreviewRepairTask.java | 3 +- .../service/ActiveRepairService.java | 7 +- .../cassandra/service/CassandraDaemon.java | 13 +- .../service/SnapshotVerbHandler.java | 16 +- .../cassandra/service/StorageService.java | 388 +---- .../service/StorageServiceMBean.java | 25 +- .../snapshot/AbstractSnapshotTask.java | 42 + .../service/snapshot/ClearSnapshotTask.java | 196 +++ .../service/snapshot/GetSnapshotsTask.java | 99 ++ .../service/snapshot/ListSnapshotsTask.java | 148 ++ .../snapshot}/SnapshotDetailsTabularData.java | 39 +- .../service/snapshot/SnapshotException.java | 32 + .../service/snapshot/SnapshotLoader.java | 9 +- .../service/snapshot/SnapshotManager.java | 552 +++++++- .../snapshot/SnapshotManagerMBean.java | 109 ++ .../service/snapshot/SnapshotManifest.java | 7 +- .../service/snapshot/SnapshotOptions.java | 235 ++++ .../service/snapshot/SnapshotType.java | 42 + .../service/snapshot/TableSnapshot.java | 378 +++-- .../service/snapshot/TakeSnapshotTask.java | 310 ++++ .../snapshot/TrueSnapshotSizeTask.java | 76 + .../org/apache/cassandra/tools/NodeProbe.java | 30 +- .../tools/nodetool/ListSnapshots.java | 2 +- .../cassandra/tools/nodetool/Snapshot.java | 21 +- .../utils/DiagnosticSnapshotService.java | 26 +- .../utils/DirectorySizeCalculator.java | 4 +- .../cassandra/distributed/impl/Instance.java | 11 +- .../mock/nodetool/InternalNodeProbe.java | 2 + .../test/AllowAutoSnapshotTest.java | 6 +- .../distributed/test/AutoSnapshotTtlTest.java | 24 +- .../test/EphemeralSnapshotTest.java | 9 + .../test/PreviewRepairSnapshotTest.java | 5 +- .../distributed/test/PreviewRepairTest.java | 5 +- .../test/RepairDigestTrackingTest.java | 11 +- .../test/SSTableIdGenerationTest.java | 27 +- .../distributed/test/SnapshotsTest.java | 109 +- .../fuzz/snapshots/SnapshotsTest.java | 1251 +++++++++++++++++ .../AbstractSnapshotManagerBase.java | 167 +++ .../test/microbench/CompactionBench.java | 4 +- .../test/microbench/SnapshotListingBench.java | 63 + .../test/microbench/SnapshotTakingBench.java | 64 + .../microbench/SnapshotTrueSizeBench.java | 65 + test/unit/org/apache/cassandra/Util.java | 34 +- .../org/apache/cassandra/cql3/CQLTester.java | 2 + .../operations/AutoSnapshotTest.java | 29 +- .../cassandra/db/ColumnFamilyStoreTest.java | 287 +++- .../apache/cassandra/db/DirectoriesTest.java | 96 +- .../org/apache/cassandra/db/KeyspaceTest.java | 34 +- .../cassandra/db/SchemaCQLHelperTest.java | 11 +- .../org/apache/cassandra/db/SnapshotTest.java | 4 +- .../cassandra/db/SystemKeyspaceTest.java | 13 +- .../db/virtual/SnapshotsTableTest.java | 59 +- .../apache/cassandra/index/sai/SAITester.java | 6 +- .../cassandra/index/sasi/SASIIndexTest.java | 8 +- .../cassandra/schema/SchemaKeyspaceTest.java | 5 +- .../service/ActiveRepairServiceTest.java | 3 +- .../service/StorageServiceServerTest.java | 14 +- .../snapshot/MetadataSnapshotsTest.java | 162 +-- .../service/snapshot/SnapshotLoaderTest.java | 8 + .../service/snapshot/SnapshotManagerTest.java | 274 ++++ .../service/snapshot/SnapshotOptionsTest.java | 64 + .../service/snapshot/TableSnapshotTest.java | 118 +- .../StandaloneUpgraderOnSStablesTest.java | 7 +- .../tools/nodetool/ClearSnapshotTest.java | 12 + .../cassandra/stress/CompactionStress.java | 4 +- 79 files changed, 5142 insertions(+), 1523 deletions(-) create mode 100644 src/java/org/apache/cassandra/notifications/TableDroppedNotification.java create mode 100644 src/java/org/apache/cassandra/notifications/TablePreScrubNotification.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/AbstractSnapshotTask.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/GetSnapshotsTask.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/ListSnapshotsTask.java rename src/java/org/apache/cassandra/{db => service/snapshot}/SnapshotDetailsTabularData.java (65%) create mode 100644 src/java/org/apache/cassandra/service/snapshot/SnapshotException.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/SnapshotManagerMBean.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/SnapshotOptions.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/SnapshotType.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/TakeSnapshotTask.java create mode 100644 src/java/org/apache/cassandra/service/snapshot/TrueSnapshotSizeTask.java create mode 100644 test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/AbstractSnapshotManagerBase.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/SnapshotListingBench.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/SnapshotTakingBench.java create mode 100644 test/microbench/org/apache/cassandra/test/microbench/SnapshotTrueSizeBench.java create mode 100644 test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java create mode 100644 test/unit/org/apache/cassandra/service/snapshot/SnapshotOptionsTest.java diff --git a/CHANGES.txt b/CHANGES.txt index c9d9cb17e0..88443f88c3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Consolidate all snapshot management to SnapshotManager and introduce SnapshotManagerMBean (CASSANDRA-18111) * Fix RequestFailureReason constants codes (CASSANDRA-20126) * Introduce SSTableSimpleScanner for compaction (CASSANDRA-20092) * Include column drop timestamp in alter table transformation (CASSANDRA-18961) diff --git a/NEWS.txt b/NEWS.txt index b4511c3bba..740be2eb96 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -89,6 +89,9 @@ New features when 'GENERATED PASSWORD' clause is used. Character sets supported are: English, Cyrillic, modern Cyrillic, German, Polish and Czech. - JMX SSL configuration can be now done in cassandra.yaml via jmx_encryption_options section instead of cassandra-env.sh + - There is new MBean of name org.apache.cassandra.service.snapshot:type=SnapshotManager which exposes user-facing + snapshot operations. Snapshot-related methods on StorageServiceMBean are still present and functional + but marked as deprecated. Upgrading diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index d56d690f4a..cbd3d97c6e 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -18,11 +18,9 @@ package org.apache.cassandra.db; import java.io.IOException; -import java.io.PrintStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -32,7 +30,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -68,7 +65,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +75,6 @@ import org.apache.cassandra.cache.RowCacheSentinel; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.concurrent.FutureTask; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -116,8 +111,6 @@ import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.index.internal.CassandraIndex; import org.apache.cassandra.index.transactions.UpdateTransaction; -import org.apache.cassandra.io.FSReadError; -import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.IScrubber; @@ -131,7 +124,6 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.File; -import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.metrics.Sampler; import org.apache.cassandra.metrics.Sampler.Sample; import org.apache.cassandra.metrics.Sampler.SamplerType; @@ -157,9 +149,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.service.paxos.TablePaxosRepairHistory; -import org.apache.cassandra.service.snapshot.SnapshotLoader; -import org.apache.cassandra.service.snapshot.SnapshotManifest; -import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.TableStreamManager; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; @@ -185,7 +175,6 @@ import static org.apache.cassandra.config.DatabaseDescriptor.getFlushWriters; import static org.apache.cassandra.db.commitlog.CommitLogPosition.NONE; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.apache.cassandra.utils.FBUtilities.now; import static org.apache.cassandra.utils.Throwables.maybeFail; import static org.apache.cassandra.utils.Throwables.merge; import static org.apache.cassandra.utils.concurrent.CountDownLatch.newCountDownLatch; @@ -260,8 +249,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner private static final String SAMPLING_RESULTS_NAME = "SAMPLING_RESULTS"; - public static final String SNAPSHOT_TRUNCATE_PREFIX = "truncated"; - public static final String SNAPSHOT_DROP_PREFIX = "dropped"; static final String TOKEN_DELIMITER = ":"; /** Special values used when the local ranges are not changed with ring changes (e.g. local tables). */ @@ -517,6 +504,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner // Note that this needs to happen before we load the first sstables, or the global sstable tracker will not // be notified on the initial loading. data.subscribe(StorageService.instance.sstablesTracker); + data.subscribe(SnapshotManager.instance); Collection sstables = null; // scan for sstables corresponding to this cf and load them @@ -785,14 +773,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner * Removes unnecessary files from the cf directory at startup: these include temp files, orphans, zero-length files * and compacted sstables. Files that cannot be recognized will be ignored. */ - public static void scrubDataDirectories(TableMetadata metadata) throws StartupException + public static void scrubDataDirectories(TableMetadata metadata) throws StartupException { Directories directories = new Directories(metadata); Set cleanedDirectories = new HashSet<>(); - // clear ephemeral snapshots that were not properly cleared last session (CASSANDRA-7357) - clearEphemeralSnapshots(directories); - directories.removeTemporaryDirectories(); logger.trace("Removing temporary or obsoleted files from unfinished operations for table {}", metadata.name); @@ -977,6 +962,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return keyspace.getName(); } + public String getKeyspaceTableName() + { + return getKeyspaceName() + '.' + getTableName(); + } + public Descriptor newSSTableDescriptor(File directory) { return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion()); @@ -1757,12 +1747,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner public CompactionManager.AllSSTableOpStatus scrub(boolean disableSnapshot, boolean alwaysFail, IScrubber.Options options, int jobs) throws ExecutionException, InterruptedException { // skip snapshot creation during scrub, SEE JIRA 5891 - if(!disableSnapshot) - { - Instant creationTime = now(); - String snapshotName = "pre-scrub-" + creationTime.toEpochMilli(); - snapshotWithoutMemtable(snapshotName, creationTime); - } + if (!disableSnapshot) + data.notifyPreScrubbed(); try { @@ -2127,262 +2113,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return metadata().comparator; } - public TableSnapshot snapshotWithoutMemtable(String snapshotName) - { - return snapshotWithoutMemtable(snapshotName, now()); - } - - public TableSnapshot snapshotWithoutMemtable(String snapshotName, Instant creationTime) - { - return snapshotWithoutMemtable(snapshotName, null, false, null, null, creationTime); - } - - /** - * @param ephemeral If this flag is set to true, the snapshot will be cleaned during next startup - */ - public TableSnapshot snapshotWithoutMemtable(String snapshotName, Predicate predicate, boolean ephemeral, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime) - { - if (ephemeral && ttl != null) - { - throw new IllegalStateException(String.format("can not take ephemeral snapshot (%s) while ttl is specified too", snapshotName)); - } - - if (rateLimiter == null) - rateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); - - Set snapshottedSSTables = new LinkedHashSet<>(); - for (ColumnFamilyStore cfs : concatWithIndexes()) - { - try (RefViewFragment currentView = cfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.apply(x)))) - { - for (SSTableReader ssTable : currentView.sstables) - { - File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName); - ssTable.createLinks(snapshotDirectory.path(), rateLimiter); // hard links - if (logger.isTraceEnabled()) - logger.trace("Snapshot for {} keyspace data file {} created in {}", keyspace, ssTable.getFilename(), snapshotDirectory); - snapshottedSSTables.add(ssTable); - } - } - } - - return createSnapshot(snapshotName, ephemeral, ttl, snapshottedSSTables, creationTime); - } - - protected TableSnapshot createSnapshot(String tag, boolean ephemeral, DurationSpec.IntSecondsBound ttl, Set sstables, Instant creationTime) { - Set snapshotDirs = sstables.stream() - .map(s -> Directories.getSnapshotDirectory(s.descriptor, tag).toAbsolute()) - .filter(dir -> !Directories.isSecondaryIndexFolder(dir)) // Remove secondary index subdirectory - .collect(Collectors.toCollection(HashSet::new)); - - // Create and write snapshot manifest - SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime, ephemeral); - File manifestFile = getDirectories().getSnapshotManifestFile(tag); - writeSnapshotManifest(manifest, manifestFile); - snapshotDirs.add(manifestFile.parent().toAbsolute()); // manifest may create empty snapshot dir - - // Write snapshot schema - if (!SchemaConstants.isLocalSystemKeyspace(metadata.keyspace) && !SchemaConstants.isReplicatedSystemKeyspace(metadata.keyspace)) - { - File schemaFile = getDirectories().getSnapshotSchemaFile(tag); - writeSnapshotSchema(schemaFile); - snapshotDirs.add(schemaFile.parent().toAbsolute()); // schema may create empty snapshot dir - } - - TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), - tag, manifest.createdAt, manifest.expiresAt, snapshotDirs, - manifest.ephemeral); - - StorageService.instance.addSnapshot(snapshot); - return snapshot; - } - - private SnapshotManifest writeSnapshotManifest(SnapshotManifest manifest, File manifestFile) - { - try - { - manifestFile.parent().tryCreateDirectories(); - manifest.serializeToJsonFile(manifestFile); - return manifest; - } - catch (IOException e) - { - throw new FSWriteError(e, manifestFile); - } - } - - private List mapToDataFilenames(Collection sstables) - { - return sstables.stream().map(s -> s.descriptor.relativeFilenameFor(Components.DATA)).collect(Collectors.toList()); - } - - private void writeSnapshotSchema(File schemaFile) - { - try - { - if (!schemaFile.parent().exists()) - schemaFile.parent().tryCreateDirectories(); - - try (PrintStream out = new PrintStream(new FileOutputStreamPlus(schemaFile))) - { - SchemaCQLHelper.reCreateStatementsForSchemaCql(metadata(), - keyspace.getMetadata()) - .forEach(out::println); - } - } - catch (IOException e) - { - throw new FSWriteError(e, schemaFile); - } - } - - protected static void clearEphemeralSnapshots(Directories directories) - { - RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); - - List ephemeralSnapshots = new SnapshotLoader(directories).loadSnapshots() - .stream() - .filter(TableSnapshot::isEphemeral) - .collect(Collectors.toList()); - - for (TableSnapshot ephemeralSnapshot : ephemeralSnapshots) - { - logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot.getId()); - Directories.clearSnapshot(ephemeralSnapshot.getTag(), directories.getCFDirectories(), clearSnapshotRateLimiter); - } - } - - public Refs getSnapshotSSTableReaders(String tag) throws IOException - { - Map active = new HashMap<>(); - for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL)) - active.put(sstable.descriptor.id, sstable); - Map> snapshots = getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list(); - Refs refs = new Refs<>(); - try - { - for (Map.Entry> entries : snapshots.entrySet()) - { - // Try acquire reference to an active sstable instead of snapshot if it exists, - // to avoid opening new sstables. If it fails, use the snapshot reference instead. - SSTableReader sstable = active.get(entries.getKey().id); - if (sstable == null || !refs.tryRef(sstable)) - { - if (logger.isTraceEnabled()) - logger.trace("using snapshot sstable {}", entries.getKey()); - // open offline so we don't modify components or track hotness. - sstable = SSTableReader.open(this, entries.getKey(), entries.getValue(), metadata, true, true); - refs.tryRef(sstable); - // release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released - sstable.selfRef().release(); - } - else if (logger.isTraceEnabled()) - { - logger.trace("using active sstable {}", entries.getKey()); - } - } - } - catch (FSReadError | RuntimeException e) - { - // In case one of the snapshot sstables fails to open, - // we must release the references to the ones we opened so far - refs.release(); - throw e; - } - return refs; - } - - /** - * Take a snap shot of this columnfamily store. - * - * @param snapshotName the name of the associated with the snapshot - */ - public TableSnapshot snapshot(String snapshotName) - { - return snapshot(snapshotName, null); - } - - public TableSnapshot snapshot(String snapshotName, DurationSpec.IntSecondsBound ttl) - { - return snapshot(snapshotName, false, ttl, null, now()); - } - - /** - * Take a snap shot of this columnfamily store. - * - * @param snapshotName the name of the associated with the snapshot - * @param skipMemtable Skip flushing the memtable - * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed - * @param rateLimiter Rate limiter for hardlinks-per-second - * @param creationTime time when this snapshot was taken - */ - public TableSnapshot snapshot(String snapshotName, boolean skipMemtable, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime) - { - return snapshot(snapshotName, null, false, skipMemtable, ttl, rateLimiter, creationTime); - } - - - /** - * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup - * @param skipMemtable Skip flushing the memtable - */ - public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipMemtable) - { - return snapshot(snapshotName, predicate, ephemeral, skipMemtable, null, null, now()); - } - - /** - * @param ephemeral If this flag is set to true, the snapshot will be cleaned up during next startup - * @param skipMemtable Skip flushing the memtable - * @param ttl duration after which the taken snapshot is removed automatically, if supplied with null, it will never be automatically removed - * @param rateLimiter Rate limiter for hardlinks-per-second - * @param creationTime time when this snapshot was taken - */ - public TableSnapshot snapshot(String snapshotName, Predicate predicate, boolean ephemeral, boolean skipMemtable, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime) - { - if (!skipMemtable) - { - Memtable current = getTracker().getView().getCurrentMemtable(); - if (!current.isClean()) - { - if (current.shouldSwitch(FlushReason.SNAPSHOT)) - FBUtilities.waitOnFuture(switchMemtableIfCurrent(current, FlushReason.SNAPSHOT)); - else - current.performSnapshot(snapshotName); - } - } - return snapshotWithoutMemtable(snapshotName, predicate, ephemeral, ttl, rateLimiter, creationTime); - } - - public boolean snapshotExists(String snapshotName) - { - return getDirectories().snapshotExists(snapshotName); - } - - - /** - * Clear all the snapshots for a given column family. - * - * @param snapshotName the user supplied snapshot name. If left empty, - * all the snapshots will be cleaned. - */ - public void clearSnapshot(String snapshotName) - { - RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); - - List snapshotDirs = getDirectories().getCFDirectories(); - Directories.clearSnapshot(snapshotName, snapshotDirs, clearSnapshotRateLimiter); - } - /** - * - * @return Return a map of all snapshots to space being used - * The pair for a snapshot has true size and size on disk. - */ - public Map listSnapshots() - { - return getDirectories().listSnapshots(); - } - /** * @return the cached partition for @param key if it is already present in the cache. * Not that this will not readAndCache the parition if it is not present, nor @@ -2771,15 +2501,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner // stream in data that is actually supposed to have been deleted ActiveRepairService.instance().abort((prs) -> prs.getTableIds().contains(metadata.id), "Stopping parent sessions {} due to truncation of tableId="+metadata.id); - data.notifyTruncated(truncatedAt); + data.notifyTruncated(noSnapshot, truncatedAt, DatabaseDescriptor.getAutoSnapshotTtl()); - if (!noSnapshot && isAutoSnapshotEnabled()) - snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, SNAPSHOT_TRUNCATE_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl()); + discardSSTables(truncatedAt); - discardSSTables(truncatedAt); - - indexManager.truncateAllIndexesBlocking(truncatedAt); - viewManager.truncateBlocking(replayAfter, truncatedAt); + indexManager.truncateAllIndexesBlocking(truncatedAt); + viewManager.truncateBlocking(replayAfter, truncatedAt); SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter); logger.trace("cleaning out row cache"); @@ -3303,10 +3030,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return allColumns > 0 ? allDroppable / allColumns : 0; } + public Set getFilesOfCfs() + { + Set files = new HashSet<>(); + + for (ColumnFamilyStore cfs : concatWithIndexes()) + cfs.getTracker().getView().liveSSTables().forEach(s -> files.addAll(s.getAllFilePaths())); + + return files; + } + @Override public long trueSnapshotsSize() { - return getDirectories().trueSnapshotsSize(); + return SnapshotManager.instance.getTrueSnapshotsSize(getKeyspaceName(), getTableName()); } /** @@ -3426,8 +3163,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner CompactionManager.instance.interruptCompactionForCFs(concatWithIndexes(), (sstable) -> true, true); - if (isAutoSnapshotEnabled()) - snapshot(Keyspace.getTimestampedSnapshotNameWithPrefix(name, ColumnFamilyStore.SNAPSHOT_DROP_PREFIX), DatabaseDescriptor.getAutoSnapshotTtl()); + data.notifyDropped(DatabaseDescriptor.getAutoSnapshotTtl()); CommitLog.instance.forceRecycleAllSegments(Collections.singleton(metadata.id)); diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java index c6e30c961d..12a64090f8 100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,8 +48,6 @@ import java.util.stream.StreamSupport; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.RateLimiter; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,8 +71,6 @@ import org.apache.cassandra.io.util.PathUtils; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.snapshot.SnapshotManifest; -import org.apache.cassandra.service.snapshot.TableSnapshot; -import org.apache.cassandra.utils.DirectorySizeCalculator; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Pair; @@ -306,6 +301,11 @@ public class Directories canonicalPathToDD = canonicalPathsBuilder.build(); } + public File[] getDataPaths() + { + return dataPaths; + } + /** * Returns SSTable location which is inside given data directory. * @@ -673,6 +673,16 @@ public class Directories } } + public Set getSnapshotDirs(String tag) + { + Set snapshotDirs = new HashSet<>(); + + for (File cfDir : getCFDirectories()) + snapshotDirs.add(Directories.getSnapshotDirectory(cfDir, tag).toAbsolute()); + + return snapshotDirs; + } + public File getSnapshotManifestFile(String snapshotName) { File snapshotDir = getSnapshotDirectory(getDirectoryForNewSSTables(), snapshotName); @@ -1171,37 +1181,6 @@ public class Directories } } - public Map listSnapshots() - { - Map> snapshotDirsByTag = listSnapshotDirsByTag(); - - Map snapshots = Maps.newHashMapWithExpectedSize(snapshotDirsByTag.size()); - - for (Map.Entry> entry : snapshotDirsByTag.entrySet()) - { - String tag = entry.getKey(); - Set snapshotDirs = entry.getValue(); - SnapshotManifest manifest = maybeLoadManifest(metadata.keyspace, metadata.name, tag, snapshotDirs); - snapshots.put(tag, buildSnapshot(tag, manifest, snapshotDirs)); - } - - return snapshots; - } - - private TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set snapshotDirs) - { - boolean ephemeral = manifest != null ? manifest.isEphemeral() : isLegacyEphemeralSnapshot(snapshotDirs); - Instant createdAt = manifest == null ? null : manifest.createdAt; - Instant expiresAt = manifest == null ? null : manifest.expiresAt; - return new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag, createdAt, expiresAt, - snapshotDirs, ephemeral); - } - - private static boolean isLegacyEphemeralSnapshot(Set snapshotDirs) - { - return snapshotDirs.stream().map(d -> new File(d, "ephemeral.snapshot")).anyMatch(File::exists); - } - @VisibleForTesting protected static SnapshotManifest maybeLoadManifest(String keyspace, String table, String tag, Set snapshotDirs) { @@ -1230,100 +1209,10 @@ public class Directories return null; } - @VisibleForTesting - protected Map> listSnapshotDirsByTag() - { - Map> snapshotDirsByTag = new HashMap<>(); - for (final File dir : dataPaths) - { - File snapshotDir = isSecondaryIndexFolder(dir) - ? new File(dir.parentPath(), SNAPSHOT_SUBDIR) - : new File(dir, SNAPSHOT_SUBDIR); - if (snapshotDir.exists() && snapshotDir.isDirectory()) - { - final File[] snapshotDirs = snapshotDir.tryList(); - if (snapshotDirs != null) - { - for (final File snapshot : snapshotDirs) - { - if (snapshot.isDirectory()) { - snapshotDirsByTag.computeIfAbsent(snapshot.name(), k -> new LinkedHashSet<>()).add(snapshot.toAbsolute()); - } - } - } - } - } - return snapshotDirsByTag; - } - - public boolean snapshotExists(String snapshotName) - { - for (File dir : dataPaths) - { - File snapshotDir; - if (isSecondaryIndexFolder(dir)) - { - snapshotDir = new File(dir.parent(), join(SNAPSHOT_SUBDIR, snapshotName, dir.name())); - } - else - { - snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName)); - } - if (snapshotDir.exists()) - return true; - } - return false; - } - - public static void clearSnapshot(String snapshotName, List tableDirectories, RateLimiter snapshotRateLimiter) - { - // If snapshotName is empty or null, we will delete the entire snapshot directory - String tag = snapshotName == null ? "" : snapshotName; - for (File tableDir : tableDirectories) - { - File snapshotDir = new File(tableDir, join(SNAPSHOT_SUBDIR, tag)); - removeSnapshotDirectory(snapshotRateLimiter, snapshotDir); - } - } - - public static void removeSnapshotDirectory(RateLimiter snapshotRateLimiter, File snapshotDir) - { - if (snapshotDir.exists()) - { - logger.trace("Removing snapshot directory {}", snapshotDir); - try - { - FileUtils.deleteRecursiveWithThrottle(snapshotDir, snapshotRateLimiter); - } - catch (RuntimeException ex) - { - if (!snapshotDir.exists()) - return; // ignore - throw ex; - } - } - } - - /** - * @return total snapshot size in byte for all snapshots. - */ - public long trueSnapshotsSize() - { - long result = 0L; - for (File dir : dataPaths) - { - File snapshotDir = isSecondaryIndexFolder(dir) - ? new File(dir.parentPath(), SNAPSHOT_SUBDIR) - : new File(dir, SNAPSHOT_SUBDIR); - result += getTrueAllocatedSizeIn(snapshotDir); - } - return result; - } - /** * @return Raw size on disk for all directories */ - public long getRawDiretoriesSize() + public long getRawDirectoriesSize() { long totalAllocatedSize = 0L; @@ -1333,25 +1222,7 @@ public class Directories return totalAllocatedSize; } - public long getTrueAllocatedSizeIn(File snapshotDir) - { - if (!snapshotDir.isDirectory()) - return 0; - - SSTableSizeSummer visitor = new SSTableSizeSummer(sstableLister(OnTxnErr.THROW).listFiles()); - try - { - Files.walkFileTree(snapshotDir.toPath(), visitor); - } - catch (IOException e) - { - logger.error("Could not calculate the size of {}. {}", snapshotDir, e.getMessage()); - } - - return visitor.getAllocatedSize(); - } - - // Recursively finds all the sub directories in the KS directory. + // Recursively finds all the subdirectories in the KS directory. public static List getKSChildDirectories(String ksName) { List result = new ArrayList<>(); @@ -1434,25 +1305,4 @@ public class Directories { return StringUtils.join(s, File.pathSeparator()); } - - private class SSTableSizeSummer extends DirectorySizeCalculator - { - private final Set toSkip; - SSTableSizeSummer(List files) - { - toSkip = files.stream().map(File::name).collect(Collectors.toSet()); - } - - @Override - public boolean isAcceptable(Path path) - { - File file = new File(path); - Descriptor desc = SSTable.tryDescriptorFromFile(file); - return desc != null - && desc.ksname.equals(metadata.keyspace) - && desc.cfname.equals(metadata.name) - && !toSkip.contains(file.name()); - } - } - } diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index 7a18f112ba..af651570bb 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -17,8 +17,6 @@ */ package org.apache.cassandra.db; -import java.io.IOException; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,18 +33,17 @@ import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.RateLimiter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.Stage; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager; import org.apache.cassandra.db.view.ViewManager; +import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexManager; @@ -61,7 +58,6 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -74,7 +70,6 @@ import org.apache.cassandra.utils.concurrent.Promise; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.cassandra.utils.FBUtilities.now; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; /** @@ -226,80 +221,19 @@ public class Keyspace return columnFamilyStores.containsKey(id); } - /** - * Take a snapshot of the specific column family, or the entire set of column families - * if columnFamily is null with a given timestamp - * - * @param snapshotName the tag associated with the name of the snapshot. This value may not be null - * @param columnFamilyName the column family to snapshot or all on null - * @param skipFlush Skip blocking flush of memtable - * @param rateLimiter Rate limiter for hardlinks-per-second - * @throws IOException if the column family doesn't exist - */ - public void snapshot(String snapshotName, String columnFamilyName, boolean skipFlush, DurationSpec.IntSecondsBound ttl, RateLimiter rateLimiter, Instant creationTime) throws IOException + public static void verifyKeyspaceIsValid(String keyspaceName) { - assert snapshotName != null; - boolean tookSnapShot = false; - for (ColumnFamilyStore cfStore : columnFamilyStores.values()) - { - if (columnFamilyName == null || cfStore.name.equals(columnFamilyName)) - { - tookSnapShot = true; - cfStore.snapshot(snapshotName, skipFlush, ttl, rateLimiter, creationTime); - } - } + if (null != VirtualKeyspaceRegistry.instance.getKeyspaceNullable(keyspaceName)) + throw new IllegalArgumentException("Cannot perform any operations against virtual keyspace " + keyspaceName); - if ((columnFamilyName != null) && !tookSnapShot) - throw new IOException("Failed taking snapshot. Table " + columnFamilyName + " does not exist."); + if (!Schema.instance.getKeyspaces().contains(keyspaceName)) + throw new IllegalArgumentException("Keyspace " + keyspaceName + " does not exist"); } - /** - * Take a snapshot of the specific column family, or the entire set of column families - * if columnFamily is null with a given timestamp - * - * @param snapshotName the tag associated with the name of the snapshot. This value may not be null - * @param columnFamilyName the column family to snapshot or all on null - * @throws IOException if the column family doesn't exist - */ - public void snapshot(String snapshotName, String columnFamilyName) throws IOException + public static Keyspace getValidKeyspace(String keyspaceName) { - snapshot(snapshotName, columnFamilyName, false, null, null, now()); - } - - /** - * @param clientSuppliedName may be null. - * @return the name of the snapshot - */ - public static String getTimestampedSnapshotName(String clientSuppliedName) - { - String snapshotName = Long.toString(currentTimeMillis()); - if (clientSuppliedName != null && !clientSuppliedName.equals("")) - { - snapshotName = snapshotName + "-" + clientSuppliedName; - } - return snapshotName; - } - - public static String getTimestampedSnapshotNameWithPrefix(String clientSuppliedName, String prefix) - { - return prefix + "-" + getTimestampedSnapshotName(clientSuppliedName); - } - - /** - * Check whether snapshots already exists for a given name. - * - * @param snapshotName the user supplied snapshot name - * @return true if the snapshot exists - */ - public boolean snapshotExists(String snapshotName) - { - assert snapshotName != null; - for (ColumnFamilyStore cfStore : columnFamilyStores.values()) - { - if (cfStore.snapshotExists(snapshotName)) - return true; - } - return false; + verifyKeyspaceIsValid(keyspaceName); + return Keyspace.open(keyspaceName); } /** @@ -313,11 +247,6 @@ public class Keyspace return list; } - public Stream getAllSnapshots() - { - return getColumnFamilyStores().stream().flatMap(cfs -> cfs.listSnapshots().values().stream()); - } - public static Keyspace forSchema(String keyspaceName, SchemaProvider schema) { return new Keyspace(keyspaceName, schema, true); diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index 05a2437546..2130d568d2 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -109,6 +109,9 @@ import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.service.paxos.uncommitted.PaxosRows; import org.apache.cassandra.service.paxos.uncommitted.PaxosUncommittedIndex; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; @@ -141,10 +144,10 @@ import static org.apache.cassandra.gms.ApplicationState.RELEASE_VERSION; import static org.apache.cassandra.gms.ApplicationState.STATUS_WITH_PORT; import static org.apache.cassandra.gms.ApplicationState.TOKENS; import static org.apache.cassandra.service.paxos.Commit.latest; +import static org.apache.cassandra.service.snapshot.SnapshotOptions.systemSnapshot; import static org.apache.cassandra.utils.CassandraVersion.NULL_VERSION; import static org.apache.cassandra.utils.CassandraVersion.UNREADABLE_VERSION; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.cassandra.utils.FBUtilities.now; public final class SystemKeyspace { @@ -1808,10 +1811,8 @@ public final class SystemKeyspace * Compare the release version in the system.local table with the one included in the distro. * If they don't match, snapshot all tables in the system and schema keyspaces. This is intended * to be called at startup to create a backup of the system tables during an upgrade - * - * @throws IOException */ - public static void snapshotOnVersionChange() throws IOException + public static void snapshotOnVersionChange() { String previous = getPreviousVersionString(); String next = FBUtilities.getReleaseVersionString(); @@ -1820,16 +1821,18 @@ public final class SystemKeyspace // if we're restarting after an upgrade, snapshot the system and schema keyspaces if (!previous.equals(NULL_VERSION.toString()) && !previous.equals(next)) - { - logger.info("Detected version upgrade from {} to {}, snapshotting system keyspaces", previous, next); - String snapshotName = Keyspace.getTimestampedSnapshotName(format("upgrade-%s-%s", - previous, - next)); - - Instant creationTime = now(); + List entities = new ArrayList<>(); for (String keyspace : SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES) - Keyspace.open(keyspace).snapshot(snapshotName, null, false, null, null, creationTime); + { + for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores()) + entities.add(cfs.getKeyspaceTableName()); + } + + logger.info("Detected version upgrade from {} to {}, snapshotting system keyspaces", previous, next); + + SnapshotOptions options = systemSnapshot(format("%s-%s", previous, next), SnapshotType.UPGRADE, entities.toArray(new String[0])).build(); + SnapshotManager.instance.takeSnapshot(options); } } diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 79368f68d3..4f27736f26 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.db.compaction; -import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -46,6 +45,9 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.io.util.File; import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Refs; @@ -53,7 +55,6 @@ import org.apache.cassandra.utils.concurrent.Refs; import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.Clock.Global.nanoTime; -import static org.apache.cassandra.utils.FBUtilities.now; public class CompactionTask extends AbstractCompactionTask { @@ -126,8 +127,8 @@ public class CompactionTask extends AbstractCompactionTask if (DatabaseDescriptor.isSnapshotBeforeCompaction()) { - Instant creationTime = now(); - cfs.snapshotWithoutMemtable(creationTime.toEpochMilli() + "-compact-" + cfs.name, creationTime); + SnapshotOptions options = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.COMPACT, cfs.getKeyspaceTableName()).skipFlush().build(); + SnapshotManager.instance.takeSnapshot(options); } try (CompactionController controller = getCompactionController(transaction.originals())) diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java index 58ffc1df3f..adc18cd3b5 100644 --- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java +++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -56,6 +57,8 @@ import org.apache.cassandra.notifications.SSTableDeletingNotification; import org.apache.cassandra.notifications.SSTableListChangedNotification; import org.apache.cassandra.notifications.SSTableMetadataChanged; import org.apache.cassandra.notifications.SSTableRepairStatusChanged; +import org.apache.cassandra.notifications.TableDroppedNotification; +import org.apache.cassandra.notifications.TablePreScrubNotification; import org.apache.cassandra.notifications.TruncationNotification; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; @@ -510,31 +513,34 @@ public class Tracker { if (repairStatusesChanged.isEmpty()) return; - INotification notification = new SSTableRepairStatusChanged(repairStatusesChanged); - for (INotificationConsumer subscriber : subscribers) - subscriber.handleNotification(notification, this); + notify(new SSTableRepairStatusChanged(repairStatusesChanged)); } public void notifySSTableMetadataChanged(SSTableReader levelChanged, StatsMetadata oldMetadata) { - INotification notification = new SSTableMetadataChanged(levelChanged, oldMetadata); - for (INotificationConsumer subscriber : subscribers) - subscriber.handleNotification(notification, this); - + notify(new SSTableMetadataChanged(levelChanged, oldMetadata)); } public void notifyDeleting(SSTableReader deleting) { - INotification notification = new SSTableDeletingNotification(deleting); - for (INotificationConsumer subscriber : subscribers) - subscriber.handleNotification(notification, this); + notify(new SSTableDeletingNotification(deleting)); } - public void notifyTruncated(long truncatedAt) + public void notifyTruncated(boolean disableSnapshot, + long truncatedAt, + DurationSpec.IntSecondsBound ttl) { - INotification notification = new TruncationNotification(truncatedAt); - for (INotificationConsumer subscriber : subscribers) - subscriber.handleNotification(notification, this); + notify(new TruncationNotification(cfstore, disableSnapshot, truncatedAt, ttl)); + } + + public void notifyDropped(DurationSpec.IntSecondsBound ttl) + { + notify(new TableDroppedNotification(cfstore, ttl)); + } + + public void notifyPreScrubbed() + { + notify(new TablePreScrubNotification(cfstore)); } public void notifyRenewed(Memtable renewed) diff --git a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java index 24e79d2454..3b6535752e 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java @@ -22,8 +22,7 @@ import java.io.IOException; import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.Future; - -import com.google.common.base.Predicate; +import java.util.function.Predicate; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.compaction.CompactionManager; @@ -36,6 +35,9 @@ import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.repair.TableRepairManager; import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.repair.NoSuchRepairSessionException; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.service.ActiveRepairService; @@ -79,17 +81,14 @@ public class CassandraTableRepairManager implements TableRepairManager try { ActiveRepairService.instance().snapshotExecutor.submit(() -> { - if (force || !cfs.snapshotExists(name)) + if (force || !SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), name)) { - cfs.snapshot(name, new Predicate() - { - public boolean apply(SSTableReader sstable) - { - return sstable != null && - !sstable.metadata().isIndex() && // exclude SSTables from 2i - new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges); - } - }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup + Predicate predicate = sstable -> sstable != null && + !sstable.metadata().isIndex() && // exclude SSTables from 2i + new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges); + + SnapshotOptions options = SnapshotOptions.systemSnapshot(name, SnapshotType.REPAIR, predicate, cfs.getKeyspaceTableName()).ephemeral().build(); + SnapshotManager.instance.takeSnapshot(options); } }).get(); } diff --git a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java index 05408e9175..5d4d88ed9e 100644 --- a/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java +++ b/src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java @@ -57,6 +57,8 @@ import org.apache.cassandra.repair.ValidationPartitionIterator; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.repair.NoSuchRepairSessionException; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Refs; @@ -179,19 +181,19 @@ public class CassandraValidationIterator extends ValidationPartitionIterator this.cfs = cfs; this.ctx = ctx; - isGlobalSnapshotValidation = cfs.snapshotExists(parentId.toString()); + isGlobalSnapshotValidation = SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), parentId.toString()); if (isGlobalSnapshotValidation) snapshotName = parentId.toString(); else snapshotName = sessionID.toString(); - isSnapshotValidation = cfs.snapshotExists(snapshotName); + isSnapshotValidation = SnapshotManager.instance.exists(cfs.getKeyspaceName(), cfs.getTableName(), snapshotName); if (isSnapshotValidation) { // If there is a snapshot created for the session then read from there. // note that we populate the parent repair session when creating the snapshot, meaning the sstables in the snapshot are the ones we // are supposed to validate. - sstables = cfs.getSnapshotSSTableReaders(snapshotName); + sstables = TableSnapshot.getSnapshotSSTableReaders(cfs, snapshotName); } else { @@ -270,7 +272,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator { // we can only clear the snapshot if we are not doing a global snapshot validation (we then clear it once anticompaction // is done). - cfs.clearSnapshot(snapshotName); + SnapshotManager.instance.clearSnapshot(cfs.getKeyspaceName(), cfs.getTableName(), snapshotName); } if (sstables != null) diff --git a/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java b/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java index d3df293903..c757ce7c54 100644 --- a/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java +++ b/src/java/org/apache/cassandra/db/virtual/SnapshotsTable.java @@ -26,7 +26,7 @@ import org.apache.cassandra.db.marshal.TimestampType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.TableSnapshot; public class SnapshotsTable extends AbstractVirtualTable @@ -62,7 +62,7 @@ public class SnapshotsTable extends AbstractVirtualTable { SimpleDataSet result = new SimpleDataSet(metadata()); - for (TableSnapshot tableSnapshot : StorageService.instance.snapshotManager.loadSnapshots()) + for (TableSnapshot tableSnapshot : SnapshotManager.instance.getSnapshots(false, true)) { SimpleDataSet row = result.row(tableSnapshot.getTag(), tableSnapshot.getKeyspaceName(), diff --git a/src/java/org/apache/cassandra/notifications/TableDroppedNotification.java b/src/java/org/apache/cassandra/notifications/TableDroppedNotification.java new file mode 100644 index 0000000000..aea1e1c9ba --- /dev/null +++ b/src/java/org/apache/cassandra/notifications/TableDroppedNotification.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.notifications; + +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.db.ColumnFamilyStore; + +public class TableDroppedNotification implements INotification +{ + public final ColumnFamilyStore cfs; + public final DurationSpec.IntSecondsBound ttl; + + public TableDroppedNotification(ColumnFamilyStore cfs, DurationSpec.IntSecondsBound ttl) + { + this.cfs = cfs; + this.ttl = ttl; + } +} diff --git a/src/java/org/apache/cassandra/notifications/TablePreScrubNotification.java b/src/java/org/apache/cassandra/notifications/TablePreScrubNotification.java new file mode 100644 index 0000000000..c8fad45b8e --- /dev/null +++ b/src/java/org/apache/cassandra/notifications/TablePreScrubNotification.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.notifications; + +import org.apache.cassandra.db.ColumnFamilyStore; + +public class TablePreScrubNotification implements INotification +{ + public final ColumnFamilyStore cfs; + + public TablePreScrubNotification(ColumnFamilyStore cfs) + { + this.cfs = cfs; + } +} diff --git a/src/java/org/apache/cassandra/notifications/TruncationNotification.java b/src/java/org/apache/cassandra/notifications/TruncationNotification.java index 345dd17e29..cdf1ba515a 100644 --- a/src/java/org/apache/cassandra/notifications/TruncationNotification.java +++ b/src/java/org/apache/cassandra/notifications/TruncationNotification.java @@ -17,16 +17,28 @@ */ package org.apache.cassandra.notifications; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.db.ColumnFamilyStore; + /** * Fired during truncate, after the memtable has been flushed but before any * snapshot is taken and SSTables are discarded */ public class TruncationNotification implements INotification { + public final ColumnFamilyStore cfs; + public final boolean disableSnapshot; public final long truncatedAt; + public final DurationSpec.IntSecondsBound ttl; - public TruncationNotification(long truncatedAt) + public TruncationNotification(ColumnFamilyStore cfs, + boolean disableSnapshot, + long truncatedAt, + DurationSpec.IntSecondsBound ttl) { + this.cfs = cfs; + this.disableSnapshot = disableSnapshot; this.truncatedAt = truncatedAt; + this.ttl = ttl; } } diff --git a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java index edee11cf20..95c7a63f94 100644 --- a/src/java/org/apache/cassandra/repair/PreviewRepairTask.java +++ b/src/java/org/apache/cassandra/repair/PreviewRepairTask.java @@ -32,6 +32,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.RepairMetrics; import org.apache.cassandra.repair.consistent.SyncStatSummary; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.DiagnosticSnapshotService; import org.apache.cassandra.utils.TimeUUID; @@ -128,7 +129,7 @@ public class PreviewRepairTask extends AbstractRepairTask for (String table : mismatchingTables) { // we can just check snapshot existence locally since the repair coordinator is always a replica (unlike in the read case) - if (!Keyspace.open(keyspace).getColumnFamilyStore(table).snapshotExists(snapshotName)) + if (!SnapshotManager.instance.exists(keyspace, table, snapshotName)) { List> normalizedRanges = Range.normalize(ranges); logger.info("{} Snapshotting {}.{} for preview repair mismatch for ranges {} with tag {} on instances {}", diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index b5fbc48d43..1a60bd3067 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -106,6 +106,7 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.PaxosRepair; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanup; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.utils.ExecutorUtils; @@ -884,10 +885,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai .map(cfs -> cfs.metadata().toString()).collect(Collectors.joining(", "))); long startNanos = ctx.clock().nanoTime(); for (ColumnFamilyStore cfs : session.columnFamilyStores.values()) - { - if (cfs.snapshotExists(snapshotName)) - cfs.clearSnapshot(snapshotName); - } + SnapshotManager.instance.clearSnapshot(cfs.keyspace.getName(), cfs.name, snapshotName); + logger.info("[repair #{}] Cleared snapshots in {}ms", parentSessionId, TimeUnit.NANOSECONDS.toMillis(ctx.clock().nanoTime() - startNanos)); }); } diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index ec66cf2fe9..ef6c689edf 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -71,6 +71,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.security.ThreadAwareSecurityManager; import org.apache.cassandra.service.paxos.PaxosState; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tcm.ClusterMetadata; @@ -266,8 +267,16 @@ public class CassandraDaemon DatabaseDescriptor.createAllDirectories(); Keyspace.setInitialized(); CommitLog.instance.start(); - runStartupChecks(); + SnapshotManager.instance.start(false); + SnapshotManager.instance.clearExpiredSnapshots(); + SnapshotManager.instance.clearEphemeralSnapshots(); + SnapshotManager.instance.resumeSnapshotCleanup(); + SnapshotManager.instance.registerMBean(); + + // clearing of snapshots above here will in fact clear all ephemeral snapshots + // which were cleared as part of startup checks before CASSANDRA-18111 + runStartupChecks(); try { @@ -291,7 +300,7 @@ public class CassandraDaemon { SystemKeyspace.snapshotOnVersionChange(); } - catch (IOException e) + catch (Throwable e) { exitOrFail(StartupException.ERR_WRONG_DISK_STATE, e.getMessage(), e.getCause()); } diff --git a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java index db09578981..fc80089d37 100644 --- a/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java +++ b/src/java/org/apache/cassandra/service/SnapshotVerbHandler.java @@ -20,11 +20,13 @@ package org.apache.cassandra.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SnapshotCommand; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.utils.DiagnosticSnapshotService; @@ -38,7 +40,7 @@ public class SnapshotVerbHandler implements IVerbHandler SnapshotCommand command = message.payload; if (command.clear_snapshot) { - StorageService.instance.clearSnapshot(command.snapshot_name, command.keyspace); + SnapshotManager.instance.clearSnapshots(command.snapshot_name, command.keyspace); } else if (DiagnosticSnapshotService.isDiagnosticSnapshotRequest(command)) { @@ -46,7 +48,15 @@ public class SnapshotVerbHandler implements IVerbHandler } else { - Keyspace.open(command.keyspace).getColumnFamilyStore(command.column_family).snapshot(command.snapshot_name); + try + { + SnapshotOptions options = SnapshotOptions.systemSnapshot(command.snapshot_name, SnapshotType.DIAGNOSTICS, command.keyspace + '.' + command.column_family).build(); + SnapshotManager.instance.takeSnapshot(options); + } + catch (Exception ex) + { + throw new RuntimeException(ex); + } } logger.debug("Enqueuing response to snapshot request {} to {}", command.snapshot_name, message.from()); diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index ab8ddfe9bb..aa21f59ef2 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.time.Instant; -import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -59,7 +57,6 @@ import javax.management.NotificationListener; import javax.management.openmbean.CompositeData; import javax.management.openmbean.OpenDataException; import javax.management.openmbean.TabularData; -import javax.management.openmbean.TabularDataSupport; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -72,7 +69,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.RateLimiter; import com.google.common.util.concurrent.Uninterruptibles; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -96,21 +92,18 @@ import org.apache.cassandra.config.Config.PaxosStatePurging; import org.apache.cassandra.config.Converters; import org.apache.cassandra.config.DataStorageSpec; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.QueryHandler; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SizeEstimatesRecorder; -import org.apache.cassandra.db.SnapshotDetailsTabularData; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.compaction.CompactionManager; import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.guardrails.Guardrails; import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.dht.BootStrapper; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.OwnedRanges; @@ -179,7 +172,6 @@ import org.apache.cassandra.service.paxos.PaxosState; import org.apache.cassandra.service.paxos.cleanup.PaxosCleanupLocalCoordinator; import org.apache.cassandra.service.paxos.cleanup.PaxosRepairState; import org.apache.cassandra.service.snapshot.SnapshotManager; -import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamState; @@ -208,7 +200,6 @@ import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.tcm.transformations.Unregister; import org.apache.cassandra.transport.ClientResourceLimits; import org.apache.cassandra.transport.ProtocolVersion; -import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -263,7 +254,6 @@ import static org.apache.cassandra.tcm.membership.NodeState.MOVING; import static org.apache.cassandra.tcm.membership.NodeState.REGISTERED; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; -import static org.apache.cassandra.utils.FBUtilities.now; /** * This abstraction contains the token/identifier of this node @@ -312,8 +302,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private final List preShutdownHooks = new ArrayList<>(); private final List postShutdownHooks = new ArrayList<>(); - public final SnapshotManager snapshotManager = new SnapshotManager(); - public static final StorageService instance = new StorageService(); private final SamplingManager samplingManager = new SamplingManager(); @@ -884,7 +872,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE DiskUsageBroadcaster.instance.startBroadcasting(); HintsService.instance.startDispatch(); BatchlogManager.instance.start(); - startSnapshotManager(); servicesInitialized = true; } @@ -925,12 +912,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return DatabaseDescriptor.getSeeds().contains(getBroadcastAddressAndPort()); } - @VisibleForTesting - public void startSnapshotManager() - { - snapshotManager.start(); - } - public static boolean isReplacingSameAddress() { InetAddressAndPort replaceAddress = DatabaseDescriptor.getReplaceAddress(); @@ -2742,55 +2723,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return status.statusCode; } - /** - * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. - * - * @param tag - * the tag given to the snapshot; may not be null or empty - * @param options - * Map of options (skipFlush is the only supported option for now) - * @param entities - * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... - */ - @Override - public void takeSnapshot(String tag, Map options, String... entities) throws IOException - { - DurationSpec.IntSecondsBound ttl = options.containsKey("ttl") ? new DurationSpec.IntSecondsBound(options.get("ttl")) : null; - if (ttl != null) - { - int minAllowedTtlSecs = CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.getInt(); - if (ttl.toSeconds() < minAllowedTtlSecs) - throw new IllegalArgumentException(String.format("ttl for snapshot must be at least %d seconds", minAllowedTtlSecs)); - } - - boolean skipFlush = Boolean.parseBoolean(options.getOrDefault("skipFlush", "false")); - if (entities != null && entities.length > 0 && entities[0].contains(".")) - { - takeMultipleTableSnapshot(tag, skipFlush, ttl, entities); - } - else - { - takeSnapshot(tag, skipFlush, ttl, entities); - } - } - - /** - * Takes the snapshot of a specific table. A snapshot name must be - * specified. - * - * @param keyspaceName - * the keyspace which holds the specified table - * @param tableName - * the table to snapshot - * @param tag - * the tag given to the snapshot; may not be null or empty - */ - public void takeTableSnapshot(String keyspaceName, String tableName, String tag) - throws IOException - { - takeMultipleTableSnapshot(tag, false, null, keyspaceName + "." + tableName); - } - @Override public void forceKeyspaceCompactionForTokenRange(String keyspaceName, String startToken, String endToken, String... tableNames) throws IOException, ExecutionException, InterruptedException { @@ -2840,296 +2772,126 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void forceCompactionKeysIgnoringGcGrace(String keyspaceName, String tableName, String... partitionKeysIgnoreGcGrace) throws IOException, ExecutionException, InterruptedException { - ColumnFamilyStore cfStore = getValidKeyspace(keyspaceName).getColumnFamilyStore(tableName); + ColumnFamilyStore cfStore = Keyspace.getValidKeyspace(keyspaceName).getColumnFamilyStore(tableName); cfStore.forceCompactionKeysIgnoringGcGrace(partitionKeysIgnoreGcGrace); } + /** + * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * + * @param tag the tag given to the snapshot; may not be null or empty + * @param options Map of options (skipFlush is the only supported option for now) + * @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + * + * @deprecated See CASSANDRA-18111 + */ + @Override + @Deprecated(since = "5.1") + public void takeSnapshot(String tag, Map options, String... entities) throws IOException + { + SnapshotManager.instance.takeSnapshot(tag, options, entities); + } + + /** + * Takes the snapshot of a specific table. A snapshot name must be + * specified. + * + * @param keyspaceName the keyspace which holds the specified table + * @param tableName the table to snapshot + * @param tag the tag given to the snapshot; may not be null or empty + * + * @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907 + */ + @Override + @Deprecated(since = "3.4") + public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException + { + SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), keyspaceName + '.' + tableName); + } + /** * Takes the snapshot for the given keyspaces. A snapshot name must be specified. * * @param tag the tag given to the snapshot; may not be null or empty * @param keyspaceNames the names of the keyspaces to snapshot; empty means "all." + * + * @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907 */ + @Override + @Deprecated(since = "3.4") public void takeSnapshot(String tag, String... keyspaceNames) throws IOException { - takeSnapshot(tag, false, null, keyspaceNames); + SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), keyspaceNames); } /** * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. * - * @param tag - * the tag given to the snapshot; may not be null or empty - * @param tableList - * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 - */ - public void takeMultipleTableSnapshot(String tag, String... tableList) - throws IOException - { - takeMultipleTableSnapshot(tag, false, null, tableList); - } - - /** - * Takes the snapshot for the given keyspaces. A snapshot name must be specified. - * * @param tag the tag given to the snapshot; may not be null or empty - * @param skipFlush Skip blocking flush of memtable - * @param keyspaceNames the names of the keyspaces to snapshot; empty means "all." - */ - private void takeSnapshot(String tag, boolean skipFlush, DurationSpec.IntSecondsBound ttl, String... keyspaceNames) throws IOException - { - if (operationMode() == Mode.JOINING) - throw new IOException("Cannot snapshot until bootstrap completes"); - if (tag == null || tag.equals("")) - throw new IOException("You must supply a snapshot name."); - - Iterable keyspaces; - if (keyspaceNames.length == 0) - { - keyspaces = Keyspace.all(); - } - else - { - ArrayList t = new ArrayList<>(keyspaceNames.length); - for (String keyspaceName : keyspaceNames) - t.add(getValidKeyspace(keyspaceName)); - keyspaces = t; - } - - // Do a check to see if this snapshot exists before we actually snapshot - for (Keyspace keyspace : keyspaces) - if (keyspace.snapshotExists(tag)) - throw new IOException("Snapshot " + tag + " already exists."); - - - RateLimiter snapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); - Instant creationTime = now(); - - for (Keyspace keyspace : keyspaces) - { - keyspace.snapshot(tag, null, skipFlush, ttl, snapshotRateLimiter, creationTime); - } - } - - /** - * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * @param tableList list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 * - * - * @param tag - * the tag given to the snapshot; may not be null or empty - * @param skipFlush - * Skip blocking flush of memtable - * @param tableList - * list of tables from different keyspace in the form of ks1.cf1 ks2.cf2 + * @deprecated See CASSANDRA-10907 */ - private void takeMultipleTableSnapshot(String tag, boolean skipFlush, DurationSpec.IntSecondsBound ttl, String... tableList) - throws IOException + @Override + @Deprecated(since = "3.4") + public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException { - Map> keyspaceColumnfamily = new HashMap>(); - for (String table : tableList) - { - String splittedString[] = StringUtils.split(table, '.'); - if (splittedString.length == 2) - { - String keyspaceName = splittedString[0]; - String tableName = splittedString[1]; - - if (keyspaceName == null) - throw new IOException("You must supply a keyspace name"); - if (operationMode() == Mode.JOINING) - throw new IOException("Cannot snapshot until bootstrap completes"); - - if (tableName == null) - throw new IOException("You must supply a table name"); - if (tag == null || tag.equals("")) - throw new IOException("You must supply a snapshot name."); - - Keyspace keyspace = getValidKeyspace(keyspaceName); - ColumnFamilyStore columnFamilyStore = keyspace.getColumnFamilyStore(tableName); - // As there can be multiple column family from same keyspace check if snapshot exist for that specific - // columnfamily and not for whole keyspace - - if (columnFamilyStore.snapshotExists(tag)) - throw new IOException("Snapshot " + tag + " already exists."); - if (!keyspaceColumnfamily.containsKey(keyspace)) - { - keyspaceColumnfamily.put(keyspace, new ArrayList()); - } - - // Add Keyspace columnfamily to map in order to support atomicity for snapshot process. - // So no snapshot should happen if any one of the above conditions fail for any keyspace or columnfamily - keyspaceColumnfamily.get(keyspace).add(tableName); - - } - else - { - throw new IllegalArgumentException( - "Cannot take a snapshot on secondary index or invalid column family name. You must supply a column family name in the form of keyspace.columnfamily"); - } - } - - RateLimiter snapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); - Instant creationTime = now(); - - for (Entry> entry : keyspaceColumnfamily.entrySet()) - { - for (String table : entry.getValue()) - entry.getKey().snapshot(tag, table, skipFlush, ttl, snapshotRateLimiter, creationTime); - } - } - - private void verifyKeyspaceIsValid(String keyspaceName) - { - if (null != VirtualKeyspaceRegistry.instance.getKeyspaceNullable(keyspaceName)) - throw new IllegalArgumentException("Cannot perform any operations against virtual keyspace " + keyspaceName); - - if (!Schema.instance.getKeyspaces().contains(keyspaceName)) - throw new IllegalArgumentException("Keyspace " + keyspaceName + " does not exist"); - } - - private Keyspace getValidKeyspace(String keyspaceName) - { - verifyKeyspaceIsValid(keyspaceName); - return Keyspace.open(keyspaceName); + SnapshotManager.instance.takeSnapshot(tag, Collections.emptyMap(), tableList); } /** * Remove the snapshot with the given name from the given keyspaces. * If no tag is specified we will remove all snapshots. + * + * @deprecated See CASSANDRA-16860 */ - public void clearSnapshot(String tag, String... keyspaceNames) + @Override + @Deprecated(since = "5.0") + public void clearSnapshot(String tag, String... keyspaceNames) throws IOException { - clearSnapshot(Collections.emptyMap(), tag, keyspaceNames); + SnapshotManager.instance.clearSnapshot(tag, Collections.emptyMap(), keyspaceNames); } + @Override + @Deprecated(since = "5.1") public void clearSnapshot(Map options, String tag, String... keyspaceNames) { - if (tag == null) - tag = ""; - - if (options == null) - options = Collections.emptyMap(); - - Set keyspaces = new HashSet<>(); - for (String dataDir : DatabaseDescriptor.getAllDataFileLocations()) - { - for (String keyspaceDir : new File(dataDir).tryListNames()) - { - // Only add a ks if it has been specified as a param, assuming params were actually provided. - if (keyspaceNames.length > 0 && !Arrays.asList(keyspaceNames).contains(keyspaceDir)) - continue; - keyspaces.add(keyspaceDir); - } - } - - Object olderThan = options.get("older_than"); - Object olderThanTimestamp = options.get("older_than_timestamp"); - - final long clearOlderThanTimestamp; - if (olderThan != null) - { - assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String"; - clearOlderThanTimestamp = Clock.Global.currentTimeMillis() - new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds(); - } - else if (olderThanTimestamp != null) - { - assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String"; - try - { - clearOlderThanTimestamp = Instant.parse((String) olderThanTimestamp).toEpochMilli(); - } - catch (DateTimeParseException ex) - { - throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format."); - } - } - else - clearOlderThanTimestamp = 0L; - - for (String keyspace : keyspaces) - clearKeyspaceSnapshot(keyspace, tag, clearOlderThanTimestamp); - - if (logger.isDebugEnabled()) - logger.debug("Cleared out snapshot directories"); - } - - /** - * Clear snapshots for a given keyspace. - * @param keyspace keyspace to remove snapshots for - * @param tag the user supplied snapshot name. If empty or null, all the snapshots will be cleaned - * @param olderThanTimestamp if a snapshot was created before this timestamp, it will be cleared, - * if its value is 0, this parameter is effectively ignored. - */ - private void clearKeyspaceSnapshot(String keyspace, String tag, long olderThanTimestamp) - { - Set snapshotsToClear = snapshotManager.loadSnapshots(keyspace) - .stream() - .filter(TableSnapshot.shouldClearSnapshot(tag, olderThanTimestamp)) - .collect(Collectors.toSet()); - for (TableSnapshot snapshot : snapshotsToClear) - snapshotManager.clearSnapshot(snapshot); + SnapshotManager.instance.clearSnapshot(tag, options, keyspaceNames); } + @Override + @Deprecated(since = "5.1") public Map getSnapshotDetails(Map options) { - boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false")); - boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false")); - - Map snapshotMap = new HashMap<>(); - - for (TableSnapshot snapshot : snapshotManager.loadSnapshots()) - { - if (skipExpiring && snapshot.isExpiring()) - continue; - if (!includeEphemeral && snapshot.isEphemeral()) - continue; - - TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag()); - if (data == null) - { - data = new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE); - snapshotMap.put(snapshot.getTag(), data); - } - - SnapshotDetailsTabularData.from(snapshot, data); - } - - return snapshotMap; + return SnapshotManager.instance.listSnapshots(options); } - /** @deprecated See CASSANDRA-16789 */ + @Override @Deprecated(since = "4.1") public Map getSnapshotDetails() { - return getSnapshotDetails(ImmutableMap.of()); + return SnapshotManager.instance.listSnapshots(ImmutableMap.of()); } + @Override + @Deprecated(since = "5.1") public long trueSnapshotsSize() { - long total = 0; - for (Keyspace keyspace : Keyspace.all()) - { - if (isLocalSystemKeyspace(keyspace.getName())) - continue; - - for (ColumnFamilyStore cfStore : keyspace.getColumnFamilyStores()) - { - total += cfStore.trueSnapshotsSize(); - } - } - - return total; + return SnapshotManager.instance.getTrueSnapshotSize(); } + @Override + @Deprecated(since = "5.1") public void setSnapshotLinksPerSecond(long throttle) { - logger.info("Setting snapshot throttle to {}", throttle); - DatabaseDescriptor.setSnapshotLinksPerSecond(throttle); + SnapshotManager.instance.setSnapshotLinksPerSecond(throttle); } + @Override + @Deprecated(since = "5.1") public long getSnapshotLinksPerSecond() { - return DatabaseDescriptor.getSnapshotLinksPerSecond(); + return SnapshotManager.instance.getSnapshotLinksPerSecond(); } public void refreshSizeEstimates() throws ExecutionException @@ -3152,7 +2914,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE */ public Iterable getValidColumnFamilies(boolean allowIndexes, boolean autoAddIndexes, String keyspaceName, String... cfNames) { - Keyspace keyspace = getValidKeyspace(keyspaceName); + Keyspace keyspace = Keyspace.getValidKeyspace(keyspaceName); return keyspace.getValidColumnFamilies(allowIndexes, autoAddIndexes, cfNames); } @@ -3944,7 +3706,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE logger.error("Batchlog manager timed out shutting down", t); } - snapshotManager.stop(); + SnapshotManager.instance.close(); HintsService.instance.pauseDispatch(); if (daemon != null) @@ -4038,7 +3800,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } FBUtilities.waitOnFutures(flushes); - SnapshotManager.shutdownAndWait(1L, MINUTES); + SnapshotManager.instance.shutdownAndWait(1L, MINUTES); HintsService.instance.shutdownBlocking(); // Interrupt ongoing compactions and shutdown CM to prevent further compactions. @@ -4167,7 +3929,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void truncate(String keyspace, String table) throws TimeoutException, IOException { - verifyKeyspaceIsValid(keyspace); + Keyspace.verifyKeyspaceIsValid(keyspace); try { @@ -4537,7 +4299,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { if (!isInitialized()) throw new RuntimeException("Not yet initialized, can't load new sstables"); - verifyKeyspaceIsValid(ksName); + Keyspace.verifyKeyspaceIsValid(ksName); ColumnFamilyStore.loadNewSSTables(ksName, cfName); } @@ -5240,10 +5002,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return Math.toIntExact(Guardrails.instance.getPartitionTombstonesWarnThreshold()); } - public void addSnapshot(TableSnapshot snapshot) { - snapshotManager.addSnapshot(snapshot); - } - @Override public boolean getReadThresholdsEnabled() { diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index f23b3b4543..2a9096762a 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -273,7 +273,9 @@ public interface StorageServiceMBean extends NotificationEmitter public void takeTableSnapshot(String keyspaceName, String tableName, String tag) throws IOException; /** - * @deprecated use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. See CASSANDRA-10907 + * Use {@link #takeSnapshot(String tag, Map options, String... entities)} instead. + * + * @deprecated See CASSANDRA-10907 */ @Deprecated(since = "3.4") public void takeMultipleTableSnapshot(String tag, String... tableList) throws IOException; @@ -281,13 +283,12 @@ public interface StorageServiceMBean extends NotificationEmitter /** * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. * - * @param tag - * the tag given to the snapshot; may not be null or empty - * @param options - * Map of options (skipFlush is the only supported option for now) - * @param entities - * list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + * @param tag the tag given to the snapshot; may not be null or empty + * @param options Map of options (skipFlush is the only supported option for now) + * @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public void takeSnapshot(String tag, Map options, String... entities) throws IOException; /** @@ -308,7 +309,9 @@ public interface StorageServiceMBean extends NotificationEmitter * @param options map of options for cleanup operation, consult nodetool's ClearSnapshot * @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared * @param keyspaceNames name of keyspaces to clear snapshots for + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public void clearSnapshot(Map options, String tag, String... keyspaceNames) throws IOException; /** @@ -324,13 +327,17 @@ public interface StorageServiceMBean extends NotificationEmitter * * @param options map of options used for filtering of snapshots * @return A map of snapshotName to all its details in Tabular form. + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public Map getSnapshotDetails(Map options); /** * Get the true size taken by all snapshots across all keyspaces. * @return True size taken by all the snapshots. + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public long trueSnapshotsSize(); /** @@ -338,7 +345,9 @@ public interface StorageServiceMBean extends NotificationEmitter * A setting of zero indicates no throttling * * @param throttle + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public void setSnapshotLinksPerSecond(long throttle); /** @@ -346,7 +355,9 @@ public interface StorageServiceMBean extends NotificationEmitter * A setting of zero indicates no throttling. * * @return snapshot links-per-second throttle + * @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public long getSnapshotLinksPerSecond(); /** diff --git a/src/java/org/apache/cassandra/service/snapshot/AbstractSnapshotTask.java b/src/java/org/apache/cassandra/service/snapshot/AbstractSnapshotTask.java new file mode 100644 index 0000000000..318a0042d5 --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/AbstractSnapshotTask.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.util.concurrent.Callable; + +public abstract class AbstractSnapshotTask implements Callable +{ + public enum SnapshotTaskType + { + SNAPSHOT, + CLEAR, + RELOAD, + LIST, + SIZE + } + + protected final SnapshotOptions options; + + public AbstractSnapshotTask(SnapshotOptions options) + { + this.options = options; + } + + public abstract SnapshotTaskType getTaskType(); +} diff --git a/src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java b/src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java new file mode 100644 index 0000000000..a2e2e828bc --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/ClearSnapshotTask.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.utils.Clock; + +public class ClearSnapshotTask extends AbstractSnapshotTask +{ + private static final Logger logger = LoggerFactory.getLogger(ClearSnapshotTask.class); + + private final SnapshotManager manager; + private final Predicate predicateForToBeCleanedSnapshots; + private final boolean deleteData; + + public ClearSnapshotTask(SnapshotManager manager, + Predicate predicateForToBeCleanedSnapshots, + boolean deleteData) + { + super(null); + this.manager = manager; + this.predicateForToBeCleanedSnapshots = predicateForToBeCleanedSnapshots; + this.deleteData = deleteData; + } + + @Override + public SnapshotTaskType getTaskType() + { + return SnapshotTaskType.CLEAR; + } + + @Override + public Void call() + { + Set toRemove = new HashSet<>(); + + for (TableSnapshot snapshot : new GetSnapshotsTask(manager, predicateForToBeCleanedSnapshots, false).call()) + { + logger.debug("Removing snapshot {}{}", snapshot, deleteData ? ", deleting data" : ""); + + toRemove.add(snapshot); + + if (deleteData) + { + for (File snapshotDir : snapshot.getDirectories()) + { + try + { + removeSnapshotDirectory(snapshotDir); + } + catch (Throwable ex) + { + logger.warn("Unable to remove snapshot directory {}", snapshotDir, ex); + } + } + } + } + + manager.getSnapshots().removeAll(toRemove); + + return null; + } + + /** + * Returns predicate which will pass the test when arguments match. + * + * @param tag name of snapshot + * @param options options for filtering + * @param keyspaceNames names of keyspaces a snapshot is supposed to be from + * @return predicate which will pass the test when arguments match. + */ + static Predicate getPredicateForCleanedSnapshots(String tag, Map options, String... keyspaceNames) + { + if (options == null) + options = Collections.emptyMap(); + + Object olderThan = options.get("older_than"); + Object olderThanTimestamp = options.get("older_than_timestamp"); + + long maxCreatedAt = Clock.Global.currentTimeMillis(); + if (olderThan != null) + { + assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String"; + maxCreatedAt -= new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds(); + } + else if (olderThanTimestamp != null) + { + assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String"; + try + { + maxCreatedAt = Instant.parse((String) olderThanTimestamp).toEpochMilli(); + } + catch (DateTimeParseException ex) + { + throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format."); + } + } + + return getClearSnapshotPredicate(tag, Set.of(keyspaceNames), maxCreatedAt, false); + } + + /** + * Returns a predicate based on which a snapshot will be included for deletion or not. + * + * @param tag name of snapshot to remove + * @param keyspaces keyspaces this snapshot belongs to + * @param olderThanTimestamp clear the snapshot if it is older than given timestamp + * @param includeEphemeral whether to include ephemeral snapshots as well + * @return predicate which filters snapshots on given parameters + */ + static Predicate getClearSnapshotPredicate(String tag, + Set keyspaces, + long olderThanTimestamp, + boolean includeEphemeral) + { + return ts -> + { + // When no tag is supplied, all snapshots must be cleared + boolean clearAll = tag == null || tag.isEmpty(); + if (!clearAll && ts.isEphemeral() && !includeEphemeral) + logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " + + "Ephemeral snapshots are not removable by a user.", + tag, ts.getKeyspaceName()); + boolean passedEphemeralTest = !ts.isEphemeral() || (ts.isEphemeral() && includeEphemeral); + boolean shouldClearTag = clearAll || ts.getTag().equals(tag); + boolean byTimestamp = true; + + if (olderThanTimestamp > 0L) + { + Instant createdAt = ts.getCreatedAt(); + if (createdAt != null) + byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp)); + } + + boolean byKeyspace = (keyspaces.isEmpty() || keyspaces.contains(ts.getKeyspaceName())); + + return passedEphemeralTest && shouldClearTag && byTimestamp && byKeyspace; + }; + } + + private void removeSnapshotDirectory(File snapshotDir) + { + if (snapshotDir.exists()) + { + logger.trace("Removing snapshot directory {}", snapshotDir); + try + { + + snapshotDir.deleteRecursive(DatabaseDescriptor.getSnapshotRateLimiter()); + } + catch (RuntimeException ex) + { + if (!snapshotDir.exists()) + return; // ignore + throw ex; + } + } + } + + @Override + public String toString() + { + return "ClearSnapshotTask{" + + "deleteData=" + deleteData + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/GetSnapshotsTask.java b/src/java/org/apache/cassandra/service/snapshot/GetSnapshotsTask.java new file mode 100644 index 0000000000..6180425d1a --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/GetSnapshotsTask.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.function.Predicate; + +public class GetSnapshotsTask implements Callable> +{ + private final SnapshotManager snapshotManager; + private final Predicate predicate; + private final boolean shouldRemoveIfNotExists; + + public GetSnapshotsTask(SnapshotManager snapshotManager, + Predicate predicate, + boolean shouldRemoveIfNotExists) + { + this.snapshotManager = snapshotManager; + this.predicate = predicate; + this.shouldRemoveIfNotExists = shouldRemoveIfNotExists; + } + + @Override + public List call() + { + if (shouldRemoveIfNotExists) + return getWithRemoval(); + else + return getWithoutRemoval(); + } + + private List getWithRemoval() + { + List notExistingAnymore = new ArrayList<>(); + List snapshots = new ArrayList<>(); + for (TableSnapshot snapshot : snapshotManager.getSnapshots()) + { + if (snapshot.isCompleted() && predicate.test(snapshot)) + { + if (!snapshot.hasManifest()) + notExistingAnymore.add(snapshot); + else + snapshots.add(snapshot); + } + } + + // we do not want to clear snapshots which do not exist periodically + // because that would beat the purpose of caching (we would need to go to the disk + // to see if manifests still exists every time), hence, we will clean on listing, + // we do not need to have cache clean of non-existing snapshots when nobody is looking + if (!notExistingAnymore.isEmpty()) + { + snapshotManager.getSnapshots().removeAll(notExistingAnymore); + } + + return snapshots; + } + + private List getWithoutRemoval() + { + List snapshots = new ArrayList<>(); + for (TableSnapshot snapshot : snapshotManager.getSnapshots()) + { + if (snapshot.isCompleted() && predicate.test(snapshot)) + { + if (snapshot.hasManifest()) + snapshots.add(snapshot); + } + } + + return snapshots; + } + + @Override + public String toString() + { + return "GetSnapshotsTask{" + + "shouldRemoveIfNotExists=" + shouldRemoveIfNotExists + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/ListSnapshotsTask.java b/src/java/org/apache/cassandra/service/snapshot/ListSnapshotsTask.java new file mode 100644 index 0000000000..1351c12d0b --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/ListSnapshotsTask.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.Predicate; +import javax.management.openmbean.TabularData; +import javax.management.openmbean.TabularDataSupport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; + +public class ListSnapshotsTask implements Callable> +{ + private static final Logger logger = LoggerFactory.getLogger(ListSnapshotsTask.class); + + private final SnapshotManager snapshotManager; + private final Predicate predicate; + private final Map options; + private final boolean shouldRemoveIfNotExists; + + public ListSnapshotsTask(SnapshotManager snapshotManager, + Map options, + boolean shouldRemoveIfNotExists) + { + this.snapshotManager = snapshotManager; + this.options = options; + this.predicate = getListingSnapshotsPredicate(options); + this.shouldRemoveIfNotExists = shouldRemoveIfNotExists; + } + + private static Predicate getListingSnapshotsPredicate(Map options) + { + boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false")); + boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false")); + String selectedKeyspace = options != null ? options.get("keyspace") : null; + String selectedTable = options != null ? options.get("table") : null; + String selectedSnapshotName = options != null ? options.get("snapshot") : null; + + return s -> { + if (selectedSnapshotName != null && !s.getTag().equals(selectedSnapshotName)) + return false; + + if (skipExpiring && s.isExpiring()) + return false; + + if (!includeEphemeral && s.isEphemeral()) + return false; + + if (selectedKeyspace != null && !s.getKeyspaceName().equals(selectedKeyspace)) + return false; + + return selectedTable == null || s.getTableName().equals(selectedTable); + }; + } + + @Override + public Map call() + { + List filteredSnapshots; + + try + { + filteredSnapshots = new GetSnapshotsTask(snapshotManager, predicate, shouldRemoveIfNotExists).call(); + } + catch (Exception e) + { + logger.trace("Unable to get snapshots for listing purposes", e); + return Map.of(); + } + + Map snapshotMap = new HashMap<>(); + Set tags = new HashSet<>(); + + for (TableSnapshot t : filteredSnapshots) + tags.add(t.getTag()); + + for (String tag : tags) + snapshotMap.put(tag, new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE)); + + Map> keyspaceTables = new HashMap<>(); + for (TableSnapshot s : filteredSnapshots) + { + keyspaceTables.computeIfAbsent(s.getKeyspaceName(), ignore -> new HashSet<>()); + keyspaceTables.get(s.getKeyspaceName()).add(s.getTableName()); + } + + Map> cfsFiles = new HashMap<>(); + + for (Map.Entry> entry : keyspaceTables.entrySet()) + { + for (String table : entry.getValue()) + { + try + { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(entry.getKey(), table); + if (cfs == null) + continue; + + cfsFiles.put(cfs.getKeyspaceName() + '.' + cfs.name, cfs.getFilesOfCfs()); + } + catch (Throwable t) + { + logger.debug("Unable to get all files of live SSTables for {}.{}", entry.getKey(), entry.getValue()); + } + } + } + + for (TableSnapshot snapshot : filteredSnapshots) + { + TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag()); + SnapshotDetailsTabularData.from(snapshot, data, cfsFiles.get(snapshot.getKeyspaceTable())); + } + + return snapshotMap; + } + + @Override + public String toString() + { + return "ListSnapshotsTask{" + + "options=" + options + + '}'; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotDetailsTabularData.java similarity index 65% rename from src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java rename to src/java/org/apache/cassandra/service/snapshot/SnapshotDetailsTabularData.java index c5debe43f4..23c84e5056 100644 --- a/src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotDetailsTabularData.java @@ -15,33 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.db; +package org.apache.cassandra.service.snapshot; +import java.util.Set; import javax.management.openmbean.*; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.service.snapshot.TableSnapshot; public class SnapshotDetailsTabularData { private static final String[] ITEM_NAMES = new String[]{"Snapshot name", - "Keyspace name", - "Column family name", - "True size", - "Size on disk", - "Creation time", - "Expiration time", - "Ephemeral"}; + "Keyspace name", + "Column family name", + "True size", + "Size on disk", + "Creation time", + "Expiration time", + "Ephemeral"}; private static final String[] ITEM_DESCS = new String[]{"snapshot_name", - "keyspace_name", - "columnfamily_name", - "TrueDiskSpaceUsed", - "TotalDiskSpaceUsed", - "created_at", - "expires_at", - "ephemeral"}; + "keyspace_name", + "columnfamily_name", + "TrueDiskSpaceUsed", + "TotalDiskSpaceUsed", + "created_at", + "expires_at", + "ephemeral"}; private static final String TYPE_NAME = "SnapshotDetails"; @@ -70,17 +70,18 @@ public class SnapshotDetailsTabularData } - public static void from(TableSnapshot details, TabularDataSupport result) + public static void from(TableSnapshot details, TabularDataSupport result, Set files) { try { final String totalSize = FileUtils.stringifyFileSize(details.computeSizeOnDiskBytes()); + long trueSizeBytes = details.computeTrueSizeBytes(files); final String liveSize = FileUtils.stringifyFileSize(details.computeTrueSizeBytes()); String createdAt = safeToString(details.getCreatedAt()); String expiresAt = safeToString(details.getExpiresAt()); String ephemeral = Boolean.toString(details.isEphemeral()); result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES, - new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral })); + new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral })); } catch (OpenDataException e) { @@ -92,4 +93,4 @@ public class SnapshotDetailsTabularData { return object == null ? null : object.toString(); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotException.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotException.java new file mode 100644 index 0000000000..39cdf7ef77 --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotException.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +public class SnapshotException extends RuntimeException +{ + public SnapshotException(String message) + { + super(message); + } + + public SnapshotException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java index be1d6621bc..0e0ecc11c0 100644 --- a/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -153,7 +154,7 @@ public class SnapshotLoader String tag = snapshotDirMatcher.group("tag"); String snapshotId = buildSnapshotId(keyspaceName, tableName, tableId, tag); TableSnapshot.Builder builder = snapshots.computeIfAbsent(snapshotId, k -> new TableSnapshot.Builder(keyspaceName, tableName, tableId, tag)); - builder.addSnapshotDir(new File(snapshotDir)); + builder.addSnapshotDir(new File(snapshotDir).toAbsolute()); } } @@ -184,7 +185,11 @@ public class SnapshotLoader } } - return snapshots.values().stream().map(TableSnapshot.Builder::build).collect(Collectors.toSet()); + Set tableSnapshots = new HashSet<>(); + for (TableSnapshot.Builder snapshotBuilder : snapshots.values()) + tableSnapshots.add(snapshotBuilder.build()); + + return tableSnapshots; } public Set loadSnapshots() diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java index 3925f3f9dc..bea0c04669 100644 --- a/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManager.java @@ -17,81 +17,213 @@ */ package org.apache.cassandra.service.snapshot; - -import java.util.Collection; -import java.util.PriorityQueue; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; +import javax.management.openmbean.TabularData; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.Directories; - -import java.util.concurrent.TimeoutException; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; - -import org.apache.cassandra.io.util.File; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.notifications.INotification; +import org.apache.cassandra.notifications.INotificationConsumer; +import org.apache.cassandra.notifications.TableDroppedNotification; +import org.apache.cassandra.notifications.TablePreScrubNotification; +import org.apache.cassandra.notifications.TruncationNotification; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.ExecutorUtils; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MBeanWrapper; -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.toList; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; -import static org.apache.cassandra.utils.FBUtilities.now; - -public class SnapshotManager { - - private static final ScheduledExecutorPlus executor = executorFactory().scheduled(false, "SnapshotCleanup"); +import static org.apache.cassandra.service.snapshot.ClearSnapshotTask.getClearSnapshotPredicate; +import static org.apache.cassandra.service.snapshot.ClearSnapshotTask.getPredicateForCleanedSnapshots; +public class SnapshotManager implements SnapshotManagerMBean, INotificationConsumer, AutoCloseable +{ private static final Logger logger = LoggerFactory.getLogger(SnapshotManager.class); + private ScheduledExecutorPlus snapshotCleanupExecutor; + + public static final SnapshotManager instance = new SnapshotManager(); + private final long initialDelaySeconds; private final long cleanupPeriodSeconds; - private final SnapshotLoader snapshotLoader; - @VisibleForTesting - protected volatile ScheduledFuture cleanupTaskFuture; + private volatile ScheduledFuture cleanupTaskFuture; + + private final String[] dataDirs; + + private volatile boolean started = false; /** - * Expiring snapshots ordered by expiration date, to allow only iterating over snapshots - * that need to be removed on {@link this#clearExpiredSnapshots()} + * We read / list snapshots way more often than write / create them so COW is ideal to use here. + * This enables us to not submit listing tasks or tasks computing snapshot sizes to any executor's queue as they + * can be just run concurrently which gives way better throughput in case + * of excessive listing from clients (dashboards and similar) where snapshot metrics are gathered. */ - private final PriorityQueue expiringSnapshots = new PriorityQueue<>(comparing(TableSnapshot::getExpiresAt)); + private final List snapshots = new CopyOnWriteArrayList<>(); - public SnapshotManager() + private SnapshotManager() { this(CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(), - CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt()); + CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.getInt(), + DatabaseDescriptor.getAllDataFileLocations()); } @VisibleForTesting - protected SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds) + SnapshotManager(long initialDelaySeconds, long cleanupPeriodSeconds, String[] dataDirs) { this.initialDelaySeconds = initialDelaySeconds; this.cleanupPeriodSeconds = cleanupPeriodSeconds; - snapshotLoader = new SnapshotLoader(DatabaseDescriptor.getAllDataFileLocations()); + this.dataDirs = dataDirs; + this.snapshotCleanupExecutor = createSnapshotCleanupExecutor(); } - public Collection getExpiringSnapshots() + public void registerMBean() { - return expiringSnapshots; + MBeanWrapper.instance.registerMBean(this, MBEAN_NAME); } - public synchronized void start() + public void unregisterMBean() { - addSnapshots(loadSnapshots()); - resumeSnapshotCleanup(); + MBeanWrapper.instance.unregisterMBean(MBEAN_NAME); } - public synchronized void stop() throws InterruptedException, TimeoutException + public synchronized SnapshotManager start(boolean runPeriodicSnapshotCleaner) + { + if (started) + return this; + + if (snapshotCleanupExecutor == null) + snapshotCleanupExecutor = createSnapshotCleanupExecutor(); + + executeTask(new ReloadSnapshotsTask(dataDirs)); + + if (runPeriodicSnapshotCleaner) + resumeSnapshotCleanup(); + + started = true; + return this; + } + + @Override + public synchronized void close() + { + if (!started) + return; + + pauseSnapshotCleanup(); + + shutdownAndWait(1, TimeUnit.MINUTES); + snapshots.clear(); + + started = false; + } + + public synchronized void shutdownAndWait(long timeout, TimeUnit unit) + { + try + { + ExecutorUtils.shutdownNowAndWait(timeout, unit, snapshotCleanupExecutor); + } + catch (InterruptedException | TimeoutException ex) + { + throw new RuntimeException(ex); + } + finally + { + snapshotCleanupExecutor = null; + } + } + + public synchronized void restart(boolean runPeriodicSnapshotCleaner) + { + if (!started) + return; + + logger.debug("Restarting SnapshotManager"); + close(); + start(runPeriodicSnapshotCleaner); + logger.debug("SnapshotManager restarted"); + } + + + public synchronized void restart() + { + restart(true); + } + + private static class ReloadSnapshotsTask extends AbstractSnapshotTask> + { + private final String[] dataDirs; + + public ReloadSnapshotsTask(String[] dataDirs) + { + super(null); + this.dataDirs = dataDirs; + } + + @Override + public Set call() + { + Set tableSnapshots = new SnapshotLoader(dataDirs).loadSnapshots(); + new ClearSnapshotTask(SnapshotManager.instance, snapshot -> true, false).call(); + for (TableSnapshot snapshot : tableSnapshots) + SnapshotManager.instance.addSnapshot(snapshot); + + return tableSnapshots; + } + + @Override + public SnapshotTaskType getTaskType() + { + return SnapshotTaskType.RELOAD; + } + } + + void addSnapshot(TableSnapshot snapshot) + { + logger.debug("Adding snapshot {}", snapshot); + snapshots.add(snapshot); + } + + List getSnapshots() + { + return snapshots; + } + + public void resumeSnapshotCleanup() + { + if (cleanupTaskFuture == null) + { + logger.info("Scheduling expired snapshots cleanup with initialDelaySeconds={} and cleanupPeriodSeconds={}", + initialDelaySeconds, cleanupPeriodSeconds); + + cleanupTaskFuture = snapshotCleanupExecutor.scheduleWithFixedDelay(SnapshotManager.instance::clearExpiredSnapshots, + initialDelaySeconds, + cleanupPeriodSeconds, + SECONDS); + } + } + + private void pauseSnapshotCleanup() { - expiringSnapshots.clear(); if (cleanupTaskFuture != null) { cleanupTaskFuture.cancel(false); @@ -99,74 +231,350 @@ public class SnapshotManager { } } - public synchronized void addSnapshot(TableSnapshot snapshot) + /** + * Deletes snapshot and removes it from manager. + * + * @param snapshot snapshot to clear + */ + void clearSnapshot(TableSnapshot snapshot) { - // We currently only care about expiring snapshots - if (snapshot.isExpiring()) + executeTask(new ClearSnapshotTask(this, s -> s.equals(snapshot), true)); + } + + /** + * Returns list of snapshots of given keyspace + * + * @param keyspace keyspace of a snapshot + * @return list of snapshots of given keyspace. + */ + public List getSnapshots(String keyspace) + { + return getSnapshots(snapshot -> snapshot.getKeyspaceName().equals(keyspace)); + } + + /** + * Return snapshots based on given parameters. + * + * @param skipExpiring if expiring snapshots should be skipped + * @param includeEphemeral if ephemeral snapshots should be included + * @return snapshots based on given parameters + */ + public List getSnapshots(boolean skipExpiring, boolean includeEphemeral) + { + return getSnapshots(s -> (!skipExpiring || !s.isExpiring()) && (includeEphemeral || !s.isEphemeral())); + } + + /** + * Returns all snapshots passing the given predicate. + * + * @param predicate predicate to filter all snapshots of + * @return list of snapshots passing the predicate + */ + public List getSnapshots(Predicate predicate) + { + return new GetSnapshotsTask(this, predicate, true).call(); + } + + /** + * Returns a snapshot or empty optional based on the given parameters. + * + * @param keyspace keyspace of a snapshot + * @param table table of a snapshot + * @param tag name of a snapshot + * @return empty optional if there is not such snapshot, non-empty otherwise + */ + public Optional getSnapshot(String keyspace, String table, String tag) + { + List foundSnapshots = new GetSnapshotsTask(this, + snapshot -> snapshot.getKeyspaceName().equals(keyspace) && + snapshot.getTableName().equals(table) && + snapshot.getTag().equals(tag) || (tag != null && tag.isEmpty()), + true).call(); + + if (foundSnapshots.isEmpty()) + return Optional.empty(); + else + return Optional.of(foundSnapshots.get(0)); + } + + /** + * Checks whether a snapshot for given keyspace and table exists of a given name exists. + * + * @param keyspace keyspace to get a snapshot of + * @param table table to get a snapshot of + * @param tag name of a snapshot + * @return true of a snapshot of given properties exist, false otherwise + */ + public boolean exists(String keyspace, String table, String tag) + { + return getSnapshot(keyspace, table, tag).isPresent(); + } + + /** + * Checks whether a snapshot which satisfies given predicate exists. + * + * @param predicate predicate to check the existence of a snapshot + * @return true if a snapshot which satisfies a predicate exists, false otherwise + */ + public boolean exists(Predicate predicate) + { + return !getSnapshots(predicate).isEmpty(); + } + + /** + * Clear snapshots of given tag from given keyspace. Does not remove ephemeral snapshots. + *

+ * + * @param tag snapshot name + * @param keyspace keyspace to clear all snapshots of a given tag of + */ + public void clearSnapshots(String tag, String keyspace) + { + clearSnapshots(tag, Set.of(keyspace), Clock.Global.currentTimeMillis()); + } + + /** + * Removes a snapshot. + *

+ * + * @param keyspace keyspace of a snapshot to remove + * @param table table of a snapshot to remove + * @param tag name of a snapshot to remove. + */ + public void clearSnapshot(String keyspace, String table, String tag) + { + executeTask(new ClearSnapshotTask(this, + snapshot -> snapshot.getKeyspaceName().equals(keyspace) + && snapshot.getTableName().equals(table) + && snapshot.getTag().equals(tag), + true)); + } + + /** + * Removes all snapshots for given keyspace and table. + * + * @param keyspace keyspace to remove snapshots for + * @param table table in a given keyspace to remove snapshots for + */ + public void clearAllSnapshots(String keyspace, String table) + { + executeTask(new ClearSnapshotTask(this, + snapshot -> snapshot.getKeyspaceName().equals(keyspace) + && snapshot.getTableName().equals(table), + true)); + } + + /** + * Clears all snapshots, expiring and ephemeral as well. + */ + public void clearAllSnapshots() + { + executeTask(new ClearSnapshotTask(this, snapshot -> true, true)); + } + + /** + * Clear snapshots based on a given predicate + * + * @param predicate predicate to filter snapshots on + */ + public void clearSnapshot(Predicate predicate) + { + executeTask(new ClearSnapshotTask(this, predicate, true)); + } + + /** + * Clears all ephemeral snapshots in a node. + */ + public void clearEphemeralSnapshots() + { + executeTask(new ClearSnapshotTask(this, TableSnapshot::isEphemeral, true)); + } + + /** + * Clears all expired snapshots in a node. + */ + public void clearExpiredSnapshots() + { + Instant now = FBUtilities.now(); + executeTask(new ClearSnapshotTask(this, s -> s.isExpired(now), true)); + } + + /** + * Clear snapshots of given tag from given keyspaces. + *

+ * If tag is not present / is empty, all snapshots are considered to be cleared. + * If keyspaces are empty, all snapshots of given tag and older than maxCreatedAt are removed. + * + * @param tag optional tag of snapshot to clear + * @param keyspaces keyspaces to remove snapshots for + * @param maxCreatedAt clear all such snapshots which were created before this timestamp + */ + private void clearSnapshots(String tag, Set keyspaces, long maxCreatedAt) + { + executeTask(new ClearSnapshotTask(this, getClearSnapshotPredicate(tag, keyspaces, maxCreatedAt, false), true)); + } + + public List takeSnapshot(SnapshotOptions options) + { + return executeTask(new TakeSnapshotTask(this, options)); + } + + // Super methods + + @Override + public void takeSnapshot(String tag, String... entities) + { + takeSnapshot(SnapshotOptions.userSnapshot(tag, Map.of(), entities)); + } + + @Override + public void takeSnapshot(String tag, Map optMap, String... entities) throws IOException + { + try { - logger.debug("Adding expiring snapshot {}", snapshot); - expiringSnapshots.add(snapshot); + takeSnapshot(SnapshotOptions.userSnapshot(tag, optMap, entities)); + } + catch (SnapshotException ex) + { + // to be compatible with deprecated methods in StorageService + throw new IOException(ex); } } - public synchronized Set loadSnapshots(String keyspace) + @Override + public void clearSnapshot(String tag, Map options, String... keyspaceNames) { - return snapshotLoader.loadSnapshots(keyspace); + executeTask(new ClearSnapshotTask(this, getPredicateForCleanedSnapshots(tag, options, keyspaceNames), true)); } - public synchronized Set loadSnapshots() + @Override + public Map listSnapshots(Map options) { - return snapshotLoader.loadSnapshots(); + return new ListSnapshotsTask(this, options, true).call(); } - @VisibleForTesting - protected synchronized void addSnapshots(Collection snapshots) + @Override + public long getTrueSnapshotSize() { - logger.debug("Adding snapshots: {}.", Joiner.on(", ").join(snapshots.stream().map(TableSnapshot::getId).collect(toList()))); - snapshots.forEach(this::addSnapshot); + return new TrueSnapshotSizeTask(this, s -> true).call(); } - // TODO: Support pausing snapshot cleanup - @VisibleForTesting - synchronized void resumeSnapshotCleanup() + @Override + public long getTrueSnapshotsSize(String keyspace) { - if (cleanupTaskFuture == null) + return new TrueSnapshotSizeTask(this, s -> s.getKeyspaceName().equals(keyspace)).call(); + } + + @Override + public long getTrueSnapshotsSize(String keyspace, String table) + { + return new TrueSnapshotSizeTask(this, s -> s.getKeyspaceName().equals(keyspace) && s.getTableName().equals(table)).call(); + } + + @Override + public void setSnapshotLinksPerSecond(long throttle) + { + logger.info("Setting snapshot throttle to {}", throttle); + DatabaseDescriptor.setSnapshotLinksPerSecond(throttle); + } + + @Override + public long getSnapshotLinksPerSecond() + { + return DatabaseDescriptor.getSnapshotLinksPerSecond(); + } + + @Override + public void handleNotification(INotification notification, Object sender) + { + if (notification instanceof TruncationNotification) { - logger.info("Scheduling expired snapshot cleanup with initialDelaySeconds={} and cleanupPeriodSeconds={}", - initialDelaySeconds, cleanupPeriodSeconds); - cleanupTaskFuture = executor.scheduleWithFixedDelay(this::clearExpiredSnapshots, initialDelaySeconds, - cleanupPeriodSeconds, TimeUnit.SECONDS); + TruncationNotification truncationNotification = (TruncationNotification) notification; + ColumnFamilyStore cfs = truncationNotification.cfs; + if (!truncationNotification.disableSnapshot && cfs.isAutoSnapshotEnabled()) + { + SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.TRUNCATE, cfs.getKeyspaceTableName()) + .ttl(truncationNotification.ttl).build(); + takeSnapshot(opts); + } + } + else if (notification instanceof TableDroppedNotification) + { + TableDroppedNotification tableDroppedNotification = (TableDroppedNotification) notification; + ColumnFamilyStore cfs = tableDroppedNotification.cfs; + if (cfs.isAutoSnapshotEnabled()) + { + SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.DROP, cfs.getKeyspaceTableName()) + .cfs(cfs).ttl(tableDroppedNotification.ttl).build(); + takeSnapshot(opts); + } + } + else if (notification instanceof TablePreScrubNotification) + { + ColumnFamilyStore cfs = ((TablePreScrubNotification) notification).cfs; + SnapshotOptions opts = SnapshotOptions.systemSnapshot(cfs.name, SnapshotType.PRE_SCRUB, cfs.getKeyspaceTableName()).build(); + takeSnapshot(opts); } } @VisibleForTesting - protected synchronized void clearExpiredSnapshots() + List executeTask(TakeSnapshotTask task) { - TableSnapshot expiredSnapshot; - while ((expiredSnapshot = expiringSnapshots.peek()) != null) + try { - if (!expiredSnapshot.isExpired(now())) - break; // the earliest expiring snapshot is not expired yet, so there is no more expired snapshots to remove + prePopulateSnapshots(task); + return task.call(); + } + catch (Throwable t) + { + throw new SnapshotException(String.format("Exception occured while executing %s: %s", task.toString(), t.getMessage()), t); + } + } - logger.debug("Removing expired snapshot {}.", expiredSnapshot); - clearSnapshot(expiredSnapshot); + @VisibleForTesting + T executeTask(AbstractSnapshotTask task) + { + try + { + return task.call(); + } + catch (Throwable t) + { + throw new SnapshotException(String.format("Exception occured while executing %s", task.toString()), t); } } /** - * Deletes snapshot and remove it from manager + * Add table snapshots to snaphots cow list in advance in order to be sure that the snapshots to be created + * are not existing already. "executeTask" method can be invoked by multiple threads, and they hit + * this synchronized method which populates snapshots so next thread will fail to advance (exception is thrown) + * if table snapshots of some other tasks are already present in snapshots cow list. + * + * Added snapshots to snapshot list are at this point "incomplete", they will not appear in listing output + * until they are indeed taken and exist on disk, otherwise we would see "phantom" snapshots in listings, + * they would be present but in fact they are just going through the process of being taken. + * + * @param task task to process */ - public synchronized void clearSnapshot(TableSnapshot snapshot) + private synchronized void prePopulateSnapshots(TakeSnapshotTask task) { - for (File snapshotDir : snapshot.getDirectories()) - Directories.removeSnapshotDirectory(DatabaseDescriptor.getSnapshotRateLimiter(), snapshotDir); + Map snapshotsToCreate = task.getSnapshotsToCreate(); + for (Map.Entry toCreateEntry : snapshotsToCreate.entrySet()) + { + if (snapshots.contains(toCreateEntry.getValue())) + { + throw new RuntimeException(format("Snapshot %s for %s.%s already exists.", + toCreateEntry.getValue().getTag(), + toCreateEntry.getValue().getKeyspaceName(), + toCreateEntry.getValue().getTableName())); + } + } - expiringSnapshots.remove(snapshot); + snapshots.addAll(snapshotsToCreate.values()); } - @VisibleForTesting - public static void shutdownAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException + private static ScheduledExecutorPlus createSnapshotCleanupExecutor() { - ExecutorUtils.shutdownNowAndWait(timeout, unit, executor); + return executorFactory().scheduled(false, "SnapshotCleanup"); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManagerMBean.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManagerMBean.java new file mode 100644 index 0000000000..2d69a7ca97 --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManagerMBean.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.io.IOException; +import java.util.Map; +import javax.management.openmbean.TabularData; + +public interface SnapshotManagerMBean +{ + String MBEAN_NAME = "org.apache.cassandra.service.snapshot:type=SnapshotManager"; + + /** + * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * + * @param tag the tag given to the snapshot; may not be null or empty + * @param options Map of options (skipFlush is the only supported option for now) + * @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + */ + void takeSnapshot(String tag, Map options, String... entities) throws IOException; + + /** + * Takes the snapshot of a multiple column family from different keyspaces. A snapshot name must be specified. + * + * @param tag the tag given to the snapshot; may not be null or empty + * @param entities list of keyspaces / tables in the form of empty | ks1 ks2 ... | ks1.cf1,ks2.cf2,... + */ + void takeSnapshot(String tag, String... entities) throws IOException; + + /** + * Remove the snapshot with the given name from the given keyspaces. + * If no tag is specified we will remove all snapshots. + * + * @param tag name of snapshot to clear, if null or empty string, + * all snapshots of given keyspace will be cleared + * @param options map of options for cleanup operation, consult nodetool's ClearSnapshot + * @param keyspaceNames name of keyspaces to clear snapshots for + */ + void clearSnapshot(String tag, Map options, String... keyspaceNames) throws IOException; + + /** + * Get the details of all the snapshots + * + * @param options map of options used for filtering of snapshots + * @return A map of snapshotName to all its details in Tabular form. + */ + Map listSnapshots(Map options); + + /** + * Get the true size taken by all snapshots across all keyspaces. + * + * @return True size taken by all the snapshots. + */ + long getTrueSnapshotSize(); + + /** + * Get the true size take by all snapshots in given keyspace. + * + * @param keyspace keyspace to get true size of all snapshots of + * @return true size of all snapshots in given keyspace + */ + long getTrueSnapshotsSize(String keyspace); + + /** + * Get the true size take by all snapshots in given keyspace and table. + * + * @param keyspace keyspace to get true size of all snapshots of + * @param table table in a keyspace to get true size of all snapshots of + * @return true size of all snapshots in given keyspace and table + */ + long getTrueSnapshotsSize(String keyspace, String table); + + /** + * Set the current hardlink-per-second throttle for snapshots + * A setting of zero indicates no throttling + * + * @param throttle hard-links-per-second + */ + void setSnapshotLinksPerSecond(long throttle); + + /** + * Get the current hardlink-per-second throttle for snapshots + * A setting of zero indicates no throttling. + * + * @return snapshot links-per-second throttle + */ + long getSnapshotLinksPerSecond(); + + /** + * Restarting means that snapshots will be reloaded from disk. + */ + void restart(); +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java index 8ee737cf82..6f23eba84d 100644 --- a/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java @@ -63,7 +63,7 @@ public class SnapshotManifest { this.files = files; this.createdAt = creationTime; - this.expiresAt = ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds()); + this.expiresAt = computeExpiration(ttl, creationTime); this.ephemeral = ephemeral; } @@ -97,6 +97,11 @@ public class SnapshotManifest return JsonUtils.deserializeFromJsonFile(SnapshotManifest.class, file); } + public static Instant computeExpiration(DurationSpec.IntSecondsBound ttl, Instant createdAt) + { + return ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds()); + } + @Override public boolean equals(Object o) { diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotOptions.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotOptions.java new file mode 100644 index 0000000000..409288f8bf --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotOptions.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.function.Predicate; + +import com.google.common.util.concurrent.RateLimiter; +import org.apache.commons.lang3.StringUtils; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.DurationSpec; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.io.sstable.format.SSTableReader; + +import static java.lang.String.format; + +public class SnapshotOptions +{ + public static final String SKIP_FLUSH = "skipFlush"; + public static final String TTL = "ttl"; + public final SnapshotType type; + public final String tag; + public final DurationSpec.IntSecondsBound ttl; + public final Instant creationTime; + public final boolean skipFlush; + public final boolean ephemeral; + public final String[] entities; + public final RateLimiter rateLimiter; + public final Predicate sstableFilter; + public final ColumnFamilyStore cfs; + + private SnapshotOptions(SnapshotType type, + String tag, + DurationSpec.IntSecondsBound ttl, + Instant creationTime, + boolean skipFlush, + boolean ephemeral, + String[] entities, + RateLimiter rateLimiter, + Predicate sstableFilter, + ColumnFamilyStore cfs) + { + this.type = type; + this.tag = tag; + this.ttl = ttl; + this.creationTime = creationTime; + this.skipFlush = skipFlush; + this.ephemeral = ephemeral; + this.entities = entities; + this.rateLimiter = rateLimiter; + this.sstableFilter = sstableFilter; + this.cfs = cfs; + } + + public static Builder systemSnapshot(String tag, SnapshotType type, String... entities) + { + return new Builder(tag, type, ssTableReader -> true, entities); + } + + public static Builder systemSnapshot(String tag, SnapshotType type, Predicate sstableFilter, String... entities) + { + return new Builder(tag, type, sstableFilter, entities); + } + + public static SnapshotOptions userSnapshot(String tag, String... entities) + { + return userSnapshot(tag, Collections.emptyMap(), entities); + } + + public static SnapshotOptions userSnapshot(String tag, Map options, String... entities) + { + Builder builder = new Builder(tag, SnapshotType.USER, ssTableReader -> true, entities).ttl(options.get(TTL)); + if (Boolean.parseBoolean(options.getOrDefault(SKIP_FLUSH, Boolean.FALSE.toString()))) + builder.skipFlush(); + return builder.build(); + } + + public String getSnapshotName(Instant creationTime) + { + // Diagnostic snapshots have very specific naming convention hence we are keeping it. + // Repair snapshots rely on snapshots having name of their repair session ids + if (type == SnapshotType.USER || type == SnapshotType.DIAGNOSTICS || type == SnapshotType.REPAIR) + return tag; + // System snapshots have the creation timestamp on the name + String snapshotName = format("%d-%s", creationTime.toEpochMilli(), type.label); + if (StringUtils.isNotBlank(tag)) + snapshotName = snapshotName + '-' + tag; + return snapshotName; + } + + public static class Builder + { + private final String tag; + private final String[] entities; + private DurationSpec.IntSecondsBound ttl; + private Instant creationTime; + private boolean skipFlush = false; + private boolean ephemeral = false; + private ColumnFamilyStore cfs; + private final Predicate sstableFilter; + private final SnapshotType type; + private RateLimiter rateLimiter; + + public Builder(String tag, SnapshotType type, Predicate sstableFilter, String... entities) + { + this.tag = tag; + this.type = type; + this.entities = entities; + this.sstableFilter = sstableFilter; + } + + public Builder ttl(String ttl) + { + if (ttl != null) + this.ttl = new DurationSpec.IntSecondsBound(ttl); + + return this; + } + + public Builder ttl(DurationSpec.IntSecondsBound ttl) + { + this.ttl = ttl; + return this; + } + + public Builder creationTime(String creationTime) + { + if (creationTime != null) + { + try + { + return creationTime(Long.parseLong(creationTime)); + } + catch (Exception ex) + { + throw new RuntimeException("Unable to parse creation time from " + creationTime); + } + } + + return this; + } + + public Builder creationTime(Instant creationTime) + { + this.creationTime = creationTime; + return this; + } + + public Builder creationTime(long creationTime) + { + return creationTime(Instant.ofEpochMilli(creationTime)); + } + + public Builder skipFlush() + { + skipFlush = true; + return this; + } + + public Builder ephemeral() + { + ephemeral = true; + return this; + } + + public Builder cfs(ColumnFamilyStore cfs) + { + this.cfs = cfs; + return this; + } + + public Builder rateLimiter(RateLimiter rateLimiter) + { + this.rateLimiter = rateLimiter; + return this; + } + + public SnapshotOptions build() + { + if (tag == null || tag.isEmpty()) + throw new RuntimeException("You must supply a snapshot name."); + + if (ttl != null) + { + int minAllowedTtlSecs = CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.getInt(); + if (ttl.toSeconds() < minAllowedTtlSecs) + throw new IllegalArgumentException(format("ttl for snapshot must be at least %d seconds", minAllowedTtlSecs)); + } + + if (ephemeral && ttl != null) + throw new IllegalStateException(format("can not take ephemeral snapshot (%s) while ttl is specified too", tag)); + + if (rateLimiter == null) + rateLimiter = DatabaseDescriptor.getSnapshotRateLimiter(); + + return new SnapshotOptions(type, tag, ttl, creationTime, skipFlush, ephemeral, entities, rateLimiter, + sstableFilter, cfs); + } + } + + @Override + public String toString() + { + return "CreateSnapshotOptions{" + + "type=" + type + + ", tag='" + tag + '\'' + + ", ttl=" + ttl + + ", creationTime=" + creationTime + + ", skipFlush=" + skipFlush + + ", ephemeral=" + ephemeral + + ", entities=" + Arrays.toString(entities) + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/SnapshotType.java b/src/java/org/apache/cassandra/service/snapshot/SnapshotType.java new file mode 100644 index 0000000000..67beb091c1 --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/SnapshotType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +public enum SnapshotType +{ + USER("user"), + + TRUNCATE("truncated"), + DROP("dropped"), + PRE_SCRUB("pre-scrub"), + COMPACT("compact"), + UPGRADE("upgrade"), + DIAGNOSTICS("diagnostics"), + + REPAIR("repair"), + + MISC("misc"); + + public final String label; + + SnapshotType(String label) + { + this.label = label; + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java index d10092a41a..94bcd07d9a 100644 --- a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java +++ b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java @@ -21,21 +21,33 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.function.Predicate; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableId; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.utils.DirectorySizeCalculator; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.concurrent.Refs; public class TableSnapshot { @@ -43,6 +55,7 @@ public class TableSnapshot private final String keyspaceName; private final String tableName; + private final String keyspaceTable; private final UUID tableId; private final String tag; private final boolean ephemeral; @@ -52,25 +65,51 @@ public class TableSnapshot private final Set snapshotDirs; + private volatile long sizeOnDisk = 0; + + private volatile long manifestsSize; + private volatile long schemasSize; + + private volatile boolean inProgress = false; + public TableSnapshot(String keyspaceName, String tableName, UUID tableId, String tag, Instant createdAt, Instant expiresAt, Set snapshotDirs, boolean ephemeral) { this.keyspaceName = keyspaceName; this.tableName = tableName; + this.keyspaceTable = keyspaceName + '.' + tableName; this.tableId = tableId; this.tag = tag; this.createdAt = createdAt; this.expiresAt = expiresAt; this.snapshotDirs = snapshotDirs; this.ephemeral = ephemeral; + + manifestsSize = getManifestsSize(); + schemasSize = getSchemasSize(); + } + + public boolean isCompleted() + { + return !inProgress; + } + + public void incomplete() + { + inProgress = true; + } + + public void complete() + { + inProgress = false; } /** * Unique identifier of a snapshot. Used * only to deduplicate snapshots internally, * not exposed externally. - * + *

* Format: "$ks:$table_name:$table_id:$tag" */ public String getId() @@ -88,6 +127,11 @@ public class TableSnapshot return tableName; } + public String getKeyspaceTable() + { + return keyspaceTable; + } + public String getTag() { return tag; @@ -95,13 +139,22 @@ public class TableSnapshot public Instant getCreatedAt() { + if (createdAt == null) { - long minCreation = snapshotDirs.stream().mapToLong(File::lastModified).min().orElse(0); - if (minCreation != 0) + long minCreation = 0; + for (File snapshotDir : snapshotDirs) { - return Instant.ofEpochMilli(minCreation); + long lastModified = snapshotDir.lastModified(); + if (lastModified == 0) + continue; + + if (minCreation == 0 || minCreation > lastModified) + minCreation = lastModified; } + + if (minCreation != 0) + return Instant.ofEpochMilli(minCreation); } return createdAt; } @@ -123,7 +176,11 @@ public class TableSnapshot public boolean exists() { - return snapshotDirs.stream().anyMatch(File::exists); + for (File snapshotDir : snapshotDirs) + if (snapshotDir.exists()) + return true; + + return false; } public boolean isEphemeral() @@ -138,26 +195,106 @@ public class TableSnapshot public long computeSizeOnDiskBytes() { - return snapshotDirs.stream().mapToLong(FileUtils::folderSize).sum(); + long sum = sizeOnDisk; + if (sum == 0) + { + for (File snapshotDir : snapshotDirs) + sum += FileUtils.folderSize(snapshotDir); + + sizeOnDisk = sum; + } + + return sum; } public long computeTrueSizeBytes() { - DirectorySizeCalculator visitor = new SnapshotTrueSizeCalculator(); + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName); + if (cfs == null) + return 0; - for (File snapshotDir : snapshotDirs) + return computeTrueSizeBytes(cfs.getFilesOfCfs()); + } + + public long computeTrueSizeBytes(Set files) + { + long size = manifestsSize + schemasSize; + + for (File dataPath : getDirectories()) { - try + List snapshotFiles = listDir(dataPath.toPath()); + for (Path snapshotFile : snapshotFiles) { - Files.walkFileTree(snapshotDir.toPath(), visitor); - } - catch (IOException e) - { - logger.error("Could not calculate the size of {}.", snapshotDir, e); + if (!snapshotFile.endsWith("manifest.json") && !snapshotFile.endsWith("schema.cql")) + { + // files == null means that the underlying table was most probably dropped + // so in that case we indeed go to count snapshots file in for true size + if (files == null || (!files.contains(getLiveFileFromSnapshotFile(snapshotFile)))) + size += getFileSize(snapshotFile); + } } } - return visitor.getAllocatedSize(); + return size; + } + + private List listDir(Path dir) + { + List paths = new ArrayList<>(); + try (Stream stream = Files.list(dir)) + { + stream.forEach(p -> { + if (p.getFileName().toString().startsWith(".")) + { + paths.addAll(listDir(p)); + } + else + { + paths.add(p); + } + }); + } + catch (Throwable t) + { + logger.error("Could not list directory content {}", dir, t); + } + + return paths; + } + + private long getFileSize(Path file) + { + try + { + return Files.size(file); + } + catch (Throwable t) + { + return 0; + } + } + + /** + * Returns the corresponding live file for a given snapshot file. + *

+ * Example: + * - Base table: + * - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db + * - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db + * - Secondary index: + * - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db + * - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db + */ + static String getLiveFileFromSnapshotFile(Path snapshotFilePath) + { + // Snapshot directory structure format is {data_dir}/snapshots/{snapshot_name}/{snapshot_file} + Path liveDir = snapshotFilePath.getParent().getParent().getParent(); + if (Directories.isSecondaryIndexFolder(snapshotFilePath.getParent())) + { + // Snapshot file structure format is {data_dir}/snapshots/{snapshot_name}/.{index}/{sstable-component}.db + liveDir = File.getPath(liveDir.getParent().toString(), snapshotFilePath.getParent().getFileName().toString()); + } + return liveDir.resolve(snapshotFilePath.getFileName().toString()).toAbsolutePath().toString(); } public Collection getDirectories() @@ -165,48 +302,88 @@ public class TableSnapshot return snapshotDirs; } - public Optional getManifestFile() + /** + * Returns all manifest files of a snapshot. + *

+ * In practice, there might be multiple manifest files, as many as we have snapshot dirs. + * Each snapshot dir will hold its view of a snapshot, containing only sstables located in such snapshot dir. + * + * @return all manifest files + */ + public Set getManifestFiles() { + Set manifestFiles = new HashSet<>(); for (File snapshotDir : snapshotDirs) { File manifestFile = Directories.getSnapshotManifestFile(snapshotDir); if (manifestFile.exists()) - { - return Optional.of(manifestFile); - } + manifestFiles.add(manifestFile); } - return Optional.empty(); + return manifestFiles; } - public Optional getSchemaFile() + public boolean hasManifest() { + for (File snapshotDir : snapshotDirs) + { + if (Directories.getSnapshotManifestFile(snapshotDir).exists()) + return true; + } + return false; + } + + /** + * Returns all schemas files of a snapshot. + * + * @return all schema files + */ + public Set getSchemaFiles() + { + Set schemaFiles = new HashSet<>(); for (File snapshotDir : snapshotDirs) { File schemaFile = Directories.getSnapshotSchemaFile(snapshotDir); if (schemaFile.exists()) - { - return Optional.of(schemaFile); - } + schemaFiles.add(schemaFile); } - return Optional.empty(); + return schemaFiles; } + public long getManifestsSize() + { + long size = 0; + for (File manifestFile : getManifestFiles()) + size += manifestFile.length(); + + return size; + } + + public long getSchemasSize() + { + long size = 0; + for (File schemaFile : getSchemaFiles()) + size += schemaFile.length(); + + return size; + } + + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TableSnapshot snapshot = (TableSnapshot) o; - return Objects.equals(keyspaceName, snapshot.keyspaceName) && Objects.equals(tableName, snapshot.tableName) && - Objects.equals(tableId, snapshot.tableId) && Objects.equals(tag, snapshot.tag) && - Objects.equals(createdAt, snapshot.createdAt) && Objects.equals(expiresAt, snapshot.expiresAt) && - Objects.equals(snapshotDirs, snapshot.snapshotDirs) && Objects.equals(ephemeral, snapshot.ephemeral); + return Objects.equals(keyspaceName, snapshot.keyspaceName) && + Objects.equals(tableName, snapshot.tableName) && + Objects.equals(tableId, snapshot.tableId) && + Objects.equals(tag, snapshot.tag); } @Override public int hashCode() { - return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral); + return Objects.hash(keyspaceName, tableName, tableId, tag); } @Override @@ -224,7 +401,16 @@ public class TableSnapshot '}'; } - static class Builder { + public void updateMetadataSize() + { + if (manifestsSize == 0) + manifestsSize = getManifestsSize(); + if (schemasSize == 0) + schemasSize = getSchemasSize(); + } + + static class Builder + { private final String keyspaceName; private final String tableName; private final UUID tableId; @@ -287,66 +473,82 @@ public class TableSnapshot return String.format("%s:%s:%s:%s", keyspaceName, tableName, tableId, tag); } - public static class SnapshotTrueSizeCalculator extends DirectorySizeCalculator + public static Set getSnapshotDescriptors(String keyspace, String table, String tag) { - /** - * Snapshots are composed of hard-linked sstables. The true snapshot size should only include - * snapshot files which do not contain a corresponding "live" sstable file. - */ - @Override - public boolean isAcceptable(Path snapshotFilePath) + try { - return !getLiveFileFromSnapshotFile(snapshotFilePath).exists(); - } - } + Refs snapshotSSTableReaders = getSnapshotSSTableReaders(keyspace, table, tag); - /** - * Returns the corresponding live file for a given snapshot file. - * - * Example: - * - Base table: - * - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db - * - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db - * - Secondary index: - * - Snapshot file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db - * - Live file: ~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db - * - */ - static File getLiveFileFromSnapshotFile(Path snapshotFilePath) - { - // Snapshot directory structure format is {data_dir}/snapshots/{snapshot_name}/{snapshot_file} - Path liveDir = snapshotFilePath.getParent().getParent().getParent(); - if (Directories.isSecondaryIndexFolder(snapshotFilePath.getParent())) - { - // Snapshot file structure format is {data_dir}/snapshots/{snapshot_name}/.{index}/{sstable-component}.db - liveDir = File.getPath(liveDir.getParent().toString(), snapshotFilePath.getParent().getFileName().toString()); - } - return new File(liveDir.toString(), snapshotFilePath.getFileName().toString()); - } - - public static Predicate shouldClearSnapshot(String tag, long olderThanTimestamp) - { - return ts -> - { - // When no tag is supplied, all snapshots must be cleared - boolean clearAll = tag == null || tag.isEmpty(); - if (!clearAll && ts.isEphemeral()) - logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " + - "Ephemeral snapshots are not removable by a user.", - tag, ts.keyspaceName); - boolean notEphemeral = !ts.isEphemeral(); - boolean shouldClearTag = clearAll || ts.tag.equals(tag); - boolean byTimestamp = true; - - if (olderThanTimestamp > 0L) + Set descriptors = new HashSet<>(); + for (SSTableReader ssTableReader : snapshotSSTableReaders) { - Instant createdAt = ts.getCreatedAt(); - if (createdAt != null) - byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp)); + descriptors.add(ssTableReader.descriptor); } - return notEphemeral && shouldClearTag && byTimestamp; - }; + return descriptors; + } + catch (IOException e) + { + throw Throwables.unchecked(e); + } } + public static Refs getSnapshotSSTableReaders(String keyspace, String table, String tag) throws IOException + { + return getSnapshotSSTableReaders(Keyspace.open(keyspace).getColumnFamilyStore(table), tag); + } + + public static Refs getSnapshotSSTableReaders(ColumnFamilyStore cfs, String tag) + { + Map active = new HashMap<>(); + for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL)) + active.put(sstable.descriptor.id, sstable); + Map> snapshots = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list(); + Refs refs = new Refs<>(); + try + { + for (Map.Entry> entries : snapshots.entrySet()) + { + // Try to acquire reference to an active sstable instead of snapshot if it exists, + // to avoid opening new sstables. If it fails, use the snapshot reference instead. + SSTableReader sstable = active.get(entries.getKey().id); + if (sstable == null || !refs.tryRef(sstable)) + { + if (logger.isTraceEnabled()) + logger.trace("using snapshot sstable {}", entries.getKey()); + // open offline so we don't modify components or track hotness. + sstable = SSTableReader.open(cfs, entries.getKey(), entries.getValue(), cfs.metadata, true, true); + refs.tryRef(sstable); + // release the self ref as we never add the snapshot sstable to DataTracker where it is otherwise released + sstable.selfRef().release(); + } + else if (logger.isTraceEnabled()) + { + logger.trace("using active sstable {}", entries.getKey()); + } + } + } + catch (FSReadError | RuntimeException e) + { + // In case one of the snapshot sstables fails to open, + // we must release the references to the ones we opened so far + refs.release(); + throw e; + } + return refs; + } + + public static String getTimestampedSnapshotName(String clientSuppliedName, long timestamp) + { + String snapshotName = Long.toString(timestamp); + if (clientSuppliedName != null && !clientSuppliedName.isEmpty()) + snapshotName = snapshotName + '-' + clientSuppliedName; + + return snapshotName; + } + + public static String getTimestampedSnapshotNameWithPrefix(String clientSuppliedName, long timestamp, String prefix) + { + return prefix + '-' + getTimestampedSnapshotName(clientSuppliedName, timestamp); + } } diff --git a/src/java/org/apache/cassandra/service/snapshot/TakeSnapshotTask.java b/src/java/org/apache/cassandra/service/snapshot/TakeSnapshotTask.java new file mode 100644 index 0000000000..2b13459765 --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/TakeSnapshotTask.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.io.IOException; +import java.io.PrintStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SchemaCQLHelper; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.internal.CassandraIndex; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; + +public class TakeSnapshotTask extends AbstractSnapshotTask> +{ + private static final Logger logger = LoggerFactory.getLogger(TakeSnapshotTask.class); + private final SnapshotManager manager; + + private Instant creationTime; + private String snapshotName; + private final Map snapshotsToCreate = new HashMap<>(); + + public TakeSnapshotTask(SnapshotManager manager, SnapshotOptions options) + { + super(options); + this.manager = manager; + } + + @Override + public SnapshotTaskType getTaskType() + { + return SnapshotTaskType.SNAPSHOT; + } + + public Map getSnapshotsToCreate() + { + if (StorageService.instance.operationMode() == StorageService.Mode.JOINING) + throw new RuntimeException("Cannot snapshot until bootstrap completes"); + + // This is not in builder's build method on purpose in order to postpone the timestamp for as long as possible + // until the actual snapshot is taken. If we constructed a task and have not done anything with it for 5 minutes + // then by the time a snapshot would be taken the creation time would be quite off + Instant creationTimeInOptions = options.creationTime; + if (creationTimeInOptions == null) + creationTime = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()); + else + creationTime = options.creationTime; + + snapshotName = options.getSnapshotName(creationTime); + + Set entitiesForSnapshot = options.cfs == null ? parseEntitiesForSnapshot(options.entities) : Set.of(options.cfs); + + for (ColumnFamilyStore cfs : entitiesForSnapshot) + { + Set snapshotDirs = cfs.getDirectories().getSnapshotDirs(snapshotName); + + TableSnapshot tableSnapshot = new TableSnapshot(cfs.metadata.keyspace, + cfs.metadata.name, + cfs.metadata.id.asUUID(), + snapshotName, + creationTime, + SnapshotManifest.computeExpiration(options.ttl, creationTime), + snapshotDirs, + options.ephemeral); + // this snapshot does not have any actual representation on disk + // because that snapshot was technically not taken yet + tableSnapshot.incomplete(); + snapshotsToCreate.put(cfs, tableSnapshot); + } + + return snapshotsToCreate; + } + + @Override + public List call() + { + assert snapshotName != null : "You need to call getSnapshotsToCreate() first"; + + List createdSnapshots = new ArrayList<>(); + + for (Map.Entry entry : snapshotsToCreate.entrySet()) + { + try + { + ColumnFamilyStore cfs = entry.getKey(); + if (!options.skipFlush) + { + Memtable current = cfs.getTracker().getView().getCurrentMemtable(); + if (!current.isClean()) + { + if (current.shouldSwitch(ColumnFamilyStore.FlushReason.SNAPSHOT)) + FBUtilities.waitOnFuture(cfs.switchMemtableIfCurrent(current, ColumnFamilyStore.FlushReason.SNAPSHOT)); + else + current.performSnapshot(snapshotName); + } + } + + createSnapshot(cfs, entry.getValue(), snapshotName, creationTime); + createdSnapshots.add(entry.getValue()); + } + catch (Throwable t) + { + logger.warn(String.format("Unable to create snapshot %s for %s", entry.getValue().getTag(), entry.getKey().getKeyspaceTableName()), t); + // if we fail to take a snapshot, there is its phantom / in-progress representation among manager's taken snapshots, + // so we need to remove it from there to not appear in the outputs / not leak + manager.getSnapshots().remove(entry.getValue()); + } + } + + return createdSnapshots; + } + + private void createSnapshot(ColumnFamilyStore cfs, TableSnapshot snapshotToCreate, String snapshotName, Instant creationTime) + { + Predicate predicate = options.sstableFilter; + Set sstables = new LinkedHashSet<>(); + for (ColumnFamilyStore aCfs : cfs.concatWithIndexes()) + { + try (ColumnFamilyStore.RefViewFragment currentView = aCfs.selectAndReference(View.select(SSTableSet.CANONICAL, (x) -> predicate == null || predicate.test(x)))) + { + for (SSTableReader ssTable : currentView.sstables) + { + File snapshotDirectory = Directories.getSnapshotDirectory(ssTable.descriptor, snapshotName); + ssTable.createLinks(snapshotDirectory.path(), options.rateLimiter); // hard links + if (logger.isTraceEnabled()) + logger.trace("Snapshot for {} keyspace data file {} created in {}", cfs.keyspace, ssTable.getFilename(), snapshotDirectory); + sstables.add(ssTable); + } + } + } + + List dataComponents = new ArrayList<>(); + for (SSTableReader sstable : sstables) + dataComponents.add(sstable.descriptor.relativeFilenameFor(SSTableFormat.Components.DATA)); + + SnapshotManifest manifest = new SnapshotManifest(dataComponents, options.ttl, creationTime, options.ephemeral); + Set snapshotDirs = cfs.getDirectories().getSnapshotDirs(snapshotName); + for (File snapshotDir : snapshotDirs) + { + writeSnapshotManifest(manifest, Directories.getSnapshotManifestFile(snapshotDir)); + + if (!SchemaConstants.isLocalSystemKeyspace(cfs.metadata.keyspace) + && !SchemaConstants.isReplicatedSystemKeyspace(cfs.metadata.keyspace)) + { + writeSnapshotSchema(Directories.getSnapshotSchemaFile(snapshotDir), cfs); + } + } + + snapshotToCreate.updateMetadataSize(); + snapshotToCreate.complete(); + } + + private Set parseEntitiesForSnapshot(String... entities) + { + Set entitiesForSnapshot = new HashSet<>(); + + if (entities != null && entities.length > 0 && entities[0].contains(".")) + { + for (String entity : entities) + { + String[] splitted = StringUtils.split(entity, '.'); + if (splitted.length == 2) + { + String keyspaceName = splitted[0]; + String tableName = splitted[1]; + + if (keyspaceName == null) + throw new RuntimeException("You must supply a keyspace name"); + if (tableName == null) + throw new RuntimeException("You must supply a table name"); + + Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspaceName); + ColumnFamilyStore existingTable = validKeyspace.getColumnFamilyStore(tableName); + + entitiesForSnapshot.add(existingTable); + } + // special case for index which we can not normally create a snapshot for + // but a snapshot is apparently taken before a secondary index is scrubbed, + // so we preserve this behavior + else if (splitted.length == 3) + { + String keyspaceName = splitted[0]; + String tableName = splitted[1]; + + Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspaceName); + ColumnFamilyStore existingTable = validKeyspace.getColumnFamilyStore(tableName); + Index indexByName = existingTable.indexManager.getIndexByName(splitted[2]); + if (indexByName instanceof CassandraIndex) + { + ColumnFamilyStore indexCfs = ((CassandraIndex) indexByName).getIndexCfs(); + entitiesForSnapshot.add(indexCfs); + } + else + { + throw new IllegalArgumentException("Unknown index " + entity); + } + } + else + { + throw new IllegalArgumentException("Cannot take a snapshot on secondary index or invalid column " + + "family name. You must supply a column family name in the " + + "form of keyspace.columnfamily"); + } + } + } + else + { + if (entities != null && entities.length == 0) + { + for (Keyspace keyspace : Keyspace.all()) + { + entitiesForSnapshot.addAll(keyspace.getColumnFamilyStores()); + } + } + else if (entities != null) + { + for (String keyspace : entities) + { + Keyspace validKeyspace = Keyspace.getValidKeyspace(keyspace); + entitiesForSnapshot.addAll(validKeyspace.getColumnFamilyStores()); + } + } + } + + return entitiesForSnapshot; + } + + + private void writeSnapshotManifest(SnapshotManifest manifest, File manifestFile) + { + try + { + manifestFile.parent().tryCreateDirectories(); + manifest.serializeToJsonFile(manifestFile); + } + catch (IOException e) + { + throw new FSWriteError(e, manifestFile); + } + } + + private void writeSnapshotSchema(File schemaFile, ColumnFamilyStore cfs) + { + try + { + if (!schemaFile.parent().exists()) + schemaFile.parent().tryCreateDirectories(); + + try (PrintStream out = new PrintStream(new FileOutputStreamPlus(schemaFile))) + { + SchemaCQLHelper.reCreateStatementsForSchemaCql(cfs.metadata(), cfs.keyspace.getMetadata()) + .forEach(out::println); + } + } + catch (IOException e) + { + throw new FSWriteError(e, schemaFile); + } + } + + @Override + public String toString() + { + return "TakeSnapshotTask{" + + "options=" + options + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/service/snapshot/TrueSnapshotSizeTask.java b/src/java/org/apache/cassandra/service/snapshot/TrueSnapshotSizeTask.java new file mode 100644 index 0000000000..bcffa1fdac --- /dev/null +++ b/src/java/org/apache/cassandra/service/snapshot/TrueSnapshotSizeTask.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.Predicate; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; + +public class TrueSnapshotSizeTask implements Callable +{ + private final SnapshotManager snapshotManager; + private final Predicate predicate; + + public TrueSnapshotSizeTask(SnapshotManager snapshotManager, Predicate predicate) + { + this.snapshotManager = snapshotManager; + this.predicate = predicate; + } + + @Override + public Long call() + { + long size = 0; + for (TableSnapshot snapshot : snapshotManager.getSnapshots()) + { + if (predicate.test(snapshot)) + try + { + size += snapshot.computeTrueSizeBytes(getTablesFiles(snapshot.getKeyspaceName(), snapshot.getTableName())); + } + catch (Throwable ex) + { + // if any error happens while computing size, we don't include + return 0L; + } + } + + return size; + } + + private Set getTablesFiles(String keyspaceName, String tableName) + { + try + { + Keyspace keyspace = Keyspace.getValidKeyspace(keyspaceName); + ColumnFamilyStore table = keyspace.getColumnFamilyStore(tableName); + + return table.getFilesOfCfs(); + } + catch (IllegalArgumentException ex) + { + // keyspace / table not found + return null; + } + } + +} diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 5c75985bda..b2951a49b0 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -102,6 +102,7 @@ import org.apache.cassandra.net.MessagingServiceMBean; import org.apache.cassandra.service.ActiveRepairServiceMBean; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.CacheServiceMBean; +import org.apache.cassandra.service.snapshot.SnapshotManagerMBean; import org.apache.cassandra.tcm.CMSOperationsMBean; import org.apache.cassandra.service.GCInspector; import org.apache.cassandra.service.GCInspectorMXBean; @@ -150,6 +151,7 @@ public class NodeProbe implements AutoCloseable protected MBeanServerConnection mbeanServerConn; protected CompactionManagerMBean compactionProxy; protected StorageServiceMBean ssProxy; + protected SnapshotManagerMBean snapshotProxy; protected CMSOperationsMBean cmsProxy; protected GossiperMBean gossProxy; protected MemoryMXBean memProxy; @@ -263,6 +265,8 @@ public class NodeProbe implements AutoCloseable { ObjectName name = new ObjectName(ssObjName); ssProxy = JMX.newMBeanProxy(mbeanServerConn, name, StorageServiceMBean.class); + name = new ObjectName(SnapshotManagerMBean.MBEAN_NAME); + snapshotProxy = JMX.newMBeanProxy(mbeanServerConn, name, SnapshotManagerMBean.class); name = new ObjectName(CMSOperations.MBEAN_OBJECT_NAME); cmsProxy = JMX.newMBeanProxy(mbeanServerConn, name, CMSOperationsMBean.class); name = new ObjectName(MessagingService.MBEAN_NAME); @@ -875,12 +879,12 @@ public class NodeProbe implements AutoCloseable public long getSnapshotLinksPerSecond() { - return ssProxy.getSnapshotLinksPerSecond(); + return snapshotProxy.getSnapshotLinksPerSecond(); } public void setSnapshotLinksPerSecond(long throttle) { - ssProxy.setSnapshotLinksPerSecond(throttle); + snapshotProxy.setSnapshotLinksPerSecond(throttle); } /** @@ -900,10 +904,10 @@ public class NodeProbe implements AutoCloseable throw new IOException("When specifying the table for a snapshot, you must specify one and only one keyspace"); } - ssProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table); + snapshotProxy.takeSnapshot(snapshotName, options, keyspaces[0] + "." + table); } else - ssProxy.takeSnapshot(snapshotName, options, keyspaces); + snapshotProxy.takeSnapshot(snapshotName, options, keyspaces); } /** @@ -921,11 +925,11 @@ public class NodeProbe implements AutoCloseable { if (null != tableList && tableList.length != 0) { - ssProxy.takeSnapshot(snapshotName, options, tableList); + snapshotProxy.takeSnapshot(snapshotName, options, tableList); } else { - throw new IOException("The column family List for a snapshot should not be empty or null"); + throw new IOException("The column family list for a snapshot should not be empty or null"); } } @@ -955,12 +959,20 @@ public class NodeProbe implements AutoCloseable */ public void clearSnapshot(Map options, String tag, String... keyspaces) throws IOException { - ssProxy.clearSnapshot(options, tag, keyspaces); + snapshotProxy.clearSnapshot(tag, options, keyspaces); } + /** + * Gets all snapshots' details. + * + * @param options options to use upon listing of snapshots + * @return details of snapshots + */ + /** @deprecated See CASSANDRA-18111 */ + @Deprecated(since = "5.1") public Map getSnapshotDetails(Map options) { - return ssProxy.getSnapshotDetails(options); + return snapshotProxy.listSnapshots(options); } /** @deprecated See CASSANDRA-16789 */ @@ -972,7 +984,7 @@ public class NodeProbe implements AutoCloseable public long trueSnapshotsSize() { - return ssProxy.trueSnapshotsSize(); + return snapshotProxy.getTrueSnapshotSize(); } public boolean isJoined() diff --git a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java index 803fe5a4f0..6a1c71d386 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java +++ b/src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java @@ -95,4 +95,4 @@ public class ListSnapshots extends NodeToolCmd throw new RuntimeException("Error during list snapshot", e); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java index 52cc5df27c..c3392bbcf5 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Snapshot.java @@ -29,6 +29,7 @@ import io.airlift.airline.Command; import io.airlift.airline.Option; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.snapshot.SnapshotOptions; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; @@ -68,10 +69,10 @@ public class Snapshot extends NodeToolCmd sb.append("Requested creating snapshot(s) for "); Map options = new HashMap(); - options.put("skipFlush", Boolean.toString(skipFlush)); + options.put(SnapshotOptions.SKIP_FLUSH, Boolean.toString(skipFlush)); if (null != ttl) { DurationSpec.LongNanosecondsBound d = new DurationSpec.LongNanosecondsBound(ttl); - options.put("ttl", d.toString()); + options.put(SnapshotOptions.TTL, d.toString()); } if (!snapshotName.isEmpty() && snapshotName.contains(File.pathSeparator())) @@ -83,16 +84,16 @@ public class Snapshot extends NodeToolCmd { ktList = ktList.replace(" ", ""); if (keyspaces.isEmpty() && null == table) - sb.append("[").append(ktList).append("]"); + sb.append('[').append(ktList).append(']'); else { throw new IOException( "When specifying the Keyspace table list (using -kt,--kt-list,-kc,--kc.list), you must not also specify keyspaces to snapshot"); } if (!snapshotName.isEmpty()) - sb.append(" with snapshot name [").append(snapshotName).append("]"); - sb.append(" and options ").append(options.toString()); - out.println(sb.toString()); + sb.append(" with snapshot name [").append(snapshotName).append(']'); + sb.append(" and options ").append(options); + out.println(sb); probe.takeMultipleTableSnapshot(snapshotName, options, ktList.split(",")); out.println("Snapshot directory: " + snapshotName); } @@ -101,12 +102,12 @@ public class Snapshot extends NodeToolCmd if (keyspaces.isEmpty()) sb.append("[all keyspaces]"); else - sb.append("[").append(join(keyspaces, ", ")).append("]"); + sb.append('[').append(join(keyspaces, ", ")).append(']'); if (!snapshotName.isEmpty()) - sb.append(" with snapshot name [").append(snapshotName).append("]"); - sb.append(" and options ").append(options.toString()); - out.println(sb.toString()); + sb.append(" with snapshot name [").append(snapshotName).append(']'); + sb.append(" and options ").append(options); + out.println(sb); probe.takeSnapshot(snapshotName, table, options, toArray(keyspaces, String.class)); out.println("Snapshot directory: " + snapshotName); diff --git a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java index dcae575940..caf2082bda 100644 --- a/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java +++ b/src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java @@ -27,6 +27,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Predicate; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -39,12 +40,16 @@ import org.apache.cassandra.db.SnapshotCommand; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotType; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; import static org.apache.cassandra.config.CassandraRelevantProperties.DIAGNOSTIC_SNAPSHOT_INTERVAL_NANOS; @@ -194,7 +199,8 @@ public class DiagnosticSnapshotService } ColumnFamilyStore cfs = ks.getColumnFamilyStore(command.column_family); - if (cfs.snapshotExists(command.snapshot_name)) + + if (SnapshotManager.instance.exists(command.keyspace, command.column_family, command.snapshot_name)) { logger.info("Received diagnostic snapshot request from {} for {}.{}, " + "but snapshot with tag {} already exists", @@ -210,16 +216,14 @@ public class DiagnosticSnapshotService command.column_family, command.snapshot_name); - if (ranges.isEmpty()) - cfs.snapshot(command.snapshot_name); - else - { - cfs.snapshot(command.snapshot_name, - (sstable) -> checkIntersection(ranges, - sstable.getFirst().getToken(), - sstable.getLast().getToken()), - false, false); - } + Predicate predicate = null; + if (!ranges.isEmpty()) + predicate = (sstable) -> checkIntersection(ranges, + sstable.getFirst().getToken(), + sstable.getLast().getToken()); + + SnapshotOptions options = SnapshotOptions.systemSnapshot(command.snapshot_name, SnapshotType.DIAGNOSTICS, predicate, cfs.getKeyspaceTableName()).build(); + SnapshotManager.instance.takeSnapshot(options); } catch (IllegalArgumentException e) { diff --git a/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java b/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java index 94f9b229b4..22da5c06cf 100644 --- a/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java +++ b/src/java/org/apache/cassandra/utils/DirectorySizeCalculator.java @@ -42,7 +42,7 @@ public class DirectorySizeCalculator extends SimpleFileVisitor } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { if (isAcceptable(file)) size += attrs.size(); @@ -63,7 +63,7 @@ public class DirectorySizeCalculator extends SimpleFileVisitor /** * Reset the size to 0 in case that the size calculator is used multiple times */ - protected void resetSize() + public void resetSize() { size = 0; } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index 7b5501fbad..1d2272ad4d 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -632,7 +632,6 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { partialStartup(cluster); } - StorageService.instance.startSnapshotManager(); } catch (Throwable t) { @@ -719,6 +718,12 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance CassandraDaemon.getInstanceForTesting().migrateSystemDataIfNeeded(); CommitLog.instance.start(); + + SnapshotManager.instance.start(false); + SnapshotManager.instance.clearExpiredSnapshots(); + SnapshotManager.instance.clearEphemeralSnapshots(); + SnapshotManager.instance.resumeSnapshotCleanup(); + CassandraDaemon.getInstanceForTesting().runStartupChecks(); Keyspace.setInitialized(); // TODO: this seems to be superfluous by now @@ -891,6 +896,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance Future future = async((ExecutorService executor) -> { Throwable error = null; + error = parallelRun(error, executor, SnapshotManager.instance::close); + CompactionManager.instance.forceShutdown(); error = parallelRun(error, executor, @@ -945,7 +952,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance () -> shutdownAndWait(Collections.singletonList(ActiveRepairService.repairCommandExecutor())), () -> ActiveRepairService.instance().shutdownNowAndWait(1L, MINUTES), () -> EpochAwareDebounce.instance.close(), - () -> SnapshotManager.shutdownAndWait(1L, MINUTES) + SnapshotManager.instance::close ); internodeMessagingStarted = false; diff --git a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java index a1a29f09cb..2f9d628d54 100644 --- a/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java +++ b/test/distributed/org/apache/cassandra/distributed/mock/nodetool/InternalNodeProbe.java @@ -44,6 +44,7 @@ import org.apache.cassandra.service.CacheServiceMBean; import org.apache.cassandra.service.GCInspector; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.streaming.StreamManager; import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tools.NodeProbe; @@ -69,6 +70,7 @@ public class InternalNodeProbe extends NodeProbe StorageService.instance.skipNotificationListeners = !withNotifications; ssProxy = StorageService.instance; + snapshotProxy = SnapshotManager.instance; cmsProxy = CMSOperations.instance; msProxy = MessagingService.instance(); streamProxy = StreamManager.instance; diff --git a/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java index 7a94905abb..aa7da3445b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/AllowAutoSnapshotTest.java @@ -27,7 +27,7 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.IIsolatedExecutor; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import static org.apache.cassandra.distributed.Cluster.build; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; @@ -147,8 +147,8 @@ public class AllowAutoSnapshotTest extends TestBaseImpl { final int node = i; // has to be effectively final for the usage in "until" method await().until(() -> cluster.get(node).appliesOnInstance((IIsolatedExecutor.SerializableTriFunction) (shouldContainSnapshot, tableName, prefix) -> { - Stream stream = StorageService.instance.getSnapshotDetails(Collections.emptyMap()).keySet().stream(); - Predicate predicate = tag -> tag.startsWith(prefix + '-') && tag.endsWith('-' + tableName); + Stream stream = SnapshotManager.instance.listSnapshots(Collections.emptyMap()).keySet().stream(); + Predicate predicate = tag -> tag.contains(prefix + '-' + table); return shouldContainSnapshot ? stream.anyMatch(predicate) : stream.noneMatch(predicate); }).apply(shouldContain, table, snapshotPrefix)); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/AutoSnapshotTtlTest.java b/test/distributed/org/apache/cassandra/distributed/test/AutoSnapshotTtlTest.java index 550cbdc6e4..e08c16adcc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/AutoSnapshotTtlTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/AutoSnapshotTtlTest.java @@ -25,16 +25,14 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.CassandraRelevantProperties; -import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.shared.WithProperties; +import org.apache.cassandra.service.snapshot.SnapshotType; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_DROP_PREFIX; -import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_TRUNCATE_PREFIX; import static org.apache.cassandra.distributed.Cluster.build; import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked; import static org.awaitility.Awaitility.await; @@ -81,12 +79,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl cluster.schemaChange(withKeyspace("TRUNCATE %s.tbl;")); // Check snapshot is listed after table is truncated - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_TRUNCATE_PREFIX); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label); // Check snapshot is removed after 10s await().timeout(10, SECONDS) .pollInterval(1, SECONDS) - .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX)); + .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label)); } } @@ -111,12 +109,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl cluster.schemaChange(withKeyspace("DROP TABLE %s.tbl;")); // Check snapshot is listed after table is dropped - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label); // Check snapshot is removed after 10s await().timeout(10, SECONDS) .pollInterval(1, SECONDS) - .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX)); + .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label)); } } @@ -146,12 +144,12 @@ public class AutoSnapshotTtlTest extends TestBaseImpl instance.startup(); // Check snapshot is listed after restart - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label); // Check snapshot is removed after at most auto_snapshot_ttl + 1s await().timeout(ONE_MINUTE + 1, SECONDS) .pollInterval(1, SECONDS) - .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SNAPSHOT_DROP_PREFIX)); + .until(() -> !instance.nodetoolResult("listsnapshots").getStdout().contains(SnapshotType.DROP.label)); } } @@ -178,13 +176,13 @@ public class AutoSnapshotTtlTest extends TestBaseImpl cluster.schemaChange(withKeyspace("DROP TABLE %s.tbl;")); // Check snapshots are created after table is truncated and dropped - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_TRUNCATE_PREFIX); - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SNAPSHOT_DROP_PREFIX); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label); // Check snapshot are *NOT* expired after 10s Thread.sleep(2 * FIVE_SECONDS * 1000L); - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(ColumnFamilyStore.SNAPSHOT_TRUNCATE_PREFIX); - instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(ColumnFamilyStore.SNAPSHOT_DROP_PREFIX); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.TRUNCATE.label); + instance.nodetoolResult("listsnapshots").asserts().success().stdoutContains(SnapshotType.DROP.label); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java index 2de8f54694..f4d423cf23 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java @@ -30,6 +30,7 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.utils.Pair; @@ -61,6 +62,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl rewriteManifestToEphemeral(initialisationData.left, initialisationData.right); + c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + verify(c.get(1)); } } @@ -88,6 +91,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl Files.createFile(ephemeralMarkerFile); + c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + verify(c.get(1)); } } @@ -104,6 +109,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl Pair initialisationData = initialise(c); rewriteManifestToEphemeral(initialisationData.left, initialisationData.right); + c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName)); instance.nodetoolResult("clearsnapshot", "-t", snapshotName).asserts().success(); // ephemeral snapshot was not removed as it can not be (from nodetool / user operation) @@ -127,6 +134,8 @@ public class EphemeralSnapshotTest extends TestBaseImpl rewriteManifestToEphemeral(initialisationData.left, initialisationData.right); + c.get(1).runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> SnapshotManager.instance.restart(true)); + instance.nodetoolResult("clearsnapshot", "--all").asserts().success(); assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName)); assertFalse(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName2)); diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java index 529e7a3e5a..10c7dc187a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java @@ -36,6 +36,7 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.concurrent.Refs; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -126,7 +127,7 @@ public class PreviewRepairSnapshotTest extends TestBaseImpl String snapshotTag = await().atMost(1, MINUTES) .pollInterval(100, MILLISECONDS) .until(() -> { - for (String tag : cfs.listSnapshots().keySet()) + for (String tag : Util.listSnapshots(cfs).keySet()) { // we create the snapshot schema file last, so when this exists we know the snapshot is complete; if (cfs.getDirectories().getSnapshotSchemaFile(tag).exists()) @@ -138,7 +139,7 @@ public class PreviewRepairSnapshotTest extends TestBaseImpl Set inSnapshot = new HashSet<>(); - try (Refs sstables = cfs.getSnapshotSSTableReaders(snapshotTag)) + try (Refs sstables = TableSnapshot.getSnapshotSSTableReaders(cfs.getKeyspaceName(), cfs.name, snapshotTag)) { inSnapshot.addAll(sstables); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java index a0b643f0d3..82c81f221d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java @@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.util.concurrent.Uninterruptibles; +import org.apache.cassandra.Util; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.concurrent.Condition; import org.junit.BeforeClass; @@ -434,11 +435,11 @@ public class PreviewRepairTest extends TestBaseImpl ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table); if(shouldBeEmpty) { - assertTrue(cfs.listSnapshots().isEmpty()); + assertTrue(Util.listSnapshots(cfs).isEmpty()); } else { - while (cfs.listSnapshots().isEmpty()) + while (Util.listSnapshots(cfs).isEmpty()) Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); } })); diff --git a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java index 1c50f0ee3b..1e5773c67a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java @@ -30,6 +30,8 @@ import java.util.stream.Stream; import com.google.common.util.concurrent.Uninterruptibles; import org.junit.Assert; + +import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.SEPExecutor; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.dht.Token; @@ -39,6 +41,7 @@ import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.locator.ReplicaUtils; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.Throwables; import org.junit.Test; @@ -591,7 +594,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl int attempts = 100; ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); - while (cfs.listSnapshots().isEmpty()) + while (Util.listSnapshots(cfs).isEmpty()) { Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); if (attempts-- < 0) @@ -602,11 +605,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl private IInvokableInstance.SerializableRunnable assertSnapshotNotPresent(String snapshotName) { - return () -> - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); - Assert.assertFalse(cfs.snapshotExists(snapshotName)); - }; + return () -> Assert.assertFalse(SnapshotManager.instance.exists(KEYSPACE, TABLE, snapshotName)); } private long getConfirmedInconsistencies(IInvokableInstance instance) diff --git a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java index a44532ee15..c45108eea3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SSTableIdGenerationTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Function; @@ -51,6 +52,9 @@ import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.UUIDBasedSSTableId; import org.apache.cassandra.io.util.File; import org.apache.cassandra.metrics.RestorableMeter; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.tools.SystemExitException; import org.apache.cassandra.utils.TimeUUID; import org.assertj.core.api.Assertions; @@ -60,7 +64,6 @@ import static java.lang.String.format; import static org.apache.cassandra.Util.bulkLoadSSTables; import static org.apache.cassandra.Util.getBackups; import static org.apache.cassandra.Util.getSSTables; -import static org.apache.cassandra.Util.getSnapshots; import static org.apache.cassandra.Util.relativizePath; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.db.SystemKeyspace.LEGACY_SSTABLE_ACTIVITY; @@ -404,12 +407,20 @@ public class SSTableIdGenerationTest extends TestBaseImpl private static Set snapshot(IInvokableInstance instance, String ks, String tableName) { - Set snapshotDirs = instance.callOnInstance(() -> ColumnFamilyStore.getIfExists(ks, tableName) - .snapshot(SNAPSHOT_TAG) - .getDirectories() - .stream() - .map(File::toString) - .collect(Collectors.toSet())); + Set snapshotDirs = instance.callOnInstance(() -> { + + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ks, tableName); + + if (cfs == null) + return Set.of(); + + TableSnapshot tableSnapshot = SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(SNAPSHOT_TAG, cfs.getKeyspaceTableName())).iterator().next(); + Set dirs = new HashSet<>(); + for (File dir : tableSnapshot.getDirectories()) + dirs.add(dir.toString()); + + return dirs; + }); assertThat(snapshotDirs).isNotEmpty(); return snapshotDirs; } @@ -458,7 +469,7 @@ public class SSTableIdGenerationTest extends TestBaseImpl private static void assertSnapshotSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames) { - instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(getSnapshots(ks, tableName, SNAPSHOT_TAG), tableName, expectedSeqGenIds, expectedUUIDGenIds)))); + instance.runOnInstance(rethrow(() -> Arrays.stream(tableNames).forEach(tableName -> assertSSTablesCount(TableSnapshot.getSnapshotDescriptors(ks, tableName, SNAPSHOT_TAG), tableName, expectedSeqGenIds, expectedUUIDGenIds)))); } private static void assertBackupSSTablesCount(IInvokableInstance instance, int expectedSeqGenIds, int expectedUUIDGenIds, String ks, String... tableNames) diff --git a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java index 8ba3a90560..9ca5e05233 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/SnapshotsTest.java @@ -19,6 +19,9 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; @@ -29,9 +32,11 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableCallable; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.shared.WithProperties; import org.apache.cassandra.utils.Clock; @@ -40,8 +45,10 @@ import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class SnapshotsTest extends TestBaseImpl { @@ -51,9 +58,9 @@ public class SnapshotsTest extends TestBaseImpl private static final WithProperties properties = new WithProperties(); private static Cluster cluster; - private final String[] exoticSnapshotNames = new String[] { "snapshot", "snapshots", "backup", "backups", - "Snapshot", "Snapshots", "Backups", "Backup", - "snapshot.with.dots-and-dashes"}; + private final String[] exoticSnapshotNames = new String[]{ "snapshot", "snapshots", "backup", "backups", + "Snapshot", "Snapshots", "Backups", "Backup", + "snapshot.with.dots-and-dashes" }; @BeforeClass public static void before() throws IOException @@ -61,15 +68,18 @@ public class SnapshotsTest extends TestBaseImpl properties.set(CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS, 0); properties.set(CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS, SNAPSHOT_CLEANUP_PERIOD_SECONDS); properties.set(CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS, FIVE_SECONDS); - cluster = init(Cluster.build(1).start()); + cluster = init(Cluster.build(1) + .withDataDirCount(3) + .start()); } @After public void clearAllSnapshots() { cluster.schemaChange(withKeyspace("DROP TABLE IF EXISTS %s.tbl;")); + cluster.schemaChange(withKeyspace("DROP TABLE IF EXISTS %s.tbl2;")); cluster.get(1).nodetoolResult("clearsnapshot", "--all").asserts().success(); - for (String tag : new String[] {"basic", "first", "second", "tag1"}) + for (String tag : new String[]{ "basic", "first", "second", "tag1" }) waitForSnapshotCleared(tag); for (String tag : exoticSnapshotNames) waitForSnapshot(tag, false, true); @@ -83,6 +93,34 @@ public class SnapshotsTest extends TestBaseImpl cluster.close(); } + @Test + public void testEverySnapshotDirHasManifestAndSchema() + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))")); + String[] dataDirs = (String[]) cluster.get(1).config().get("data_file_directories"); + String tableId = cluster.get(1).callOnInstance((SerializableCallable) () -> { + return ColumnFamilyStore.getIfExists("distributed_test_keyspace", "tbl").metadata().id.toHexString(); + }); + + cluster.get(1) + .nodetoolResult("snapshot", "-t", "mysnapshot", "-kt", format("%s.tbl", KEYSPACE)) + .asserts() + .success(); + + for (String dataDir : dataDirs) + { + Path snapshotDir = Paths.get(dataDir) + .resolve(KEYSPACE) + .resolve("tbl-" + tableId) + .resolve("snapshots") + .resolve("mysnapshot"); + + assertTrue(snapshotDir.toFile().exists()); + assertTrue(snapshotDir.resolve("manifest.json").toFile().exists()); + assertTrue(snapshotDir.resolve("schema.cql").toFile().exists()); + } + } + @Test public void testSnapshotsCleanupByTTL() { @@ -161,7 +199,8 @@ public class SnapshotsTest extends TestBaseImpl } @Test - public void testManualSnapshotCleanup() { + public void testManualSnapshotCleanup() + { // take snapshots with ttl cluster.get(1).nodetoolResult("snapshot", "--ttl", format("%ds", TEN_SECONDS), @@ -210,8 +249,8 @@ public class SnapshotsTest extends TestBaseImpl populate(cluster); instance.nodetoolResult("snapshot", - "-t", "tag1", - "-kt", withKeyspace("%s.tbl")).asserts().success(); + "-t", "tag1", + "-kt", withKeyspace("%s.tbl")).asserts().success(); // Check snapshot is listed when table is not dropped waitForSnapshotPresent("tag1"); @@ -314,21 +353,47 @@ public class SnapshotsTest extends TestBaseImpl Pattern COMPILE = Pattern.compile(" +"); long distinctTimestamps = Arrays.stream(result.getStdout().split("\n")) - .filter(line -> line.startsWith("sametimestamp")) - .map(line -> COMPILE.matcher(line).replaceAll(" ").split(" ")[7]) - .distinct() - .count(); + .filter(line -> line.startsWith("sametimestamp")) + .map(line -> COMPILE.matcher(line).replaceAll(" ").split(" ")[7]) + .distinct() + .count(); // assert all dates are same so there is just one value accross all individual tables assertEquals(1, distinctTimestamps); } + @Test + public void testFailureToSnapshotTwiceOnSameEntityWithSameSnapshotName() + { + cluster.get(1).nodetoolResult("snapshot", "-t", "somename").asserts().success(); + + NodeToolResult failedSnapshotResult = cluster.get(1).nodetoolResult("snapshot", "-t", "somename"); + failedSnapshotResult.asserts().failure(); + Throwable error = failedSnapshotResult.getError(); + assertThat(error.getMessage()).contains("already exists"); + } + + @Test + public void testTakingSnapshoWithSameNameOnDifferentTablesDoesNotFail() + { + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (key int, value text, PRIMARY KEY (key))")); + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl2 (key int, value text, PRIMARY KEY (key))")); + cluster.get(1).nodetoolResult("snapshot", "-t", "somename", "-kt", String.format("%s.tbl", KEYSPACE)).asserts().success(); + cluster.get(1).nodetoolResult("snapshot", "-t", "somename", "-kt", String.format("%s.tbl2", KEYSPACE)).asserts().success(); + } + private void populate(Cluster cluster) { for (int i = 0; i < 100; i++) cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (key, value) VALUES (?, 'txt')"), ConsistencyLevel.ONE, i); } + private void populate(Cluster cluster, String keyspace, String table) + { + for (int i = 0; i < 100; i++) + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (key, value) VALUES (?, 'txt')", keyspace, table), ConsistencyLevel.ONE, i); + } + private void waitForSnapshotPresent(String snapshotName) { waitForSnapshot(snapshotName, true, false); @@ -339,20 +404,28 @@ public class SnapshotsTest extends TestBaseImpl waitForSnapshot(snapshotName, false, false); } - private void waitForSnapshot(String snapshotName, boolean expectPresent, boolean noTTL) + private void waitForSnapshot(String keyspaceName, String tableName, String snapshotName, boolean expectPresent, boolean noTTL) { await().timeout(20, SECONDS) .pollDelay(0, SECONDS) .pollInterval(1, SECONDS) - .until(() -> waitForSnapshotInternal(snapshotName, expectPresent, noTTL)); + .until(() -> waitForSnapshotInternal(keyspaceName, tableName, snapshotName, expectPresent, noTTL)); } - private boolean waitForSnapshotInternal(String snapshotName, boolean expectPresent, boolean noTTL) { + private void waitForSnapshot(String snapshotName, boolean expectPresent, boolean noTTL) + { + waitForSnapshot(null, null, snapshotName, expectPresent, noTTL); + } + + private boolean waitForSnapshotInternal(String keyspaceName, String tableName, String snapshotName, boolean expectPresent, boolean noTTL) + { + List args = new ArrayList<>(); + args.add("listsnapshots"); NodeToolResult listsnapshots; if (noTTL) - listsnapshots = cluster.get(1).nodetoolResult("listsnapshots", "-nt"); - else - listsnapshots = cluster.get(1).nodetoolResult("listsnapshots"); + args.add("-nt"); + + listsnapshots = cluster.get(1).nodetoolResult(args.toArray(new String[0])); List lines = Arrays.stream(listsnapshots.getStdout().split("\n")) .filter(line -> !line.isEmpty()) diff --git a/test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java b/test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java new file mode 100644 index 0000000000..8c82b1cb11 --- /dev/null +++ b/test/distributed/org/apache/cassandra/fuzz/snapshots/SnapshotsTest.java @@ -0,0 +1,1251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.fuzz.snapshots; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import javax.management.openmbean.TabularData; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.MapDifference.ValueDifference; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.Test; + +import accord.utils.Property.StateOnlyCommand; +import accord.utils.RandomSource; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableFunction; +import org.apache.cassandra.distributed.api.IIsolatedExecutor.SerializableTriConsumer; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.fuzz.snapshots.SnapshotsTest.State.TestSnapshot; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.utils.Generators; +import org.apache.cassandra.utils.LocalizeString; +import org.apache.cassandra.utils.Pair; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.generators.SourceDSL; +import org.quicktheories.impl.JavaRandom; + +import static accord.utils.Property.commands; +import static accord.utils.Property.stateful; +import static com.google.common.collect.Sets.difference; +import static java.lang.String.format; +import static java.util.UUID.randomUUID; +import static org.apache.cassandra.fuzz.snapshots.SnapshotsTest.CreateKeyspace.createKeypace; +import static org.apache.cassandra.fuzz.snapshots.SnapshotsTest.CreateTable.createTable; +import static org.junit.Assert.assertEquals; + +public class SnapshotsTest +{ + @Test + public void fuzzySnapshotsTest() + { + stateful() + .withExamples(1) + .withSteps(500) + .withStepTimeout(Duration.ofMinutes(1)) + .check(commands(() -> State::new) + .add(CreateKeyspace::new) + .add(DropKeyspace::new) + .add(CreateTable::new) + .add(DropTable::new) + .add(TruncateTable::new) + .add(TakeSnapshot::new) + .add(ClearSnapshot::new) + .add(ListSnapshots::new) + .destroyState(State::destroy) + .build()); + } + + public static class ListSnapshots extends AbstractCommand + { + private final Pair> listingParams; + + public ListSnapshots(RandomSource rs, State state) + { + super(rs, state); + listingParams = generateParams(); + } + + @Override + public void doWork(State state) + { + assertEquals(listingParams.left, categorize(list(listingParams.right))); + } + + @Override + public String toString() + { + return "List snapshots with parameters: " + listingParams; + } + + private Pair> generateParams() + { + Map listingParams = new HashMap<>(); + List toShuffle = state.getShuffledListOfInts(4); + + String keyspace = null; + for (int i = 0; i < 4; i++) + { + boolean picked = false; + switch (toShuffle.get(i)) + { + case 1: + if (!state.truncatedSnapshots.isEmpty()) + { + keyspace = state.rs.pick(state.truncatedSnapshots.keySet()).split("\\.")[0]; + picked = true; + } + break; + case 2: + if (!state.droppedSnapshots.isEmpty()) + { + keyspace = state.rs.pick(state.droppedSnapshots).split("\\.")[0]; + picked = true; + } + break; + case 3: + if (!state.snapshots.isEmpty()) + { + keyspace = state.rs.pick(state.snapshots).getKeyspaceName(); + picked = true; + } + break; + default: + // keyspace will be null so all snapshost will be listed + picked = true; + break; + } + if (picked) + break; + } + + // we need to populate expected snapshots after listing + SnapshotsHolder holder = new SnapshotsHolder(); + + if (keyspace == null) + { + holder.normal.addAll(state.snapshots); + holder.dropped.addAll(state.droppedSnapshots); + holder.truncated.putAll(state.truncatedSnapshots); + } + else + { + listingParams.put("keyspace", keyspace); + for (String s : state.droppedSnapshots) + if (s.startsWith(keyspace + '.')) + holder.dropped.add(s); + + for (Map.Entry entry : state.truncatedSnapshots.entrySet()) + if (entry.getKey().startsWith(keyspace + '.')) + holder.truncated.put(entry.getKey(), entry.getValue()); + + for (TestSnapshot s : state.snapshots) + if (s.getKeyspaceName().equals(keyspace)) + holder.normal.add(s); + } + + return Pair.create(holder, listingParams); + } + + private List list(Map parameters) + { + return getNode().applyOnInstance((SerializableFunction, List>) (params) -> + { + Map listingResult = SnapshotManager.instance.listSnapshots(params); + + List snapshots = new ArrayList<>(); + for (final Map.Entry snapshotDetail : listingResult.entrySet()) + { + Set values = snapshotDetail.getValue().keySet(); + for (Object eachValue : values) + { + final List value = (List) eachValue; + String tag = (String) value.get(0); + String keyspace = (String) value.get(1); + String table = (String) value.get(2); + snapshots.add(format("%s.%s.%s", keyspace, table, tag)); + } + } + + return snapshots; + }, parameters); + } + + private SnapshotsHolder categorize(List listedSnapshots) + { + SnapshotsHolder holder = new SnapshotsHolder(); + for (String snapshot : listedSnapshots) + { + if (snapshot.contains("dropped-")) + { + String[] split = snapshot.split("\\."); + holder.dropped.add(format("%s.%s", split[0], split[1])); + } + else if (snapshot.contains("truncated-")) + { + String[] split = snapshot.split("\\."); + String ksTb = format("%s.%s", split[0], split[1]); + holder.truncated.merge(ksTb, 1, Integer::sum); + } + else + { + String[] split = snapshot.split("\\."); + holder.normal.add(new TestSnapshot(split[0], split[1], split[2])); + } + } + + return holder; + } + + private static class SnapshotsHolder + { + Set normal = new HashSet<>(); + Set dropped = new HashSet<>(); + Map truncated = new HashMap<>(); + + @Override + public String toString() + { + return "SnapshotsHolder{" + + "normal=" + normal + + ", dropped=" + dropped + + ", truncated=" + truncated + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SnapshotsHolder holder = (SnapshotsHolder) o; + return Objects.equals(normal, holder.normal) && + Objects.equals(dropped, holder.dropped) && + Objects.equals(truncated, holder.truncated); + } + + @Override + public int hashCode() + { + return Objects.hash(normal, dropped, truncated); + } + } + } + + public static class CreateKeyspace extends AbstractCommand + { + private static final String CREATE_KEYSPACE_QUERY = + "CREATE KEYSPACE %s WITH replication = { 'class' : 'SimpleStrategy', 'replication_factor' : '1' }"; + + private final Set keyspaceToCreate = new HashSet<>(); + + public CreateKeyspace(RandomSource rs, State state) + { + super(rs, state); + keyspaceToCreate.add(state.addRandomKeyspace()); + } + + @Override + public void doWork(State state) + { + createKeypace(state); + assertEquals(state.schema.keySet(), state.getKeyspaces()); + state.populate(); + } + + public static void createKeypace(State state) + { + Set existingKeyspaces = state.getKeyspaces(); + Set keyspacesToBe = state.schema.keySet(); + + Set difference = difference(keyspacesToBe, existingKeyspaces); + if (!difference.isEmpty()) + state.getNode().executeInternal(format(CREATE_KEYSPACE_QUERY, difference.iterator().next())); + } + + @Override + public String toString() + { + return "Create keyspace " + keyspaceToCreate; + } + } + + public static class DropKeyspace extends AbstractCommand + { + private static final String DROP_KEYSPACE_QUERY = "DROP KEYSPACE %s"; + + private String keyspaceToDrop; + + private final boolean shouldApply; + + public DropKeyspace(RandomSource rs, State state) + { + super(rs, state); + + Optional>> maybeDroppedKeyspaceWithTables = state.removeRandomKeyspace(); + shouldApply = maybeDroppedKeyspaceWithTables.isPresent(); + + if (!shouldApply) + return; + + // dropping of a keyspace will create dropped tables which create "dropped" snapshots + String keyspace = maybeDroppedKeyspaceWithTables.get().left; + List tables = maybeDroppedKeyspaceWithTables.get().right; + for (String table : tables) + state.addDroppedSnapshot(keyspace, table); + + Set existingKeyspaces = state.getKeyspaces(); + Set keyspacesToBe = state.schema.keySet(); + Set diff = difference(existingKeyspaces, keyspacesToBe); + assert diff.size() == 1; + keyspaceToDrop = diff.iterator().next(); + } + + @Override + public void doWork(State state) + { + if (!shouldApply) + return; + + executeQuery(format(DROP_KEYSPACE_QUERY, keyspaceToDrop)); + assertEquals(state.schema.keySet(), state.getKeyspaces()); + state.populate(); + } + + @Override + public String toString() + { + if (keyspaceToDrop != null) + return "Drop keyspace " + keyspaceToDrop; + else + return "Drop keyspace skipping"; + } + } + + public static class CreateTable extends AbstractCommand + { + private static final String CREATE_TABLE_QUERY = "CREATE TABLE %s.%s (id uuid primary key, val uuid)"; + private static final String INSERT_QUERY = "INSERT INTO %s.%s (id, val) VALUES (%s, %s)"; + + private final Pair table; + + public CreateTable(RandomSource rs, State state) + { + super(rs, state); + table = state.addRandomTable(); + } + + @Override + public void doWork(State state) + { + Map> expected = state.schema; + createTable(state); + assertEquals(expected, state.getSchema()); + state.populate(); + } + + public static Map> createTable(State state) + { + Map> existingSchema = state.getSchema(); + Map> schemaToBe = state.schema; + + MapDifference> difference = Maps.difference(schemaToBe, existingSchema); + + Map> tablesToCreate = new HashMap<>(); + + for (Entry>> entry : difference.entriesDiffering().entrySet()) + { + String keyspaceName = entry.getKey(); + tablesToCreate.put(keyspaceName, new ArrayList<>()); + ValueDifference> tableNames = entry.getValue(); + for (String table : difference(new HashSet<>(tableNames.leftValue()), + new HashSet<>(tableNames.rightValue()))) + { + tablesToCreate.get(keyspaceName).add(table); + } + } + + for (Map.Entry> entry : tablesToCreate.entrySet()) + { + String keyspace = entry.getKey(); + for (String table : entry.getValue()) + { + state.executeQuery(format(CREATE_TABLE_QUERY, keyspace, table)); + populateTable(state, keyspace, table); + } + } + + return tablesToCreate; + } + + public static void populateTable(State state, String keypace, String table) + { + // create between 1 and 10 sstables per table + for (int i = 0; i < state.rs.nextInt(1, 10); i++) + { + state.executeQuery(format(INSERT_QUERY, keypace, table, randomUUID(), randomUUID())); + state.nodetool("flush", keypace, table); + } + } + + @Override + public String toString() + { + return "Create table " + format("%s.%s", table.left, table.right); + } + } + + public static class CreateMoreData extends AbstractCommand + { + private Pair tableToPopulate; + private final boolean shouldApply; + + public CreateMoreData(RandomSource rs, State state) + { + super(rs, state); + + Optional> randomTable = state.pickRandomTable(false); + shouldApply = randomTable.isPresent(); + + if (!shouldApply) + return; + + tableToPopulate = randomTable.get(); + } + + @Override + public void doWork(State state) + { + if (!shouldApply) + return; + + CreateTable.populateTable(state, tableToPopulate.left, tableToPopulate.right); + } + + @Override + public String toString() + { + if (tableToPopulate != null) + return "Create more data " + format("%s.%s", tableToPopulate.left, tableToPopulate.right); + else + return "Create more data skipped"; + } + } + + public static class DropTable extends AbstractCommand + { + private static final String DROP_TABLE_QUERY = "DROP TABLE %s.%s"; + + private Pair tableToDrop; + private final boolean shouldApply; + + public DropTable(RandomSource rs, State state) + { + super(rs, state); + Optional> randomTable = state.pickRandomTable(false); + shouldApply = randomTable.isPresent(); + + if (!shouldApply) + return; + + tableToDrop = randomTable.get(); + state.removeTable(tableToDrop.left, tableToDrop.right); + + // if we drop a table, it will make a snapshot with "dropped-" prefix + // hence it will be among snapshot as well, however we do not know its name + // in advance because it contains timestamp in its name produced by Cassandra + // hence we can not add it among "normal" snapshots, hence special "droppedSnapshots" set. + state.addDroppedSnapshot(tableToDrop.left, tableToDrop.right); + } + + @Override + public void doWork(State state) + { + if (!shouldApply) + return; + + Map> existingSchema = state.getSchema(); + Map> schemaToBe = state.schema; + + MapDifference> difference = Maps.difference(schemaToBe, existingSchema); + + for (Entry>> entry : difference.entriesDiffering().entrySet()) + { + String keyspaceName = entry.getKey(); + ValueDifference> tableNames = entry.getValue(); + + Set left = new HashSet<>(tableNames.leftValue()); + Set right = new HashSet<>(tableNames.rightValue()); + + for (String table : difference(right, left)) + executeQuery(format(DROP_TABLE_QUERY, keyspaceName, table)); + } + + assertEquals(schemaToBe, state.getSchema()); + + state.populate(); + } + + @Override + public String toString() + { + if (tableToDrop != null) + return "Drop table " + format("%s.%s", tableToDrop.left, tableToDrop.right); + else + return "Drop table skipped"; + } + } + + public static class TruncateTable extends AbstractCommand + { + private final Pair toTruncate; + + public TruncateTable(RandomSource rs, State state) + { + super(rs, state); + toTruncate = state.pickRandomTable(true).get(); // there is always in-built table to truncate + state.addTruncatedSnapshot(toTruncate.left, toTruncate.right); + } + + @Override + public void doWork(State state) + { + // create missing tables if any + createKeypace(state); + createTable(state); + + executeQuery(format("TRUNCATE TABLE %s.%s", toTruncate.left, toTruncate.right)); + assertEquals(state.truncatedSnapshots, state.getSnapshotsOfTruncatedTables()); + + state.populate(); + } + + @Override + public String toString() + { + return "Truncate table " + format("%s.%s", toTruncate.left, toTruncate.right); + } + } + + public static class TakeSnapshot extends AbstractCommand + { + private final String snapshotToTake; + + public TakeSnapshot(RandomSource rs, State state) + { + super(rs, state); + // there is always in-built table to take snapshot on + Pair randomTable = state.pickRandomTable(true).get(); + String tag = state.addSnapshot(randomTable.left, randomTable.right); + snapshotToTake = format("%s.%s.%s", randomTable.left, randomTable.right, tag); + } + + @Override + public void doWork(State state) + { + // see what snapshots are to be taken + Set existing = state.getSnapshots(); + Set toBe = state.snapshots; + + Set diff = difference(toBe, existing); + + assert diff.size() == 1 : "expecting one snapshot to take!"; + + state.takeSnapshots(diff); + assertEquals(toBe, state.getSnapshots()); + state.populate(); + } + + @Override + public String toString() + { + return "Snapshot " + snapshotToTake; + } + } + + public static class ClearSnapshot extends AbstractCommand + { + private static final String DROPPED_SNAPSHOT_PREFIX = "dropped-"; + private static final String TRUNCATED_SNAPSHOT_PREFIX = "truncated-"; + + private final Set normalDiff = new HashSet<>(); + private final Set truncatedDiff = new HashSet<>(); + private final Set droppedDiff = new HashSet<>(); + + public ClearSnapshot(RandomSource rs, State state) + { + super(rs, state); + prepare(); + } + + private void prepare() + { + List toShuffle = state.getShuffledListOfInts(3); + + for (int i = 0; i < 3; i++) + { + boolean picked = false; + switch (toShuffle.get(i)) + { + case 1: + if (!state.truncatedSnapshots.isEmpty()) + { + String randomKsTb = state.rs.pick(state.truncatedSnapshots.keySet()); + Integer numberOfTruncatedSnapshots = state.truncatedSnapshots.get(randomKsTb); + if (numberOfTruncatedSnapshots == 1) + state.truncatedSnapshots.remove(randomKsTb); + else + { + int newNumberOfTruncatedTables = state.truncatedSnapshots.get(randomKsTb) - 1; + state.truncatedSnapshots.put(randomKsTb, newNumberOfTruncatedTables); + } + picked = true; + } + break; + case 2: + if (!state.droppedSnapshots.isEmpty()) + { + state.droppedSnapshots.remove(state.rs.pick(state.droppedSnapshots)); + picked = true; + } + break; + case 3: + if (!state.snapshots.isEmpty()) + { + TestSnapshot pickedSnapshot = state.rs.pick(state.snapshots); + state.snapshots.remove(pickedSnapshot); + picked = true; + } + break; + } + if (picked) + break; + } + + Set existingNormal = state.getSnapshots(); + Set existingDropped = state.getSnapshotsOfDroppedTables(); + Map existingTruncated = state.getSnapshotsOfTruncatedTables(); + + Set normalToBe = state.snapshots; + Set droppedToBe = state.droppedSnapshots; + Map truncatedToBe = state.truncatedSnapshots; + + normalDiff.addAll(difference(existingNormal, normalToBe)); + truncatedDiff.addAll(difference(existingTruncated.keySet(), truncatedToBe.keySet())); + + Set diff = new HashSet<>(); + + for (Map.Entry existingTruncatedEntry : existingTruncated.entrySet()) + { + if (truncatedToBe.containsKey(existingTruncatedEntry.getKey())) + { + int toBeCount = truncatedToBe.get(existingTruncatedEntry.getKey()); + int existingCount = existingTruncatedEntry.getValue(); + if (toBeCount < existingCount) + { + diff.add(existingTruncatedEntry.getKey()); + } + } + } + + truncatedDiff.addAll(diff); + + droppedDiff.addAll(difference(existingDropped, droppedToBe)); + } + + @Override + public void doWork(State state) + { + clearSnapshot(state, normalDiff); + clearSnapshot(state, truncatedDiff, TRUNCATED_SNAPSHOT_PREFIX); + clearSnapshot(state, droppedDiff, DROPPED_SNAPSHOT_PREFIX); + + assertEquals(state.snapshots, state.getSnapshots()); + assertEquals(state.droppedSnapshots, state.getSnapshotsOfDroppedTables()); + + assertEquals(state.truncatedSnapshots, state.getSnapshotsOfTruncatedTables()); + + state.populate(); + } + + @Override + public String toString() + { + if (!normalDiff.isEmpty()) + return "Clear snapshot(s) " + normalDiff; + else if (!truncatedDiff.isEmpty()) + return "Clear snapshot(s) " + truncatedDiff; + else if (!droppedDiff.isEmpty()) + return "Clear snapshots(s) " + droppedDiff; + else + return "Clear snapshots"; + } + + private void clearSnapshot(State state, Set toRemove, String prefix) + { + for (String snapshot : toRemove) + { + String[] split = snapshot.split("\\."); + state.getNode().acceptsOnInstance((SerializableTriConsumer) (ks, tb, pref) -> + { + List selectedSnapshots = SnapshotManager.instance.getSnapshots(s -> s.getKeyspaceName().equals(ks) && + s.getTableName().equals(tb) && + s.getTag().contains(pref)); + if (!selectedSnapshots.isEmpty()) + { + TableSnapshot selectedSnapshotForRemoval = selectedSnapshots.get(0); + SnapshotManager.instance.clearSnapshot(s -> s.equals(selectedSnapshotForRemoval)); + } + }).accept(split[0], split[1], prefix); + } + } + + private void clearSnapshot(State state, Set snapshotsToClear) + { + for (TestSnapshot snapshot : snapshotsToClear) + { + state.getNode() + .acceptsOnInstance((SerializableTriConsumer) + (ks, tb, tag) -> SnapshotManager.instance.clearSnapshot(ks, tb, tag)) + .accept(snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag()); + } + } + } + + public static class State + { + private final Cluster.Builder builder = Cluster.build(1).withConfig(c -> c.with(Feature.NATIVE_PROTOCOL)); + private Cluster cluster; + + public Map> schema = new HashMap<>(); + public Set snapshots = new HashSet<>(); + public Set droppedSnapshots = new HashSet<>(); + public Map truncatedSnapshots = new HashMap<>(); + + public final RandomSource rs; + public final RandomnessSource randomnessSource; + + public String inBuiltKeyspace; + public String inBuiltTable; + + public State(RandomSource rs) + { + this.rs = rs; + this.randomnessSource = new JavaRandom(rs.asJdkRandom()); + start(); + populateSchema(); + populate(); + } + + public State populate() + { + this.schema = getSchema(); + this.snapshots = getSnapshots(); + this.droppedSnapshots = getSnapshotsOfDroppedTables(); + this.truncatedSnapshots = getSnapshotsOfTruncatedTables(); + return this; + } + + private void populateSchema() + { + new CreateKeyspace(rs, this).applyUnit(this); + new CreateTable(rs, this).applyUnit(this); + + this.inBuiltKeyspace = this.schema.keySet().iterator().next(); + this.inBuiltTable = this.schema.values().iterator().next().get(0); + } + + @Override + public String toString() + { + return "State{" + + "schema=" + schema + + ", snapshots=" + snapshots + + ", droppedSnapshots=" + droppedSnapshots + + ", truncatedSnapshots=" + truncatedSnapshots + + '}'; + } + + public Optional pickRandomKeyspace(boolean inBuiltIncluded) + { + if (inBuiltIncluded) + return pickRandomKeyspace(); + + Set withoutInBuilt = new HashSet<>(schema.keySet()); + withoutInBuilt.remove(inBuiltKeyspace); + + if (withoutInBuilt.isEmpty()) + return Optional.empty(); + else + return Optional.of(rs.pick(withoutInBuilt)); + } + + public Optional pickRandomKeyspace() + { + if (schema.keySet().isEmpty()) + return Optional.empty(); + else + return Optional.of(rs.pick(schema.keySet())); + } + + public Optional> pickRandomTable(boolean inBuiltIncluded) + { + Map> keyspacesWithTables = new HashMap<>(); + for (Map.Entry> entry : schema.entrySet()) + { + if (!entry.getValue().isEmpty()) + { + if (inBuiltIncluded) + { + keyspacesWithTables.put(entry.getKey(), entry.getValue()); + } + else + { + if (entry.getKey().equals(inBuiltKeyspace)) + { + List tablesInInBuiltKeyspace = new ArrayList<>(); + for (String tableInInBuiltKeyspace : entry.getValue()) + { + if (!tableInInBuiltKeyspace.equals(inBuiltTable)) + tablesInInBuiltKeyspace.add(tableInInBuiltKeyspace); + } + if (!tablesInInBuiltKeyspace.isEmpty()) + keyspacesWithTables.put(entry.getKey(), tablesInInBuiltKeyspace); + } + else + { + if (!entry.getValue().isEmpty()) + keyspacesWithTables.put(entry.getKey(), entry.getValue()); + } + } + } + } + + if (keyspacesWithTables.isEmpty()) + return Optional.empty(); + + String randomKeyspaceWithTables = rs.pick(keyspacesWithTables.keySet()); + List tables = keyspacesWithTables.get(randomKeyspaceWithTables); + String randomTable = rs.pick(tables); + + return Optional.of(Pair.create(randomKeyspaceWithTables, randomTable)); + } + + public String addRandomKeyspace() + { + String keyspace = createRandomKeyspaceIdentifier(); + addKeyspace(keyspace); + return keyspace; + } + + public Pair addRandomTable() + { + Optional randomKeyspace = pickRandomKeyspace(); + + assert randomKeyspace.isPresent(); + + String randomTableName = createRandomTableIdentifier(); + addTable(randomKeyspace.get(), randomTableName); + return Pair.create(randomKeyspace.get(), randomTableName); + } + + public Optional>> removeRandomKeyspace() + { + Optional randomKeyspace = pickRandomKeyspace(false); + + if (randomKeyspace.isEmpty()) + return Optional.empty(); + + return randomKeyspace.map(this::removeKeyspace); + } + + public void addKeyspace(String keyspaceName) + { + schema.put(keyspaceName, new ArrayList<>()); + } + + public Pair> removeKeyspace(String keyspaceName) + { + List removedTables = schema.remove(keyspaceName); + return Pair.create(keyspaceName, removedTables); + } + + public void addTable(String keyspaceName, String tableName) + { + List tables = schema.get(keyspaceName); + if (tables == null) + tables = new ArrayList<>(); + + tables.add(tableName); + schema.put(keyspaceName, tables); + } + + public void removeTable(String keyspace, String table) + { + List tables = schema.get(keyspace); + if (tables == null) + return; + + tables.remove(table); + } + + public String addSnapshot(String keyspace, String table) + { + String randomSnapshotIdentifier = createRandomSnapshotIdentifier(); + addSnapshot(new TestSnapshot(keyspace, table, randomSnapshotIdentifier)); + + return randomSnapshotIdentifier; + } + + public void addDroppedSnapshot(String keyspace, String table) + { + droppedSnapshots.add(keyspace + '.' + table); + } + + public void addTruncatedSnapshot(String keyspace, String table) + { + truncatedSnapshots.merge(keyspace + '.' + table, 1, Integer::sum); + } + + public void addSnapshot(TableSnapshot snapshot) + { + snapshots.add(new TestSnapshot(snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag())); + } + + public void destroy() + { + snapshots.clear(); + droppedSnapshots.clear(); + truncatedSnapshots.clear(); + schema.clear(); + + stop(); + } + + public List getShuffledListOfInts(int number) + { + List toShuffle = new ArrayList<>(); + + for (int i = 0; i < number; i++) + toShuffle.add(i); + + Collections.shuffle(toShuffle, rs.asJdkRandom()); + + return toShuffle; + } + + // we just need something to store in the state and whole thing is too much and not necessary + public static class TestSnapshot extends TableSnapshot implements Comparable + { + public TestSnapshot(String keyspaceName, String tableName, String tag) + { + this(keyspaceName, tableName, randomUUID(), tag, null, null, null, false); + } + + public TestSnapshot(String keyspaceName, String tableName, UUID tableId, String tag, + Instant createdAt, Instant expiresAt, Set snapshotDirs, boolean ephemeral) + { + super(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral); + } + + @Override + public long getManifestsSize() + { + return 0; + } + + @Override + public long getSchemasSize() + { + return 0; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TableSnapshot snapshot = (TableSnapshot) o; + return Objects.equals(getKeyspaceName(), snapshot.getKeyspaceName()) && + Objects.equals(getTableName(), snapshot.getTableName()) && + Objects.equals(getTag(), snapshot.getTag()); + } + + @Override + public int hashCode() + { + return Objects.hash(getKeyspaceName(), getTableName(), getTag()); + } + + @Override + public String toString() + { + return format("%s.%s.%s", getKeyspaceName(), getTableName(), getTag()); + } + + @Override + public int compareTo(TestSnapshot o) + { + return toString().compareTo(o.toString()); + } + } + + + private String createRandomKeyspaceIdentifier() + { + return trimIfNecessary("keyspace_" + getRandomString()); + } + + private String createRandomSnapshotIdentifier() + { + return trimIfNecessary("snapshot_" + getRandomString()); + } + + private String createRandomTableIdentifier() + { + return trimIfNecessary("table_" + getRandomString()); + } + + private String getRandomString() + { + return LocalizeString.toLowerCaseLocalized(Generators.regexWord(SourceDSL.integers().between(10, 50)).generate(randomnessSource)); + } + + private String trimIfNecessary(String maybeTooLongString) + { + if (maybeTooLongString.length() <= 48) + return maybeTooLongString; + + return maybeTooLongString.substring(0, 48); + } + + // taken from SUT + + public Map> getSchema() + { + IIsolatedExecutor.SerializableCallable>> callable = () -> { + Keyspaces keyspaces = Schema.instance.distributedKeyspaces(); + + Map> keyspacesWithTables = new HashMap<>(); + + for (KeyspaceMetadata ksm : keyspaces) + { + if (ksm.name.startsWith("system_")) + continue; + + List tables = new ArrayList<>(); + for (TableMetadata tmd : ksm.tables) + tables.add(tmd.name); + + keyspacesWithTables.put(ksm.name, tables); + } + + return keyspacesWithTables; + }; + + return getNode().callOnInstance(callable); + } + + public Set getKeyspaces() + { + return getSchema().keySet(); + } + + public Set getSnapshotsOfDroppedTables() + { + SerializableFunction> callable = (p) -> { + Set snapshots = new HashSet<>(); + for (TableSnapshot snapshot : SnapshotManager.instance.getSnapshots(snapshot -> snapshot.getTag().contains(p))) + snapshots.add(format("%s.%s", snapshot.getKeyspaceName(), snapshot.getTableName())); + + return snapshots; + }; + + return getNode().applyOnInstance(callable, "dropped-"); + } + + public Map getSnapshotsOfTruncatedTables() + { + SerializableFunction> callable = (p) -> { + Map snapshots = new HashMap<>(); + for (TableSnapshot snapshot : SnapshotManager.instance.getSnapshots(snapshot -> snapshot.getTag().contains(p))) + { + String ksTb = format("%s.%s", snapshot.getKeyspaceName(), snapshot.getTableName()); + snapshots.merge(ksTb, 1, Integer::sum); + } + + return snapshots; + }; + + return getNode().applyOnInstance(callable, "truncated-"); + } + + // these are not "dropped" nor "truncated" + public Set getSnapshots() + { + Set existingSnapshots = new HashSet<>(); + + IIsolatedExecutor.SerializableCallable> callable = () -> { + List snapshots = new ArrayList<>(); + for (TableSnapshot snapshot : SnapshotManager.instance.getSnapshots(p -> { + String tag = p.getTag(); + return !tag.contains("dropped-") && !tag.contains("truncated-"); + })) + { + snapshots.add(format("%s.%s.%s", snapshot.getKeyspaceName(), snapshot.getTableName(), snapshot.getTag())); + } + + return snapshots; + }; + + for (String tableSnapshot : getNode().callOnInstance(callable)) + { + String[] components = tableSnapshot.split("\\."); + existingSnapshots.add(new TestSnapshot(components[0], + components[1], + components[2])); + } + + return existingSnapshots; + } + + public void takeSnapshots(Set snapshotsToTake) + { + for (TestSnapshot toTake : snapshotsToTake) + getNode().acceptsOnInstance((SerializableTriConsumer) (tag, ks, tb) -> { + try + { + SnapshotManager.instance.takeSnapshot(tag, Map.of(), ks + '.' + tb); + } + catch (IOException ex) + { + throw new RuntimeException(ex); + } + }) + .accept(toTake.getTag(), toTake.getKeyspaceName(), toTake.getTableName()); + } + + public IInvokableInstance getNode() + { + if (cluster == null) + throw new RuntimeException("not started yet"); + + return cluster.get(1); + } + + public void executeQuery(String query) + { + getNode().executeInternal(query); + } + + public State start() + { + if (cluster != null) + return null; + + try + { + cluster = builder.start(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + + schema = getSchema(); + + return this; + } + + public void stop() + { + if (cluster != null) + { + try + { + cluster.close(); + } + catch (Throwable t) + { + // ignore + } + } + } + + public NodeToolResult nodetool(String... nodetoolArgs) + { + return getNode().nodetoolResult(nodetoolArgs); + } + } + + public static abstract class AbstractCommand implements StateOnlyCommand + { + protected final RandomSource rs; + protected final State state; + + public AbstractCommand(RandomSource rs, State state) + { + this.rs = rs; + this.state = state; + } + + public void executeQuery(String query) + { + state.getNode().executeInternal(query); + } + + public IInvokableInstance getNode() + { + return state.getNode(); + } + + @Override + public void applyUnit(State state) + { + Uninterruptibles.sleepUninterruptibly(Generators.TINY_TIME_SPAN_NANOS.generate(state.randomnessSource), TimeUnit.NANOSECONDS); + doWork(state); + } + + public abstract void doWork(State state); + } +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/AbstractSnapshotManagerBase.java b/test/microbench/org/apache/cassandra/test/microbench/AbstractSnapshotManagerBase.java new file mode 100644 index 0000000000..0f5bb8808d --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/AbstractSnapshotManagerBase.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import org.apache.commons.lang3.tuple.Pair; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.commitlog.CommitLog; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.tcm.ClusterMetadataService; + +import static org.apache.cassandra.service.snapshot.SnapshotOptions.userSnapshot; +import static org.junit.Assert.assertEquals; + +public abstract class AbstractSnapshotManagerBase +{ + static final String KEYSPACE = "KEYSPACE"; + + private final Random random = new Random(); + + static int NUM_SSTABLES = 10; + static int NUM_KEYSPACES = 5; + static int NUM_TABLES_PER_KEYSPACE = 10; + static int NUM_SNAPSHOTS_PER_TABLE = 10; + + public void teardown() + { + SnapshotManager.instance.clearAllSnapshots(); + CQLTester.tearDownClass(); + CommitLog.instance.stopUnsafe(true); + // TODO CASSANDRA-20119 + ClusterMetadataService.instance().log().close(); + CQLTester.cleanup(); + } + + public void setup(boolean takeSnapshotAfterTablePopulation) + { + CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(1000); + CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(60); + + DatabaseDescriptor.daemonInitialization(() -> { + Config config = DatabaseDescriptor.loadConfig(); + config.dump_heap_on_uncaught_exception = false; + return config; + }); + + SchemaLoader.prepareServer(); + SnapshotManager.instance.start(true); + + populateTables(takeSnapshotAfterTablePopulation); + } + + public void populateTables(boolean takeSnapshotAfterTablePopulation) + { + for (int i = 0; i < NUM_KEYSPACES; i++) + { + String keyspaceName = KEYSPACE + '_' + i; + + // Create Schema + TableMetadata[] tables = new TableMetadata[NUM_TABLES_PER_KEYSPACE]; + for (int j = 0; j < NUM_TABLES_PER_KEYSPACE; j++) + { + tables[j] = SchemaLoader.standardCFMD(keyspaceName, String.format("table%d", i + '_' + j)).build(); + } + + SchemaLoader.createKeyspace(keyspaceName, + KeyspaceParams.simple(1), + tables); + + for (int j = 0; j < NUM_TABLES_PER_KEYSPACE; j++) + { + String tableName = String.format("table%d", i + '_' + j); + + ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName); + cfs.disableAutoCompaction(); + for (int k = 0; k < NUM_SSTABLES; k++) + { + new RowUpdateBuilder(cfs.metadata(), 0, "key1") + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); + Util.flush(cfs); + } + if (takeSnapshotAfterTablePopulation) + { + for (int snapId = 0; snapId < NUM_SNAPSHOTS_PER_TABLE; snapId++) + SnapshotManager.instance.takeSnapshot(userSnapshot(String.format("snap_%d_%d_%d", i, j, snapId), cfs.getKeyspaceTableName())); + } + } + } + + if (takeSnapshotAfterTablePopulation) + { + List snapshots = SnapshotManager.instance.getSnapshots(t -> true); + assertEquals(NUM_KEYSPACES * NUM_TABLES_PER_KEYSPACE * NUM_SNAPSHOTS_PER_TABLE, snapshots.size()); + } + } + + public String getRandomKeyspaceTable() + { + List keyspaces = new ArrayList<>(); + Keyspace.nonSystem().forEach(keyspaces::add); + Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size())); + List cfss = new ArrayList<>(randomKeyspace.getColumnFamilyStores()); + ColumnFamilyStore cfs = cfss.get(random.nextInt(cfss.size())); + + return cfs.getKeyspaceTableName(); + } + + public String getRandomKeyspace() + { + List keyspaces = new ArrayList<>(); + Keyspace.nonSystem().forEach(keyspaces::add); + Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size())); + return randomKeyspace.getName(); + } + + public Pair getRandomKeyspaceTablePair() + { + List keyspaces = new ArrayList<>(); + Keyspace.nonSystem().forEach(keyspaces::add); + Keyspace randomKeyspace = keyspaces.get(random.nextInt(keyspaces.size())); + List cfss = new ArrayList<>(randomKeyspace.getColumnFamilyStores()); + ColumnFamilyStore cfs = cfss.get(random.nextInt(cfss.size())); + + return Pair.of(cfs.getKeyspaceName(), cfs.getTableName()); + } + + + public void takeSnapshotOnRandomTable() + { + SnapshotManager.instance.takeSnapshot(userSnapshot("snap_" + UUID.randomUUID(), getRandomKeyspaceTable())); + } +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java index 8d7e800755..edbe249b2e 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java @@ -30,6 +30,7 @@ import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.openjdk.jmh.annotations.*; @BenchmarkMode(Mode.AverageTime) @@ -78,8 +79,7 @@ public class CompactionBench extends CQLTester cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED); - cfs.snapshot("originals"); - + SnapshotManager.instance.takeSnapshot("originals", cfs.getKeyspaceTableName()); snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles(); } diff --git a/test/microbench/org/apache/cassandra/test/microbench/SnapshotListingBench.java b/test/microbench/org/apache/cassandra/test/microbench/SnapshotListingBench.java new file mode 100644 index 0000000000..92d687159a --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/SnapshotListingBench.java @@ -0,0 +1,63 @@ +package org.apache.cassandra.test.microbench; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(value = 1) +@State(Scope.Benchmark) +public class SnapshotListingBench extends AbstractSnapshotManagerBase +{ + @Setup(Level.Trial) + public void setup() + { + setup(true); + } + + @TearDown(Level.Trial) + public void teardown() + { + super.teardown(); + } + + @Benchmark + @BenchmarkMode(Mode.All) + @Warmup(iterations = 0) + @Threads(4) + @Measurement(iterations = 1, time = 60) + public void listingTest() + { + SnapshotManager.instance.listSnapshots(null); + } +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/SnapshotTakingBench.java b/test/microbench/org/apache/cassandra/test/microbench/SnapshotTakingBench.java new file mode 100644 index 0000000000..4631905eb0 --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/SnapshotTakingBench.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench; + +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(value = 1) +@State(Scope.Benchmark) +public class SnapshotTakingBench extends AbstractSnapshotManagerBase +{ + + @Setup(Level.Trial) + public void setup() + { + setup(false); + } + + @TearDown(Level.Trial) + public void teardown() + { + super.teardown(); + } + + @Benchmark + @BenchmarkMode(Mode.All) + @Warmup(iterations = 0) + @Threads(4) + @Measurement(iterations = 1, time = 60) + public void takingSnapshotTest() + { + takeSnapshotOnRandomTable(); + } +} diff --git a/test/microbench/org/apache/cassandra/test/microbench/SnapshotTrueSizeBench.java b/test/microbench/org/apache/cassandra/test/microbench/SnapshotTrueSizeBench.java new file mode 100644 index 0000000000..b8d9ffe23c --- /dev/null +++ b/test/microbench/org/apache/cassandra/test/microbench/SnapshotTrueSizeBench.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.test.microbench; + +import java.util.concurrent.TimeUnit; + +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(value = 1) +@State(Scope.Benchmark) +public class SnapshotTrueSizeBench extends AbstractSnapshotManagerBase +{ + + @Setup(Level.Trial) + public void setup() + { + setup(true); + } + + @TearDown(Level.Trial) + public void teardown() + { + super.teardown(); + } + + @Benchmark + @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) + @Warmup(iterations = 0) + @Threads(16) + @Measurement(iterations = 1, time = 60) + public void getTrueSizeOfSnapshots() + { + SnapshotManager.instance.getTrueSnapshotSize(); + } +} diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java index be883ab047..590f7e356f 100644 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@ -35,8 +35,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; @@ -133,6 +135,8 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.pager.PagingState; +import org.apache.cassandra.service.snapshot.SnapshotLoader; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.streaming.StreamState; import org.apache.cassandra.tcm.ClusterMetadata; @@ -147,7 +151,6 @@ import org.apache.cassandra.utils.CounterId; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.FilterFactory; import org.apache.cassandra.utils.OutputHandler; -import org.apache.cassandra.utils.Throwables; import org.awaitility.Awaitility; import org.hamcrest.Matcher; import org.mockito.Mockito; @@ -1170,23 +1173,6 @@ public class Util .collect(Collectors.toSet()); } - public static Set getSnapshots(String ks, String tableName, String snapshotTag) - { - try - { - return Keyspace.open(ks) - .getColumnFamilyStore(tableName) - .getSnapshotSSTableReaders(snapshotTag) - .stream() - .map(sstr -> sstr.descriptor) - .collect(Collectors.toSet()); - } - catch (IOException e) - { - throw Throwables.unchecked(e); - } - } - public static Set getBackups(String ks, String tableName) { return Keyspace.open(ks) @@ -1283,4 +1269,16 @@ public class Util { return new UnsupportedOperationException("Test must be implemented for sstable format " + DatabaseDescriptor.getSelectedSSTableFormat().getClass().getName()); } + + public static Map listSnapshots(ColumnFamilyStore cfs) + { + Set snapshots = new SnapshotLoader(cfs.getDirectories()).loadSnapshots(); + Map tagSnapshotsMap = new HashMap<>(); + + for (TableSnapshot snapshot : snapshots) + tagSnapshotsMap.put(snapshot.getTag(), snapshot); + + return tagSnapshotsMap; + } + } diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java index 54171f9a08..2f5857ae97 100644 --- a/test/unit/org/apache/cassandra/cql3/CQLTester.java +++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java @@ -172,6 +172,7 @@ import org.apache.cassandra.serializers.TypeSerializer; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.transport.Event; import org.apache.cassandra.transport.Message; @@ -445,6 +446,7 @@ public abstract class CQLTester DatabaseDescriptor.setRowCacheSizeInMiB(ROW_CACHE_SIZE_IN_MIB); StorageService.instance.registerMBeans(); StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); + SnapshotManager.instance.registerMBean(); } @AfterClass diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/AutoSnapshotTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/AutoSnapshotTest.java index eedcdb0a9a..bcb0be6440 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/operations/AutoSnapshotTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/operations/AutoSnapshotTest.java @@ -30,21 +30,28 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.assertj.core.api.Condition; import static java.util.concurrent.TimeUnit.SECONDS; import static java.lang.String.format; -import static org.apache.cassandra.db.ColumnFamilyStore.SNAPSHOT_DROP_PREFIX; import static org.assertj.core.api.Assertions.assertThat; @RunWith(Parameterized.class) public class AutoSnapshotTest extends CQLTester { + static + { + CassandraRelevantProperties.SNAPSHOT_MIN_ALLOWED_TTL_SECONDS.setInt(1); + } + static int TTL_SECS = 1; public static Boolean enabledBefore; @@ -97,7 +104,7 @@ public class AutoSnapshotTest extends CQLTester createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))"); // Check there are no snapshots ColumnFamilyStore tableDir = getCurrentColumnFamilyStore(); - assertThat(tableDir.listSnapshots()).isEmpty(); + assertThat(Util.listSnapshots(tableDir)).isEmpty(); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1); @@ -106,7 +113,7 @@ public class AutoSnapshotTest extends CQLTester execute("DROP TABLE %s"); - verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableDir, currentTable()); + verifyAutoSnapshot(SnapshotType.DROP.label, tableDir, currentTable()); } @Test @@ -115,7 +122,7 @@ public class AutoSnapshotTest extends CQLTester createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY(a, b))"); // Check there are no snapshots ColumnFamilyStore tableDir = getCurrentColumnFamilyStore(); - assertThat(tableDir.listSnapshots()).isEmpty(); + assertThat(Util.listSnapshots(tableDir)).isEmpty(); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 0, 0); execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 0, 1, 1); @@ -124,7 +131,7 @@ public class AutoSnapshotTest extends CQLTester execute("DROP TABLE %s"); - verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableDir, currentTable()); + verifyAutoSnapshot(SnapshotType.DROP.label, tableDir, currentTable()); } @Test @@ -136,13 +143,13 @@ public class AutoSnapshotTest extends CQLTester flush(); // Check no snapshots - assertThat(tableA.listSnapshots()).isEmpty(); - assertThat(tableB.listSnapshots()).isEmpty(); + assertThat(Util.listSnapshots(tableA)).isEmpty(); + assertThat(Util.listSnapshots(tableB)).isEmpty(); // Drop keyspace, should have snapshot for table A and B execute(format("DROP KEYSPACE %s", keyspace())); - verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableA, tableA.name); - verifyAutoSnapshot(SNAPSHOT_DROP_PREFIX, tableB, tableB.name); + verifyAutoSnapshot(SnapshotType.DROP.label, tableA, tableA.name); + verifyAutoSnapshot(SnapshotType.DROP.label, tableB, tableB.name); } private ColumnFamilyStore createAndPopulateTable() throws Throwable @@ -163,11 +170,11 @@ public class AutoSnapshotTest extends CQLTester */ private void verifyAutoSnapshot(String snapshotPrefix, ColumnFamilyStore tableDir, String expectedTableName) { - Map snapshots = tableDir.listSnapshots(); + Map snapshots = Util.listSnapshots(tableDir); if (autoSnapshotEnabled) { assertThat(snapshots).hasSize(1); - assertThat(snapshots).hasKeySatisfying(new Condition<>(k -> k.startsWith(snapshotPrefix), "is dropped snapshot")); + assertThat(snapshots).hasKeySatisfying(new Condition<>(k -> k.contains(snapshotPrefix), "is dropped snapshot")); TableSnapshot snapshot = snapshots.values().iterator().next(); assertThat(snapshot.getTableName()).isEqualTo(expectedTableName); if (autoSnapshotTTl == null) diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 4de0186fea..ba08437cd6 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -1,21 +1,21 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.cassandra.db; import java.io.IOException; @@ -23,18 +23,19 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterators; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -44,7 +45,9 @@ import com.googlecode.concurrenttrees.common.Iterables; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; +import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.statements.schema.IndexTarget; import org.apache.cassandra.db.ColumnFamilyStore.FlushReason; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.filter.ColumnFilter; @@ -59,7 +62,9 @@ import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.index.Index; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; @@ -71,11 +76,17 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.ClearableHistogram; import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.Indexes; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.SnapshotManifest; +import org.apache.cassandra.service.snapshot.SnapshotType; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -83,6 +94,10 @@ import org.apache.cassandra.utils.WrappedRunnable; import org.apache.cassandra.utils.concurrent.OpOrder.Barrier; import org.apache.cassandra.utils.concurrent.OpOrder.Group; +import static org.apache.cassandra.cql3.statements.schema.IndexTarget.Type.VALUES; +import static org.apache.cassandra.schema.IndexMetadata.Kind.COMPOSITES; +import static org.apache.cassandra.schema.IndexMetadata.Kind.CUSTOM; +import static org.apache.cassandra.schema.IndexMetadata.fromIndexTargets; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -121,14 +136,27 @@ public class ColumnFamilyStoreTest @Before public void truncateCFS() { - Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); - Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlocking(); - Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1).truncateBlocking(); - Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlocking(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1).truncateBlockingWithoutSnapshot(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD2).truncateBlockingWithoutSnapshot(); + Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1).truncateBlockingWithoutSnapshot(); + Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1).truncateBlockingWithoutSnapshot(); + Keyspace.open(KEYSPACE3).getColumnFamilyStore(CF_SPEC_RETRY1).truncateBlockingWithoutSnapshot(); + + SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_STANDARD1); + SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_STANDARD2); + SnapshotManager.instance.clearAllSnapshots(KEYSPACE1, CF_INDEX1); + SnapshotManager.instance.clearAllSnapshots(KEYSPACE2, CF_STANDARD1); + SnapshotManager.instance.clearAllSnapshots(KEYSPACE3, CF_SPEC_RETRY1); + } + + @After + public void afterTest() + { + SnapshotManager.instance.clearAllSnapshots(); } @Test - public void testMemtableTimestamp() throws Throwable + public void testMemtableTimestamp() { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); assertEquals(Memtable.NO_MIN_TIMESTAMP, fakeMemTableWithMinTS(cfs, EncodingStats.NO_STATS.minTimestamp).getMinTimestamp()); @@ -142,20 +170,20 @@ public class ColumnFamilyStoreTest ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); new RowUpdateBuilder(cfs.metadata(), 0, "key1") - .clustering("Column1") - .add("val", "asdf") - .build() - .applyUnsafe(); + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); Util.flush(cfs); new RowUpdateBuilder(cfs.metadata(), 1, "key1") - .clustering("Column1") - .add("val", "asdf") - .build() - .applyUnsafe(); + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); Util.flush(cfs); - ((ClearableHistogram)cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts + ((ClearableHistogram) cfs.metric.sstablesPerReadHistogram.cf).clear(); // resets counts Util.getAll(Util.cmd(cfs, "key1").includeRow("c1").build()); assertEquals(1, cfs.metric.sstablesPerReadHistogram.cf.getCount()); } @@ -165,7 +193,7 @@ public class ColumnFamilyStoreTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); - keyspace.getColumnFamilyStores().forEach(ColumnFamilyStore::truncateBlocking); + keyspace.getColumnFamilyStores().forEach(ColumnFamilyStore::truncateBlockingWithoutSnapshot); List rms = new LinkedList<>(); rms.add(new RowUpdateBuilder(cfs.metadata(), 0, "key1") @@ -195,7 +223,7 @@ public class ColumnFamilyStoreTest { Row toCheck = Util.getOnlyRowUnfiltered(Util.cmd(cfs, "key1").build()); Iterator> iter = toCheck.cells().iterator(); - assert(Iterators.size(iter) == 0); + assert (Iterators.size(iter) == 0); } }; @@ -265,42 +293,34 @@ public class ColumnFamilyStoreTest { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1); - //cleanup any previous test gargbage - cfs.clearSnapshot(""); - int numRows = 1000; - long[] colValues = new long [numRows * 2]; // each row has two columns - for (int i = 0; i < colValues.length; i+=2) + long[] colValues = new long[numRows * 2]; // each row has two columns + for (int i = 0; i < colValues.length; i += 2) { colValues[i] = (i % 4 == 0 ? 1L : 2L); // index column - colValues[i+1] = 3L; //other column + colValues[i + 1] = 3L; //other column } ScrubTest.fillIndexCF(cfs, false, colValues); - cfs.snapshot("nonEphemeralSnapshot", null, false, false); - cfs.snapshot("ephemeralSnapshot", null, true, false); + SnapshotManager.instance.takeSnapshot(SnapshotOptions.systemSnapshot("nonEphemeralSnapshot", SnapshotType.MISC, cfs.getKeyspaceTableName()).build()); + SnapshotManager.instance.takeSnapshot(SnapshotOptions.systemSnapshot("ephemeralSnapshot", SnapshotType.REPAIR, cfs.getKeyspaceTableName()).ephemeral().build()); - Map snapshotDetails = cfs.listSnapshots(); - assertEquals(2, snapshotDetails.size()); - assertTrue(snapshotDetails.containsKey("ephemeralSnapshot")); - assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot")); + assertTrue(SnapshotManager.instance.exists(p -> p.getTag().endsWith("ephemeralSnapshot"))); + assertTrue(SnapshotManager.instance.exists(p -> p.getTag().endsWith("nonEphemeralSnapshot"))); - ColumnFamilyStore.clearEphemeralSnapshots(cfs.getDirectories()); + SnapshotManager.instance.clearEphemeralSnapshots(); - snapshotDetails = cfs.listSnapshots(); - assertEquals(1, snapshotDetails.size()); - assertTrue(snapshotDetails.containsKey("nonEphemeralSnapshot")); + List snapshots = SnapshotManager.instance.getSnapshots(p -> true); + assertEquals(1, snapshots.size()); - //test cleanup - cfs.clearSnapshot(""); + + assertTrue(snapshots.get(0).getTag().endsWith("nonEphemeralSnapshot")); } @Test public void testSnapshotSize() throws IOException { - // cleanup any previous test gargbage ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); - cfs.clearSnapshot(""); // Add row new RowUpdateBuilder(cfs.metadata(), 0, "key1") @@ -311,10 +331,10 @@ public class ColumnFamilyStoreTest Util.flush(cfs); // snapshot - cfs.snapshot("basic", null, false, false); + SnapshotManager.instance.takeSnapshot("basic", cfs.getKeyspaceTableName()); // check snapshot was created - Map snapshotDetails = cfs.listSnapshots(); + Map snapshotDetails = Util.listSnapshots(cfs); assertThat(snapshotDetails).hasSize(1); assertThat(snapshotDetails).containsKey("basic"); @@ -329,13 +349,121 @@ public class ColumnFamilyStoreTest // sizeOnDisk > trueSize because trueSize does not include manifest.json // Check that truesize now is > 0 - snapshotDetails = cfs.listSnapshots(); + snapshotDetails = Util.listSnapshots(cfs); details = snapshotDetails.get("basic"); assertThat(details.computeSizeOnDiskBytes()).isEqualTo(details.computeTrueSizeBytes()); } @Test - public void testBackupAfterFlush() throws Throwable + public void testTrueSnapshotSizeWithLegacyIndex() throws Throwable + { + testTrueSnapshotSizeInternal("snapshot_sizes_legacy", "true_snapshot_size_with_legacy_index", false); + } + + @Test + public void testTrueSnapshotSizeWithSaiIndex() throws Throwable + { + testTrueSnapshotSizeInternal("snapshot_sizes_sai","true_snapshot_size_with_sai_index", true); + } + + private void testTrueSnapshotSizeInternal(String keyspace, String table, boolean forSai) throws Throwable + { + TableMetadata tableMetadata = SchemaLoader.standardCFMD(keyspace, table).partitioner(Murmur3Partitioner.instance).build(); + SchemaLoader.createKeyspace(keyspace, KeyspaceParams.simple(1), tableMetadata); + + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); + + new RowUpdateBuilder(cfs.metadata(), 0, "key1") + .clustering("Column1") + .add("val", "value1") + .build() + .applyUnsafe(); + Util.flush(cfs); + + assertThat(cfs.trueSnapshotsSize()).isZero(); + + SnapshotManager.instance.takeSnapshot("snapshot_without_index", cfs.getKeyspaceTableName()); + + long firstSnapshotsSize = cfs.trueSnapshotsSize(); + Map listedSnapshots = Util.listSnapshots(cfs); + assertThat(firstSnapshotsSize).isPositive(); + assertThat(listedSnapshots.size()).isEqualTo(1); + assertThat(listedSnapshots.get("snapshot_without_index")).isNotNull(); + long withoutIndexSize = listedSnapshots.get("snapshot_without_index").computeSizeOnDiskBytes(); + long withoutIndexTrueSize = listedSnapshots.get("snapshot_without_index").computeTrueSizeBytes(); + + assertThat(withoutIndexSize).isGreaterThan(withoutIndexTrueSize); + assertEquals(firstSnapshotsSize, withoutIndexTrueSize); + + // add index, trueSnapshotSize should reflect that + ColumnIdentifier col = ColumnIdentifier.getInterned("val", true); + + IndexMetadata indexMetadata; + if (forSai) + indexMetadata = fromIndexTargets(List.of(new IndexTarget(col, VALUES)), "idx", CUSTOM, Map.of("class_name", "sai")); + else + indexMetadata = fromIndexTargets(List.of(new IndexTarget(col, VALUES)), "idx", COMPOSITES, Map.of()); + + TableMetadata tableMetadataWithIndex = SchemaLoader.standardCFMD(keyspace, table) + .partitioner(Murmur3Partitioner.instance) + .id(tableMetadata.id) + .indexes(Indexes.of(indexMetadata)) + .build(); + + SchemaTestUtil.announceTableUpdate(tableMetadataWithIndex); + + rebuildIndices(cfs); + + SnapshotManager.instance.takeSnapshot("snapshot_with_index", new HashMap<>(), cfs.getKeyspaceTableName()); + + long secondSnapshotSize = cfs.trueSnapshotsSize(); + Map secondListedSnapshots = Util.listSnapshots(cfs); + assertThat(secondSnapshotSize).isPositive(); + assertThat(secondSnapshotSize).isGreaterThan(firstSnapshotsSize); + + assertThat(secondListedSnapshots.size()).isEqualTo(2); + assertThat(secondListedSnapshots.get("snapshot_with_index")).isNotNull(); + assertThat(secondListedSnapshots.get("snapshot_without_index")).isNotNull(); + + long withIndexSize = secondListedSnapshots.get("snapshot_with_index").computeSizeOnDiskBytes(); + long withIndexTrueSize = secondListedSnapshots.get("snapshot_with_index").computeTrueSizeBytes(); + + assertThat(withIndexSize).isGreaterThan(withIndexTrueSize); + assertEquals(secondSnapshotSize, withIndexTrueSize + withoutIndexTrueSize); + + // taking another one is basically a copy of the previous + SnapshotManager.instance.takeSnapshot("another_snapshot_with_index", new HashMap<>(), cfs.getKeyspaceTableName()); + + long thirdSnapshotSize = cfs.trueSnapshotsSize(); + Map thirdListedSnapshots = Util.listSnapshots(cfs); + assertThat(thirdSnapshotSize).isPositive(); + assertThat(thirdSnapshotSize).isGreaterThan(secondSnapshotSize); + + long anotherWithIndexSize = thirdListedSnapshots.get("another_snapshot_with_index").computeSizeOnDiskBytes(); + long anotherWithIndexTrueSize = thirdListedSnapshots.get("another_snapshot_with_index").computeTrueSizeBytes(); + + assertEquals(withIndexSize, anotherWithIndexSize); + assertEquals(withIndexTrueSize, anotherWithIndexTrueSize); + } + + private void rebuildIndices(ColumnFamilyStore cfs) + { + // be sure we build that index + for (Index index : cfs.indexManager.listIndexes()) + { + try + { + cfs.indexManager.buildIndex(index).get(); + } + catch (Throwable t) + { + throw new RuntimeException("Unable to build index", t); + } + } + } + + @Test + public void testBackupAfterFlush() { ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1); new RowUpdateBuilder(cfs.metadata(), 0, ByteBufferUtil.bytes("key1")).clustering("Column1").add("val", "asdf").build().applyUnsafe(); @@ -381,8 +509,8 @@ public class ColumnFamilyStoreTest // remember, latencies are only an estimate - off by up to 20% by the 1.2 factor between buckets. assertThat(cfs.metric.coordinatorReadLatency.getCount()).isEqualTo(count); assertThat(cfs.metric.coordinatorReadLatency.getSnapshot().getValue(0.5)) - .isBetween((double) TimeUnit.MILLISECONDS.toMicros(5839), - (double) TimeUnit.MILLISECONDS.toMicros(5840)); + .isBetween((double) TimeUnit.MILLISECONDS.toMicros(5839), + (double) TimeUnit.MILLISECONDS.toMicros(5840)); // Sanity check the metrics - 75th percentileof linear 0-10000ms assertThat(cfs.metric.coordinatorWriteLatency.getCount()).isEqualTo(count); assertThat(cfs.metric.coordinatorWriteLatency.getSnapshot().getValue(0.75)) @@ -555,7 +683,7 @@ public class ColumnFamilyStoreTest { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_INDEX1); - cfs.truncateBlocking(); + cfs.truncateBlockingWithoutSnapshot(); UpdateBuilder builder = UpdateBuilder.create(cfs.metadata.get(), "key") .newRow() @@ -565,7 +693,7 @@ public class ColumnFamilyStoreTest Util.flush(cfs); String snapshotName = "newSnapshot"; - cfs.snapshotWithoutMemtable(snapshotName); + SnapshotManager.instance.takeSnapshot(snapshotName, Map.of(SnapshotOptions.SKIP_FLUSH, "true"), cfs.getKeyspaceTableName()); File snapshotManifestFile = cfs.getDirectories().getSnapshotManifestFile(snapshotName); SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(snapshotManifestFile); @@ -595,17 +723,18 @@ public class ColumnFamilyStoreTest writeData(cfs); } - TableSnapshot snapshot = cfs.snapshot("basic"); - + SnapshotManager.instance.takeSnapshot("basic", cfs.getKeyspaceTableName()); + TableSnapshot snapshot = SnapshotManager.instance.getSnapshot(cfs.metadata.keyspace, cfs.metadata.name, "basic").get(); + assertNotNull(snapshot); assertThat(snapshot.exists()).isTrue(); - assertThat(cfs.listSnapshots().containsKey("basic")).isTrue(); - assertThat(cfs.listSnapshots().get("basic")).isEqualTo(snapshot); + assertThat(Util.listSnapshots(cfs).containsKey("basic")).isTrue(); + assertThat(Util.listSnapshots(cfs).get("basic")).isEqualTo(snapshot); snapshot.getDirectories().forEach(FileUtils::deleteRecursive); assertThat(snapshot.exists()).isFalse(); - assertFalse(cfs.listSnapshots().containsKey("basic")); + assertFalse(Util.listSnapshots(cfs).containsKey("basic")); } private void writeData(ColumnFamilyStore cfs) @@ -623,7 +752,8 @@ public class ColumnFamilyStoreTest } @Test - public void testSnapshotCreationAndDeleteEmptyTable() { + public void testSnapshotCreationAndDeleteEmptyTable() + { createSnapshotAndDelete(KEYSPACE1, CF_INDEX1, false); createSnapshotAndDelete(KEYSPACE1, CF_STANDARD1, false); createSnapshotAndDelete(KEYSPACE1, CF_STANDARD2, false); @@ -632,7 +762,8 @@ public class ColumnFamilyStoreTest } @Test - public void testSnapshotCreationAndDeletePopulatedTable() { + public void testSnapshotCreationAndDeletePopulatedTable() + { createSnapshotAndDelete(KEYSPACE1, CF_INDEX1, true); createSnapshotAndDelete(KEYSPACE1, CF_STANDARD1, true); createSnapshotAndDelete(KEYSPACE1, CF_STANDARD2, true); @@ -651,8 +782,8 @@ public class ColumnFamilyStoreTest String keyspace = path.getParent().getFileName().toString(); String table = path.getFileName().toString().split("-")[0]; - Assert.assertEquals(cfs.getTableName(), table); - Assert.assertEquals(KEYSPACE1, keyspace); + assertEquals(cfs.getTableName(), table); + assertEquals(KEYSPACE1, keyspace); } @Test @@ -685,15 +816,15 @@ public class ColumnFamilyStoreTest } @VisibleForTesting - public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot) throws IOException + public static long getSnapshotManifestAndSchemaFileSizes(TableSnapshot snapshot) { - Optional schemaFile = snapshot.getSchemaFile(); - Optional manifestFile = snapshot.getManifestFile(); - long schemaAndManifestFileSizes = 0; - schemaAndManifestFileSizes += schemaFile.isPresent() ? schemaFile.get().length() : 0; - schemaAndManifestFileSizes += manifestFile.isPresent() ? manifestFile.get().length() : 0; + for (File schemaFile : snapshot.getSchemaFiles()) + schemaAndManifestFileSizes += schemaFile.length(); + + for (File manifestFile : snapshot.getManifestFiles()) + schemaAndManifestFileSizes += manifestFile.length(); return schemaAndManifestFileSizes; } @@ -827,7 +958,7 @@ public class ColumnFamilyStoreTest @Override public UnfilteredPartitionIterator - partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener) + partitionIterator(ColumnFilter columnFilter, DataRange dataRange, SSTableReadsListener listener) { return null; } diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index 1794ae8843..b6d4126fc9 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -19,8 +19,11 @@ package org.apache.cassandra.db; import java.io.IOException; import java.nio.file.FileStore; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.FileStoreAttributeView; import java.time.Instant; @@ -93,6 +96,7 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspaceTables; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.DefaultFSErrorHandler; +import org.apache.cassandra.service.snapshot.SnapshotLoader; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -326,12 +330,24 @@ public class DirectoriesTest } } + private Map listSnapshots(Directories directories) + { + Set snapshots = new SnapshotLoader(directories).loadSnapshots(); + Map tagSnapshotsMap = new HashMap<>(); + + for (TableSnapshot snapshot : snapshots) + tagSnapshotsMap.put(snapshot.getTag(), snapshot); + + return tagSnapshotsMap; + } + @Test public void testListSnapshots() throws Exception { + // Initial state TableMetadata fakeTable = createFakeTable(TABLE_NAME); Directories directories = new Directories(fakeTable, toDataDirectories(tempDataDir)); - assertThat(directories.listSnapshots()).isEmpty(); + assertThat(listSnapshots(directories)).isEmpty(); // Create snapshot with and without manifest FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false); @@ -340,7 +356,7 @@ public class DirectoriesTest FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true); // Both snapshots should be present - Map snapshots = directories.listSnapshots(); + Map snapshots = listSnapshots(directories); assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3)); assertThat(snapshots.get(SNAPSHOT1)).isEqualTo(snapshot1.asTableSnapshot()); assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot()); @@ -350,40 +366,13 @@ public class DirectoriesTest snapshot1.snapshotDir.deleteRecursive(); // Only snapshot 2 and 3 should be present - snapshots = directories.listSnapshots(); + snapshots = listSnapshots(directories); assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3)); assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot()); assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot()); assertThat(snapshots.get(SNAPSHOT3).isEphemeral()).isTrue(); } - @Test - public void testListSnapshotDirsByTag() throws Exception { - // Initial state - TableMetadata fakeTable = createFakeTable("FakeTable"); - Directories directories = new Directories(fakeTable, toDataDirectories(tempDataDir)); - assertThat(directories.listSnapshotDirsByTag()).isEmpty(); - - // Create snapshot with and without manifest - FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false); - FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false); - FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true); - - // Both snapshots should be present - Map> snapshotDirs = directories.listSnapshotDirsByTag(); - assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3)); - assertThat(snapshotDirs.get(SNAPSHOT1)).allMatch(snapshotDir -> snapshotDir.equals(snapshot1.snapshotDir)); - assertThat(snapshotDirs.get(SNAPSHOT2)).allMatch(snapshotDir -> snapshotDir.equals(snapshot2.snapshotDir)); - assertThat(snapshotDirs.get(SNAPSHOT3)).allMatch(snapshotDir -> snapshotDir.equals(snapshot3.snapshotDir)); - - // Now remove snapshot1 - snapshot1.snapshotDir.deleteRecursive(); - - // Only snapshot 2 and 3 should be present - snapshotDirs = directories.listSnapshotDirsByTag(); - assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3)); - } - @Test public void testMaybeManifestLoading() throws Exception { for (TableMetadata cfm : CFM) @@ -412,7 +401,7 @@ public class DirectoriesTest } @Test - public void testSecondaryIndexDirectories() + public void testSecondaryIndexDirectories() throws IOException { TableMetadata.Builder builder = TableMetadata.builder(KS, "cf") @@ -447,8 +436,8 @@ public class DirectoriesTest // check if snapshot directory exists parentSnapshotDirectory.tryCreateDirectories(); - assertTrue(parentDirectories.snapshotExists("test")); - assertTrue(indexDirectories.snapshotExists("test")); + assertTrue(snapshotExists(parentDirectories, "test")); + assertTrue(snapshotExists(indexDirectories, "test")); // check true snapshot size Descriptor parentSnapshot = new Descriptor(parentSnapshotDirectory, KS, PARENT_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); @@ -456,14 +445,21 @@ public class DirectoriesTest Descriptor indexSnapshot = new Descriptor(indexSnapshotDirectory, KS, INDEX_CFM.name, sstableId(0), DatabaseDescriptor.getSelectedSSTableFormat()); createFile(indexSnapshot.fileFor(Components.DATA), 40); - assertEquals(30, parentDirectories.trueSnapshotsSize()); - assertEquals(40, indexDirectories.trueSnapshotsSize()); - // check snapshot details - Map parentSnapshotDetail = parentDirectories.listSnapshots(); + Map parentSnapshotDetail = listSnapshots(parentDirectories); assertTrue(parentSnapshotDetail.containsKey("test")); + + Set files = new HashSet<>(); + Files.walkFileTree(parentSnapshotDirectory.toPath(), new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + { + files.add(file.toAbsolutePath().toString()); + return FileVisitResult.CONTINUE; + } + }); // CASSANDRA-17357: include indexes when computing true size of parent table - assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes()); + assertEquals(70L, parentSnapshotDetail.get("test").computeTrueSizeBytes(files)); // check backup directory File parentBackupDirectory = Directories.getBackupsDirectory(parentDesc); @@ -471,6 +467,30 @@ public class DirectoriesTest assertEquals(parentBackupDirectory, indexBackupDirectory.parent()); } + private boolean snapshotExists(Directories directories, String snapshotName) + { + for (File dir : directories.getDataPaths()) + { + File snapshotDir; + if (Directories.isSecondaryIndexFolder(dir)) + { + snapshotDir = new File(dir.parent(), join(Directories.SNAPSHOT_SUBDIR, snapshotName, dir.name())); + } + else + { + snapshotDir = new File(dir, join(Directories.SNAPSHOT_SUBDIR, snapshotName)); + } + if (snapshotDir.exists()) + return true; + } + return false; + } + + private static String join(String... s) + { + return StringUtils.join(s, File.pathSeparator()); + } + private File createFile(File file, int size) { try (FileOutputStreamPlus writer = new FileOutputStreamPlus(file);) diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java index 84c5b3ecfd..f78d081dce 100644 --- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java @@ -19,34 +19,49 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Collection; +import java.util.List; -import org.assertj.core.api.Assertions; +import org.junit.Before; import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.io.sstable.AbstractRowIndexEntry; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.filter.*; -import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.big.BigTableReader; import org.apache.cassandra.metrics.ClearableHistogram; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.Assertions; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; public class KeyspaceTest extends CQLTester { // Test needs synchronous table drop to avoid flushes causing flaky failures of testLimitSSTables + @Before + public void cleanupSnapshots() + { + SnapshotManager.instance.clearAllSnapshots(); + } + @Override protected String createTable(String query) { @@ -425,10 +440,11 @@ public class KeyspaceTest extends CQLTester Keyspace ks = Keyspace.open(KEYSPACE_PER_TEST); String table = getCurrentColumnFamilyStore().name; - ks.snapshot("test", table); + SnapshotManager.instance.takeSnapshot("test", ks.getName() + '.' + table); - assertTrue(ks.snapshotExists("test")); - assertEquals(1, ks.getAllSnapshots().count()); + List snapshots = SnapshotManager.instance.getSnapshots(ks.getName()); + assertEquals(1, snapshots.size()); + assertEquals(snapshots.get(0).getTag(), "test"); } @Test diff --git a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java index 3343e6d7f6..d045584412 100644 --- a/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java +++ b/test/unit/org/apache/cassandra/db/SchemaCQLHelperTest.java @@ -35,6 +35,8 @@ import org.apache.cassandra.index.internal.CassandraIndex; import org.apache.cassandra.index.sasi.SASIIndex; import org.apache.cassandra.schema.*; import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JsonUtils; @@ -431,7 +433,7 @@ public class SchemaCQLHelperTest extends CQLTester execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1, reg2) VALUES (?, ?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null, i + 5); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName); - cfs.snapshot(SNAPSHOT); + SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(SNAPSHOT, cfs.getKeyspaceTableName())); String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset()); assertThat(schema, @@ -480,6 +482,7 @@ public class SchemaCQLHelperTest extends CQLTester "INDEX IF NOT EXISTS " + tableName + "_reg2_idx ON " + keyspace() + '.' + tableName + " (reg2)" + (" USING '" + (isIndexLegacy ? CassandraIndex.NAME : DatabaseDescriptor.getDefaultSecondaryIndex()) + "'") + ";")); + // TODO: construct manifest from SnapshotManager JsonNode manifest = JsonUtils.JSON_OBJECT_MAPPER.readTree(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).toJavaIOFile()); JsonNode files = manifest.get("files"); // two files, the second is index @@ -508,7 +511,7 @@ public class SchemaCQLHelperTest extends CQLTester execute("INSERT INTO %s (pk1, pk2, ck1, ck2, reg1) VALUES (?, ?, ?, ?, ?)", i, i + 1, i + 2, i + 3, null); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName); - cfs.snapshot(SNAPSHOT); + SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName()); String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset()); schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order @@ -551,7 +554,7 @@ public class SchemaCQLHelperTest extends CQLTester execute("INSERT INTO %s (pk1, reg1) VALUES (?, ?)", i, i + 1); ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(tableName); - cfs.snapshot(SNAPSHOT); + SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName()); String schema = Files.toString(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).toJavaIOFile(), Charset.defaultCharset()); schema = schema.substring(schema.indexOf("CREATE TABLE")); // trim to ensure order @@ -577,7 +580,7 @@ public class SchemaCQLHelperTest extends CQLTester public void testSystemKsSnapshot() { ColumnFamilyStore cfs = Keyspace.open("system").getColumnFamilyStore("peers"); - cfs.snapshot(SNAPSHOT); + SnapshotManager.instance.takeSnapshot(SNAPSHOT, cfs.getKeyspaceTableName()); Assert.assertTrue(cfs.getDirectories().getSnapshotManifestFile(SNAPSHOT).exists()); Assert.assertFalse(cfs.getDirectories().getSnapshotSchemaFile(SNAPSHOT).exists()); diff --git a/test/unit/org/apache/cassandra/db/SnapshotTest.java b/test/unit/org/apache/cassandra/db/SnapshotTest.java index fdfdd6b3aa..4db0941736 100644 --- a/test/unit/org/apache/cassandra/db/SnapshotTest.java +++ b/test/unit/org/apache/cassandra/db/SnapshotTest.java @@ -27,6 +27,7 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.service.snapshot.SnapshotManager; public class SnapshotTest extends CQLTester { @@ -41,6 +42,7 @@ public class SnapshotTest extends CQLTester File toc = sstable.descriptor.fileFor(Components.TOC); Files.write(toc.toPath(), new byte[0], StandardOpenOption.TRUNCATE_EXISTING); } - getCurrentColumnFamilyStore().snapshot("hello"); + + SnapshotManager.instance.takeSnapshot("hello", getCurrentColumnFamilyStore().getKeyspaceTableName()); } } diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java index bcaefc23d9..2047210323 100644 --- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java @@ -25,6 +25,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; @@ -34,7 +35,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspace; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.transport.ProtocolVersion; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.CassandraVersion; @@ -100,14 +101,14 @@ public class SystemKeyspaceTest // First, check that in the absence of any previous installed version, we don't create snapshots for (ColumnFamilyStore cfs : Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStores()) cfs.clearUnsafe(); - StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME); SystemKeyspace.snapshotOnVersionChange(); assertDeleted(); // now setup system.local as if we're upgrading from a previous version setupReleaseVersion(getOlderVersionString()); - StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME); assertDeleted(); // Compare versions again & verify that snapshots were created for all tables in the system ks @@ -120,7 +121,7 @@ public class SystemKeyspaceTest // clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't // see any new snapshots created this time. - StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME); setupReleaseVersion(FBUtilities.getReleaseVersionString()); SystemKeyspace.snapshotOnVersionChange(); @@ -129,7 +130,7 @@ public class SystemKeyspaceTest // 10 files expected. assertDeleted(); - StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + SnapshotManager.instance.clearSnapshot(null, Collections.emptyMap(), SchemaConstants.SYSTEM_KEYSPACE_NAME); } @Test @@ -170,7 +171,7 @@ public class SystemKeyspaceTest Set snapshottedTableNames = new HashSet<>(); for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores()) { - if (!cfs.listSnapshots().isEmpty()) + if (!Util.listSnapshots(cfs).isEmpty()) snapshottedTableNames.add(cfs.getTableName()); } return snapshottedTableNames; diff --git a/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java b/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java index 09832309c1..0d4ed611c4 100644 --- a/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/SnapshotsTableTest.java @@ -19,27 +19,30 @@ package org.apache.cassandra.db.virtual; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Date; +import java.util.List; +import java.util.Map; import com.google.common.collect.ImmutableList; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.service.snapshot.SnapshotOptions; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.TableSnapshot; public class SnapshotsTableTest extends CQLTester { private static final String KS_NAME = "vts"; private static final String SNAPSHOT_TTL = "snapshotTtl"; private static final String SNAPSHOT_NO_TTL = "snapshotNoTtl"; - private static final DurationSpec.IntSecondsBound ttl = new DurationSpec.IntSecondsBound("4h"); + private static final String TTL = "4h"; @Before public void before() throws Throwable @@ -58,48 +61,56 @@ public class SnapshotsTableTest extends CQLTester @After public void after() { - StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE); - StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_TTL, KEYSPACE); + SnapshotManager.instance.clearSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), KEYSPACE); + SnapshotManager.instance.clearSnapshot(SNAPSHOT_TTL, Collections.emptyMap(), KEYSPACE); + schemaChange(String.format("DROP TABLE %s", KEYSPACE + '.' + currentTable())); + } - schemaChange(String.format("DROP TABLE %s", KEYSPACE + "." + currentTable())); + private static Date toDate(Instant instant) + { + return new Date(instant.toEpochMilli()); + } + + private static TableSnapshot createSnapshot(String snapshotTtl, Map options, ColumnFamilyStore cfs) + { + List snapshots = SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(snapshotTtl, options, cfs.getKeyspaceTableName())); + Assert.assertEquals(1, snapshots.size()); + return snapshots.iterator().next(); } @Test - public void testSnapshots() throws Throwable + public void testSnapshots() { - Instant now = Instant.ofEpochMilli(Clock.Global.currentTimeMillis()).truncatedTo(ChronoUnit.MILLIS); - Date createdAt = new Date(now.toEpochMilli()); - Date expiresAt = new Date(now.plusSeconds(ttl.toSeconds()).toEpochMilli()); - - getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_NO_TTL, null, false, false, null, null, now); - getCurrentColumnFamilyStore(KEYSPACE).snapshot(SNAPSHOT_TTL, null, false, false, ttl, null, now); + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE); + TableSnapshot snapshotWithTtl = createSnapshot(SNAPSHOT_TTL, Map.of(SnapshotOptions.TTL, TTL), cfs); + TableSnapshot snapshotWithoutTtl = createSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), cfs); // query all from snapshots virtual table UntypedResultSet result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots"); assertRowsIgnoringOrder(result, - row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false), - row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false)); + row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false), + row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false)); // query with conditions result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where ephemeral = false"); assertRows(result, - row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false), - row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false)); + row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false), + row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false)); result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where size_on_disk > 1000"); assertRows(result, - row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), createdAt, null, false), - row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false)); + row(SNAPSHOT_NO_TTL, KEYSPACE, currentTable(), toDate(snapshotWithoutTtl.getCreatedAt()), null, false), + row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false)); result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots where name = ?", SNAPSHOT_TTL); assertRows(result, - row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false)); + row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false)); // clear some snapshots - StorageService.instance.clearSnapshot(Collections.emptyMap(), SNAPSHOT_NO_TTL, KEYSPACE); + SnapshotManager.instance.clearSnapshot(SNAPSHOT_NO_TTL, Collections.emptyMap(), KEYSPACE); result = execute("SELECT name, keyspace_name, table_name, created_at, expires_at, ephemeral FROM vts.snapshots"); assertRowsIgnoringOrder(result, - row(SNAPSHOT_TTL, KEYSPACE, currentTable(), createdAt, expiresAt, false)); + row(SNAPSHOT_TTL, KEYSPACE, currentTable(), toDate(snapshotWithTtl.getCreatedAt()), toDate(snapshotWithTtl.getExpiresAt()), false)); } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java index e1e6af7f0b..e004f0c0ee 100644 --- a/test/unit/org/apache/cassandra/index/sai/SAITester.java +++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java @@ -102,7 +102,8 @@ import org.apache.cassandra.schema.MockSchema; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.service.snapshot.TableSnapshot; +import org.apache.cassandra.service.snapshot.SnapshotManager; +import org.apache.cassandra.service.snapshot.SnapshotOptions; import org.apache.cassandra.utils.ConfigGenBuilder; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.Throwables; @@ -683,8 +684,7 @@ public abstract class SAITester extends CQLTester.Fuzzed protected int snapshot(String snapshotName) { ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); - TableSnapshot snapshot = cfs.snapshot(snapshotName); - return snapshot.getDirectories().size(); + return SnapshotManager.instance.takeSnapshot(SnapshotOptions.userSnapshot(snapshotName, cfs.getKeyspaceTableName())).iterator().next().getDirectories().size(); } protected void restoreSnapshot(String snapshot) diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index 174ebd3d56..70da2a0ab6 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -126,6 +126,7 @@ import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.serializers.TypeSerializer; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.ByteBufferUtil; @@ -199,7 +200,7 @@ public class SASIIndexTest try { - store.snapshot(snapshotName); + SnapshotManager.instance.takeSnapshot(snapshotName, store.getKeyspaceTableName()); // Compact to make true snapshot size != 0 store.forceMajorCompaction(); @@ -208,6 +209,7 @@ public class SASIIndexTest SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName)); Assert.assertFalse(ssTableReaders.isEmpty()); + Assert.assertNotNull(manifest.files); Assert.assertFalse(manifest.files.isEmpty()); Assert.assertEquals(ssTableReaders.size(), manifest.files.size()); @@ -243,7 +245,7 @@ public class SASIIndexTest } } - TableSnapshot details = store.listSnapshots().get(snapshotName); + TableSnapshot details = Util.listSnapshots(store).get(snapshotName); // check that SASI components are included in the computation of snapshot size long snapshotSize = tableSize + indexSize + getSnapshotManifestAndSchemaFileSizes(details); @@ -251,7 +253,7 @@ public class SASIIndexTest } finally { - store.clearSnapshot(snapshotName); + SnapshotManager.instance.clearSnapshot(store.getKeyspaceName(), store.getTableName(), snapshotName); } } diff --git a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java index 0a06a853ed..7ceceaad04 100644 --- a/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/schema/SchemaKeyspaceTest.java @@ -31,6 +31,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; @@ -129,7 +130,7 @@ public class SchemaKeyspaceTest SchemaTestUtil.announceTableDrop(keyspaceName, tableName); - assertFalse(cfs.listSnapshots().isEmpty()); + assertFalse(Util.listSnapshots(cfs).isEmpty()); } @Test @@ -147,7 +148,7 @@ public class SchemaKeyspaceTest SchemaTestUtil.announceTableDrop(keyspaceName, tableName); - assertTrue(cfs.listSnapshots().isEmpty()); + assertTrue(Util.listSnapshots(cfs).isEmpty()); } private static void updateTable(String keyspace, TableMetadata oldTable, TableMetadata newTable) diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index 23cfa80bfc..c59163ae0b 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -61,6 +61,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.repair.messages.RepairOption; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeAddresses; @@ -293,7 +294,7 @@ public class ActiveRepairServiceTest true, PreviewKind.NONE); createSSTables(store, 2); store.getRepairManager().snapshot(prsId.toString(), ranges, false); - try (Refs refs = store.getSnapshotSSTableReaders(prsId.toString())) + try (Refs refs = TableSnapshot.getSnapshotSSTableReaders(store, prsId.toString())) { assertEquals(original, Sets.newHashSet(refs.iterator())); } diff --git a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java index 8753413580..42ed428c76 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceServerTest.java @@ -55,6 +55,7 @@ import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.SchemaKeyspaceTables; import org.apache.cassandra.schema.SchemaTestUtil; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.membership.Location; @@ -144,6 +145,7 @@ public class StorageServiceServerTest @Before public void resetCMS() { + SnapshotManager.instance.clearAllSnapshots(); ServerTestUtils.resetCMS(); } @@ -158,21 +160,27 @@ public class StorageServiceServerTest public void testSnapshotWithFlush() throws IOException { // no need to insert extra data, even an "empty" database will have a little information in the system keyspace - StorageService.instance.takeSnapshot(UUID.randomUUID().toString()); + String snapshotName = UUID.randomUUID().toString(); + StorageService.instance.takeSnapshot(snapshotName); + Assert.assertTrue(SnapshotManager.instance.exists(p -> p.getTag().equals(snapshotName))); } @Test public void testTableSnapshot() throws IOException { // no need to insert extra data, even an "empty" database will have a little information in the system keyspace - StorageService.instance.takeTableSnapshot(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, UUID.randomUUID().toString()); + String snapshotName = UUID.randomUUID().toString(); + StorageService.instance.takeTableSnapshot(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, snapshotName); + Assert.assertTrue(SnapshotManager.instance.exists(SchemaConstants.SCHEMA_KEYSPACE_NAME, SchemaKeyspaceTables.KEYSPACES, snapshotName)); } @Test public void testSnapshot() throws IOException { // no need to insert extra data, even an "empty" database will have a little information in the system keyspace - StorageService.instance.takeSnapshot(UUID.randomUUID().toString(), SchemaConstants.SCHEMA_KEYSPACE_NAME); + String snapshotName = UUID.randomUUID().toString(); + StorageService.instance.takeSnapshot(snapshotName, SchemaConstants.SCHEMA_KEYSPACE_NAME); + Assert.assertTrue(SnapshotManager.instance.exists(p -> p.getTag().equals(snapshotName) && p.getKeyspaceName().equals(SchemaConstants.SCHEMA_KEYSPACE_NAME))); } @Test diff --git a/test/unit/org/apache/cassandra/service/snapshot/MetadataSnapshotsTest.java b/test/unit/org/apache/cassandra/service/snapshot/MetadataSnapshotsTest.java index f2565ee3f5..ee996e3a33 100644 --- a/test/unit/org/apache/cassandra/service/snapshot/MetadataSnapshotsTest.java +++ b/test/unit/org/apache/cassandra/service/snapshot/MetadataSnapshotsTest.java @@ -22,26 +22,22 @@ import java.time.Instant; import java.util.Arrays; import java.util.List; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; -import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.After; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.service.DefaultFSErrorHandler; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.cassandra.service.snapshot.TableSnapshotTest.createFolders; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.cassandra.utils.FBUtilities.now; import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.junit.Assert.assertTrue; public class MetadataSnapshotsTest { @@ -50,6 +46,9 @@ public class MetadataSnapshotsTest @BeforeClass public static void beforeClass() { + CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.setInt(3); + CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(3); + DatabaseDescriptor.daemonInitialization(); FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); } @@ -57,6 +56,21 @@ public class MetadataSnapshotsTest @ClassRule public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + private SnapshotManager manager; + + @Before + public void beforeTest() + { + manager = SnapshotManager.instance; + } + + @After + public void afterTest() throws Exception + { + SnapshotManager.instance.clearAllSnapshots(); + SnapshotManager.instance.close(); + } + private TableSnapshot generateSnapshotDetails(String tag, Instant expiration, boolean ephemeral) { try @@ -77,25 +91,28 @@ public class MetadataSnapshotsTest } @Test - public void testLoadSnapshots() throws Exception { + public void testExpiringSnapshots() + { TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false); TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS), false); TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false); List snapshots = Arrays.asList(expired, nonExpired, nonExpiring); // Create SnapshotManager with 3 snapshots: expired, non-expired and non-expiring - SnapshotManager manager = new SnapshotManager(3, 3); - manager.addSnapshots(snapshots); + manager.start(false); + for (TableSnapshot snapshot : snapshots) + manager.addSnapshot(snapshot); // Only expiring snapshots should be loaded - assertThat(manager.getExpiringSnapshots()).hasSize(2); - assertThat(manager.getExpiringSnapshots()).contains(expired); - assertThat(manager.getExpiringSnapshots()).contains(nonExpired); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expired); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired); } @Test - public void testClearExpiredSnapshots() throws Exception { - SnapshotManager manager = new SnapshotManager(3, 3); + public void testClearExpiredSnapshots() + { + manager.start(false); // Add 3 snapshots: expired, non-expired and non-expiring TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false); @@ -106,119 +123,66 @@ public class MetadataSnapshotsTest manager.addSnapshot(nonExpiring); // Only expiring snapshot should be indexed and all should exist - assertThat(manager.getExpiringSnapshots()).hasSize(2); - assertThat(manager.getExpiringSnapshots()).contains(expired); - assertThat(manager.getExpiringSnapshots()).contains(nonExpired); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expired); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired); assertThat(expired.exists()).isTrue(); assertThat(nonExpired.exists()).isTrue(); assertThat(nonExpiring.exists()).isTrue(); // After clearing expired snapshots, expired snapshot should be removed while the others should remain manager.clearExpiredSnapshots(); - assertThat(manager.getExpiringSnapshots()).hasSize(1); - assertThat(manager.getExpiringSnapshots()).contains(nonExpired); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(1); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired); assertThat(expired.exists()).isFalse(); assertThat(nonExpired.exists()).isTrue(); assertThat(nonExpiring.exists()).isTrue(); } @Test - public void testScheduledCleanup() throws Exception { - SnapshotManager manager = new SnapshotManager(0, 1); - try - { - // Start snapshot manager which should start expired snapshot cleanup thread - manager.start(); + public void testScheduledCleanup() throws Exception + { + manager.start(true); - // Add 2 expiring snapshots: one to expire in 2 seconds, another in 1 day - int TTL_SECS = 2; - TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false); - TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false); - manager.addSnapshot(toExpire); - manager.addSnapshot(nonExpired); + // Add 2 expiring snapshots: one to expire in 6 seconds, another in 1 day + int TTL_SECS = 6; + TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false); + TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false); + manager.addSnapshot(toExpire); + manager.addSnapshot(nonExpired); - // Check both snapshots still exist - assertThat(toExpire.exists()).isTrue(); - assertThat(nonExpired.exists()).isTrue(); - assertThat(manager.getExpiringSnapshots()).hasSize(2); - assertThat(manager.getExpiringSnapshots()).contains(toExpire); - assertThat(manager.getExpiringSnapshots()).contains(nonExpired); + // Check both snapshots still exist + assertThat(toExpire.exists()).isTrue(); + assertThat(nonExpired.exists()).isTrue(); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(2); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(toExpire); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired); - // Sleep 4 seconds - Thread.sleep((TTL_SECS + 2) * 1000L); + // Sleep 10 seconds + Thread.sleep((TTL_SECS + 4) * 1000L); - // Snapshot with ttl=2s should be gone, while other should remain - assertThat(manager.getExpiringSnapshots()).hasSize(1); - assertThat(manager.getExpiringSnapshots()).contains(nonExpired); - assertThat(toExpire.exists()).isFalse(); - assertThat(nonExpired.exists()).isTrue(); - } - finally - { - manager.stop(); - } + // Snapshot with ttl=6s should be gone, while other should remain + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).hasSize(1); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(nonExpired); + assertThat(toExpire.exists()).isFalse(); + assertThat(nonExpired.exists()).isTrue(); } @Test - public void testClearSnapshot() throws Exception + public void testClearSnapshot() { // Given - SnapshotManager manager = new SnapshotManager(1, 3); + manager.start(false); TableSnapshot expiringSnapshot = generateSnapshotDetails("snapshot", now().plusMillis(50000), false); manager.addSnapshot(expiringSnapshot); - assertThat(manager.getExpiringSnapshots()).contains(expiringSnapshot); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).contains(expiringSnapshot); assertThat(expiringSnapshot.exists()).isTrue(); // When manager.clearSnapshot(expiringSnapshot); // Then - assertThat(manager.getExpiringSnapshots()).doesNotContain(expiringSnapshot); + assertThat(manager.getSnapshots(TableSnapshot::isExpiring)).doesNotContain(expiringSnapshot); assertThat(expiringSnapshot.exists()).isFalse(); } - - @Test // see CASSANDRA-18211 - public void testConcurrentClearingOfSnapshots() throws Exception - { - - AtomicReference firstInvocationTime = new AtomicReference<>(0L); - AtomicReference secondInvocationTime = new AtomicReference<>(0L); - - SnapshotManager manager = new SnapshotManager(0, 5) - { - @Override - public synchronized void clearSnapshot(TableSnapshot snapshot) - { - if (snapshot.getTag().equals("mysnapshot")) - { - firstInvocationTime.set(currentTimeMillis()); - Uninterruptibles.sleepUninterruptibly(10, SECONDS); - } - else if (snapshot.getTag().equals("mysnapshot2")) - { - secondInvocationTime.set(currentTimeMillis()); - } - super.clearSnapshot(snapshot); - } - }; - - TableSnapshot expiringSnapshot = generateSnapshotDetails("mysnapshot", Instant.now().plusSeconds(15), false); - manager.addSnapshot(expiringSnapshot); - - manager.resumeSnapshotCleanup(); - - Thread nonExpiringSnapshotCleanupThred = new Thread(() -> manager.clearSnapshot(generateSnapshotDetails("mysnapshot2", null, false))); - - // wait until the first snapshot expires - await().pollInterval(1, SECONDS) - .pollDelay(0, SECONDS) - .timeout(1, MINUTES) - .until(() -> firstInvocationTime.get() > 0); - - // this will block until the first snapshot is cleaned up - nonExpiringSnapshotCleanupThred.start(); - nonExpiringSnapshotCleanupThred.join(); - - assertTrue(secondInvocationTime.get() - firstInvocationTime.get() > 10_000); - } } \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java index 9c2303f842..54ecf19d98 100644 --- a/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java +++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java @@ -28,10 +28,12 @@ import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.Directories; import org.apache.cassandra.io.util.File; @@ -67,6 +69,12 @@ public class SnapshotLoaderTest @ClassRule public static TemporaryFolder tmpDir = new TemporaryFolder(); + @BeforeClass + public static void setup() + { + DatabaseDescriptor.daemonInitialization(); + } + @Test public void testMatcher() { diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java new file mode 100644 index 0000000000..703e7996ad --- /dev/null +++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.FSWriteError; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.PathUtils; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import static java.lang.String.format; +import static org.apache.cassandra.config.CassandraRelevantProperties.SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SnapshotManagerTest +{ + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @ClassRule + public static TemporaryFolder temporaryFolder2 = new TemporaryFolder(); + + private static File rootDir1; + private static File rootDir2; + + private static String[] dataDirs; + + @BeforeClass + public static void beforeClass() throws Exception + { + CassandraRelevantProperties.SNAPSHOT_CLEANUP_PERIOD_SECONDS.setInt(10); + SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.setInt(5); + + rootDir1 = new File(temporaryFolder.getRoot()); + rootDir2 = new File(temporaryFolder2.getRoot()); + + dataDirs = new String[]{ + rootDir1.toPath().toAbsolutePath().toString(), + rootDir2.toPath().toAbsolutePath().toString() + }; + } + + private static class ThrowingTask extends AbstractSnapshotTask + { + public ThrowingTask() + { + super(null); + } + + @Override + public Void call() + { + throw new RuntimeException("an exception"); + } + + @Override + public SnapshotTaskType getTaskType() + { + return SnapshotTaskType.SNAPSHOT; + } + } + + @Test + public void testTaskThrowingException() + { + doWithManager(manager -> assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class) + .hasRootCauseMessage("an exception") + .hasMessageContaining("Exception occured while executing")); + + doWithManager(manager -> { + + manager.close(); + + assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class) + .hasRootCauseMessage("an exception") + .hasMessageContaining("Exception occured while executing"); + + manager.start(false); + + assertThatThrownBy(() -> manager.executeTask(new ThrowingTask())).isInstanceOf(RuntimeException.class) + .hasRootCauseMessage("an exception") + .hasMessageContaining("Exception occured while executing"); + }); + } + + /** + * Tests that if we remove all manifests files, that equals to stopping manager to track + * such snapshot, however no data will be removed. + */ + @Test + public void testRemovingManifestsLogicallyRemovesSnapshot() + { + doWithManager(manager -> { + List tableSnapshots = generateTableSnapshots(10, 100); + + for (TableSnapshot snapshot : tableSnapshots) + manager.addSnapshot(snapshot); + + // we still have 1000 snapshots because we removed just one manifest + removeManifestOfSnapshot(tableSnapshots.get(0)); + assertEquals(1000, manager.getSnapshots((t) -> true).size()); + + // remove the second manifest, that will render snapshot to be logically removed + removeManifestOfSnapshot(tableSnapshots.get(0)); + assertEquals(999, manager.getSnapshots((t) -> true).size()); + + // check that data are still there + assertFalse(tableSnapshots.get(0).hasManifest()); + assertTrue(tableSnapshots.get(0).exists()); + for (File snapshotDir : tableSnapshots.get(0).getDirectories()) + assertTrue(snapshotDir.exists()); + }); + } + + private void doWithManager(Consumer action) + { + try (MockedStatic mockedDD = Mockito.mockStatic(DatabaseDescriptor.class)) + { + mockedDD.when(() -> DatabaseDescriptor.getNonLocalSystemKeyspacesDataFileLocations()).thenReturn(dataDirs); + mockedDD.when(() -> DatabaseDescriptor.getLocalSystemKeyspacesDataFileLocations()).thenReturn(new String[]{}); + mockedDD.when(() -> DatabaseDescriptor.getAllDataFileLocations()).thenReturn(dataDirs); + mockedDD.when(() -> DatabaseDescriptor.getDumpHeapOnUncaughtException()).thenReturn(false); + + try (SnapshotManager snapshotManager = new SnapshotManager(SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(10), + SNAPSHOT_CLEANUP_INITIAL_DELAY_SECONDS.getInt(5), + dataDirs).start(true)) + { + action.accept(snapshotManager); + } + finally + { + clearDirectory(rootDir1.toPath()); + clearDirectory(rootDir2.toPath()); + } + } + } + + private List generateTableSnapshots(int keyspaces, int tables) throws RuntimeException + { + List tableSnapshots = new ArrayList<>(); + for (int i = 0; i < keyspaces; i++) + { + for (int j = 0; j < tables; j++) + { + String snapshotName = format("mysnapshot_%s_%s", i, j); + File dir1 = new File(Paths.get(rootDir1.absolutePath(), "ks", "tb-1b255f4def2540a60000000000000005", "snapshots", snapshotName)); + File dir2 = new File(Paths.get(rootDir2.absolutePath(), "ks", "tb-1b255f4def2540a60000000000000005", "snapshots", snapshotName)); + dir1.tryCreateDirectories(); + dir2.tryCreateDirectories(); + TableSnapshot snapshot = generateSnapshotDetails(Set.of(dir1, dir2), snapshotName, "ks", "tb-1b255f4def2540a60000000000000005", null, false); + SnapshotManifest manifest = new SnapshotManifest(List.of(), null, snapshot.getCreatedAt(), snapshot.isEphemeral()); + try + { + manifest.serializeToJsonFile(new File(dir1.toPath().resolve("manifest.json"))); + manifest.serializeToJsonFile(new File(dir2.toPath().resolve("manifest.json"))); + generateFileInSnapshot(snapshot); + } + catch (Exception ex) + { + throw new RuntimeException(ex); + } + + tableSnapshots.add(snapshot); + } + } + + return tableSnapshots; + } + + private void generateFileInSnapshot(TableSnapshot tableSnapshot) throws IOException + { + for (File snapshotDir : tableSnapshot.getDirectories()) + Files.createFile(snapshotDir.toPath().resolve("schema.cql")); + } + + private void removeManifestOfSnapshot(TableSnapshot tableSnapshot) + { + for (File snapshotDir : tableSnapshot.getDirectories()) + { + if (snapshotDir.exists()) + { + File manifest = new File(snapshotDir, "manifest.json"); + if (!manifest.exists()) + continue; + + manifest.delete(); + return; + } + } + } + + private TableSnapshot generateSnapshotDetails(Set roots, + String tag, + String keyspace, + String table, + Instant expiration, + boolean ephemeral) + { + try + { + Set snapshotDirs = new HashSet<>(); + for (File root : roots) + { + root.tryCreateDirectories(); + snapshotDirs.add(root); + } + + return new TableSnapshot(keyspace, + table, + UUID.randomUUID(), + tag, + Instant.EPOCH, + expiration, + snapshotDirs, + ephemeral); + } + catch (Exception ex) + { + throw new RuntimeException(ex); + } + } + + + /** + * Empties everything in directory of "path" but keeps the directory itself. + * + * @param path directory to be emptied + * @throws FSWriteError if any part of the tree cannot be deleted + */ + public static void clearDirectory(Path path) + { + if (PathUtils.isDirectory(path)) + PathUtils.forEach(path, PathUtils::deleteRecursive); + } + +} diff --git a/test/unit/org/apache/cassandra/service/snapshot/SnapshotOptionsTest.java b/test/unit/org/apache/cassandra/service/snapshot/SnapshotOptionsTest.java new file mode 100644 index 0000000000..e61d283962 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/snapshot/SnapshotOptionsTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.snapshot; + +import java.time.Instant; +import java.util.EnumSet; +import java.util.List; + +import com.google.common.util.concurrent.RateLimiter; +import org.junit.Test; + +import static java.lang.String.format; +import static org.junit.Assert.assertEquals; + +public class SnapshotOptionsTest +{ + @Test + public void testSnapshotName() + { + List sameNameTypes = List.of(SnapshotType.DIAGNOSTICS, SnapshotType.REPAIR, SnapshotType.USER); + + for (SnapshotType type : sameNameTypes) + { + SnapshotOptions options = SnapshotOptions.systemSnapshot("a_name", type, "ks.tb") + .rateLimiter(RateLimiter.create(5)) + .build(); + + String snapshotName = options.getSnapshotName(Instant.now()); + assertEquals("a_name", snapshotName); + } + + EnumSet snapshotTypes = EnumSet.allOf(SnapshotType.class); + snapshotTypes.removeAll(sameNameTypes); + + for (SnapshotType type : snapshotTypes) + { + SnapshotOptions options = SnapshotOptions.systemSnapshot("a_name", type, "ks.tb") + .rateLimiter(RateLimiter.create(5)) + .build(); + + Instant now = Instant.now(); + + String snapshotName = options.getSnapshotName(now); + + assertEquals(format("%d-%s-%s", now.toEpochMilli(), type.label, "a_name"), snapshotName); + } + } +} diff --git a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java index d592dd91d2..3ec0064c73 100644 --- a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java +++ b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java @@ -19,7 +19,7 @@ package org.apache.cassandra.service.snapshot; import java.io.IOException; -import java.nio.file.Paths; +import java.nio.file.Files; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -38,10 +38,16 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.Pair; +import org.mockito.MockedStatic; import static org.apache.cassandra.utils.FBUtilities.now; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; public class TableSnapshotTest { @@ -62,13 +68,68 @@ public class TableSnapshotTest { File subfolder = new File(folder, folderName); subfolder.tryCreateDirectories(); - assertThat(subfolder.exists()); + File manifest = new File(subfolder.toPath().resolve("manifest.json")); + File schema = new File(subfolder.toPath().resolve("schema.cql")); + manifest.createFileIfNotExists(); + schema.createFileIfNotExists(); + Files.write(manifest.toPath(), "{}".getBytes()); + Files.write(schema.toPath(), "cql schema".getBytes()); folders.add(subfolder); } - ; + return folders; } + @Test + public void testTableSnapshotEquality() throws IOException + { + TableSnapshot snapshot1 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, createFolders(tempFolder), false); + TableSnapshot snapshot2 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, createFolders(tempFolder), false); + + // they are not equal, because table id is random for each + assertNotEquals(snapshot1, snapshot2); + + UUID tableId = UUID.randomUUID(); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, createFolders(tempFolder), false); + snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, createFolders(tempFolder), false); + + // they are equal, even their directories differ + assertEquals(snapshot1, snapshot2); + + Set folders = createFolders(tempFolder); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", Instant.now(), null, folders, false); + snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", Instant.now().plusSeconds(1), null, folders, false); + + // they are equal, even their creation times differ + assertEquals(snapshot1, snapshot2); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, Instant.now(), folders, false); + snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some", null, Instant.now().plusSeconds(1), folders, false); + + // they are equal, even their expiration times differ + assertEquals(snapshot1, snapshot2); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some", null, null, folders, false); + snapshot2 = new TableSnapshot("ks", "tbl", UUID.randomUUID(), "some", null, null, folders, false); + + // they are not equal, because their tableId differs + assertNotEquals(snapshot1, snapshot2); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some1", null, null, folders, false); + snapshot2 = new TableSnapshot("ks", "tbl", tableId, "some2", null, null, folders, false); + + // they are not equal, because their tag differs + assertNotEquals(snapshot1, snapshot2); + + snapshot1 = new TableSnapshot("ks", "tbl", tableId, "some1", null, null, folders, false); + snapshot2 = new TableSnapshot("ks", "tbl2", tableId, "some1", null, null, folders, false); + + // they are not equal, because their table differs + assertNotEquals(snapshot1, snapshot2); + } + @Test public void testSnapshotExists() throws IOException { @@ -186,8 +247,18 @@ public class TableSnapshotTest res += FileUtils.folderSize(dir); } - assertThat(tableDetails.computeSizeOnDiskBytes()).isGreaterThan(0L); - assertThat(tableDetails.computeSizeOnDiskBytes()).isEqualTo(res); + try (MockedStatic fileUtilsMock = mockStatic(FileUtils.class)) + { + fileUtilsMock.when(() -> FileUtils.folderSize(any())).thenCallRealMethod(); + + assertThat(tableDetails.computeSizeOnDiskBytes()).isGreaterThan(0L); + assertThat(tableDetails.computeSizeOnDiskBytes()).isEqualTo(res); + + // when we invoke computeSizeOnDiskBytes for the second time, it will use cached value + // 3 invocations for folderSize are from the first invocation of computeSizeOnDiskBytes because + // we have 3 data dirs, if we have not cached it, the number of invocations would be 6. + fileUtilsMock.verify(() -> FileUtils.folderSize(any()), times(3)); + } } @Test @@ -208,15 +279,27 @@ public class TableSnapshotTest Long res = 0L; + Set files = new HashSet<>(); for (File dir : folders) { File file = new File(dir, "tmp"); + files.add(file.toAbsolute().toString()); writeBatchToFile(file); res += file.length(); + res += new File(dir, "manifest.json").length(); + res += new File(dir, "schema.cql").length(); } - assertThat(tableDetails.computeTrueSizeBytes()).isGreaterThan(0L); - assertThat(tableDetails.computeTrueSizeBytes()).isEqualTo(res); + try (MockedStatic tableSnapshotMock = mockStatic(TableSnapshot.class)) + { + tableSnapshotMock.when(() -> TableSnapshot.getLiveFileFromSnapshotFile(any())).thenCallRealMethod(); + + assertThat(tableDetails.computeTrueSizeBytes(files)).isGreaterThan(0L); + assertThat(tableDetails.computeTrueSizeBytes(files)).isEqualTo(res); + + // 6 because we avoided to call it for manifest and schema because they are cached + tableSnapshotMock.verify(() -> TableSnapshot.getLiveFileFromSnapshotFile(any()), times(6)); + } } @Test @@ -298,7 +381,7 @@ public class TableSnapshotTest // 1. snapshot to clear is not ephemeral // 2. tag to clear is null, empty, or it is equal to snapshot tag // 3. byTimestamp is true - if (TableSnapshot.shouldClearSnapshot(testingTag, olderThanTimestamp).test(snapshot)) + if (ClearSnapshotTask.getClearSnapshotPredicate(testingTag, Set.of(keyspace), olderThanTimestamp, false).test(snapshot)) { // shouldClearTag = true boolean shouldClearTag = (testingTag == null || testingTag.isEmpty()) || snapshot.getTag().equals(testingTag); @@ -321,23 +404,4 @@ public class TableSnapshotTest } } } - - @Test - public void testGetLiveFileFromSnapshotFile() - { - testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/me-1-big-Data.db", - "~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/me-1-big-Data.db"); - } - - @Test - public void testGetLiveFileFromSnapshotIndexFile() - { - testGetLiveFileFromSnapshotFile("~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/snapshots/1643481737850/.tbl_val_idx/me-1-big-Summary.db", - "~/.ccm/test/node1/data0/test_ks/tbl-e03faca0813211eca100c705ea09b5ef/.tbl_val_idx/me-1-big-Summary.db"); - } - - public void testGetLiveFileFromSnapshotFile(String snapshotFile, String expectedLiveFile) - { - assertThat(TableSnapshot.getLiveFileFromSnapshotFile(Paths.get(snapshotFile)).toString()).isEqualTo(expectedLiveFile); - } } diff --git a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java index 13705f9b34..99655ed32e 100644 --- a/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java +++ b/test/unit/org/apache/cassandra/tools/StandaloneUpgraderOnSStablesTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.tools; -import java.util.Collections; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -34,7 +33,7 @@ import org.apache.cassandra.distributed.shared.WithProperties; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.StartupException; import org.apache.cassandra.io.sstable.LegacySSTableTest; -import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.tools.ToolRunner.ToolResult; import org.assertj.core.api.Assertions; @@ -93,9 +92,7 @@ public class StandaloneUpgraderOnSStablesTest { LegacySSTableTest.truncateLegacyTables(legacyId); LegacySSTableTest.loadLegacyTables(legacyId); - StorageService.instance.takeSnapshot("testsnapshot", - Collections.emptyMap(), - "legacy_tables.legacy_" + legacyId + "_simple"); + SnapshotManager.instance.takeSnapshot("testsnapshot", "legacy_tables.legacy_" + legacyId + "_simple"); ToolResult tool = ToolRunner.invokeClass(StandaloneUpgrader.class, "-k", diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java index 379d02a66e..b0de9d32cd 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java @@ -30,6 +30,7 @@ import java.util.regex.Pattern; import javax.management.openmbean.TabularData; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -37,6 +38,7 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManager; import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.ToolRunner.ToolResult; @@ -65,6 +67,12 @@ public class ClearSnapshotTest extends CQLTester probe = new NodeProbe(jmxHost, jmxPort); } + @Before + public void clearAllSnapshots() + { + SnapshotManager.instance.clearAllSnapshots(); + } + @AfterClass public static void teardown() throws IOException { @@ -315,11 +323,15 @@ public class ClearSnapshotTest extends CQLTester String tableId2 = DASH_PATTERN.matcher(tableMetadata2.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", keyspace2, tableName2))) .id.asUUID().toString()).replaceAll(""); + SnapshotManager.instance.close(); + rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "snapshot-to-clear-ks1-tb1", start.minus(5, HOURS)); rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "some-other-snapshot-ks1-tb1", start.minus(2, HOURS)); rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "last-snapshot-ks1-tb1", start.minus(1, SECONDS)); rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "snapshot-to-clear-ks2-tb2", start.minus(5, HOURS)); rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "some-other-snapshot-ks2-tb2", start.minus(2, HOURS)); rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "last-snapshot-ks2-tb2", start.minus(1, SECONDS)); + + SnapshotManager.instance.start(true); } } diff --git a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java index 426ccae58f..eb741c6915 100644 --- a/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java +++ b/tools/stress/src/org/apache/cassandra/stress/CompactionStress.java @@ -340,7 +340,7 @@ public abstract class CompactionStress implements Runnable } double currentSizeGiB; - while ((currentSizeGiB = directories.getRawDiretoriesSize() / BYTES_IN_GIB) < totalSizeGiB) + while ((currentSizeGiB = directories.getRawDirectoriesSize() / BYTES_IN_GIB) < totalSizeGiB) { if (finished.getCount() == 0) break; @@ -353,7 +353,7 @@ public abstract class CompactionStress implements Runnable workManager.stop(); Uninterruptibles.awaitUninterruptibly(finished); - currentSizeGiB = directories.getRawDiretoriesSize() / BYTES_IN_GIB; + currentSizeGiB = directories.getRawDirectoriesSize() / BYTES_IN_GIB; System.out.println(String.format("Finished writing %.2fGB", currentSizeGiB)); } }