From 872e34c2d613a43b2c2be75efa016b6943bf87ad Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Mon, 15 Aug 2022 13:56:58 +0200 Subject: [PATCH] add --older-than and --older-than-timestamp options for nodetool clearsnapshot patch by Stefan Miklosovic; reviewed by Paulo Motta for CASSANDRA-16860 --- CHANGES.txt | 1 + NEWS.txt | 4 + .../cassandra/service/StorageService.java | 45 ++- .../service/StorageServiceMBean.java | 14 + .../service/snapshot/TableSnapshot.java | 15 +- .../org/apache/cassandra/tools/NodeProbe.java | 24 +- .../tools/nodetool/ClearSnapshot.java | 66 ++++- .../cassandra/db/SystemKeyspaceTest.java | 8 +- .../service/snapshot/TableSnapshotTest.java | 81 ++++- .../tools/nodetool/ClearSnapshotTest.java | 279 +++++++++++++++--- 10 files changed, 480 insertions(+), 57 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bda1ba04d7..f35e0fe599 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.2 + * Add --older-than and --older-than-timestamp options for nodetool clearsnapshots (CASSANDRA-16860) * Fix "open RT bound as its last item" exception (CASSANDRA-17810) * Fix leak of non-standard Java types in JMX MBeans `org.apache.cassandra.db:type=StorageService` and `org.apache.cassandra.db:type=RepairService` as clients using JMX cannot handle them. More details in NEWS.txt (CASSANDRA-17668) diff --git a/NEWS.txt b/NEWS.txt index d9c2f20942..727e711ec4 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -88,6 +88,10 @@ New features - Added new CQL table property 'allow_auto_snapshot' which is by default true. When set to false and 'auto_snapshot: true' in cassandra.yaml, there will be no snapshot taken when a table is truncated or dropped. When auto_snapshot in casandra.yaml is set to false, the newly added table property does not have any effect. + - Added --older-than and --older-than-timestamp options to nodetool clearsnapshot command. It is possible to + clear snapshots which are older than some period for example, "--older-than 5h" to remove + snapshots older than 5 hours and it is possible to clear all snapshots older than some timestamp, for example + --older-than-timestamp 2022-12-03T10:15:30Z. Upgrading --------- diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 1b1b5a8f1d..a8b27bfc22 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -27,6 +27,7 @@ import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.file.Paths; import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; @@ -4198,13 +4199,21 @@ public class StorageService extends NotificationBroadcasterSupport implements IE */ public void clearSnapshot(String tag, String... keyspaceNames) { - if(tag == null) + clearSnapshot(Collections.emptyMap(), tag, keyspaceNames); + } + + public void clearSnapshot(Map options, String tag, String... keyspaceNames) + { + if (tag == null) tag = ""; + if (options == null) + options = Collections.emptyMap(); + Set keyspaces = new HashSet<>(); for (String dataDir : DatabaseDescriptor.getAllDataFileLocations()) { - for(String keyspaceDir : new File(dataDir).tryListNames()) + for (String keyspaceDir : new File(dataDir).tryListNames()) { // Only add a ks if it has been specified as a param, assuming params were actually provided. if (keyspaceNames.length > 0 && !Arrays.asList(keyspaceNames).contains(keyspaceDir)) @@ -4213,8 +4222,32 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } } + Object olderThan = options.get("older_than"); + Object olderThanTimestamp = options.get("older_than_timestamp"); + + final long clearOlderThanTimestamp; + if (olderThan != null) + { + assert olderThan instanceof String : "it is expected that older_than is an instance of java.lang.String"; + clearOlderThanTimestamp = Clock.Global.currentTimeMillis() - new DurationSpec.LongSecondsBound((String) olderThan).toMilliseconds(); + } + else if (olderThanTimestamp != null) + { + assert olderThanTimestamp instanceof String : "it is expected that older_than_timestamp is an instance of java.lang.String"; + try + { + clearOlderThanTimestamp = Instant.parse((String) olderThanTimestamp).toEpochMilli(); + } + catch (DateTimeParseException ex) + { + throw new RuntimeException("Parameter older_than_timestamp has to be a valid instant in ISO format."); + } + } + else + clearOlderThanTimestamp = 0L; + for (String keyspace : keyspaces) - clearKeyspaceSnapshot(keyspace, tag); + clearKeyspaceSnapshot(keyspace, tag, clearOlderThanTimestamp); if (logger.isDebugEnabled()) logger.debug("Cleared out snapshot directories"); @@ -4224,12 +4257,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE * Clear snapshots for a given keyspace. * @param keyspace keyspace to remove snapshots for * @param tag the user supplied snapshot name. If empty or null, all the snapshots will be cleaned + * @param olderThanTimestamp if a snapshot was created before this timestamp, it will be cleared, + * if its value is 0, this parameter is effectively ignored. */ - private void clearKeyspaceSnapshot(String keyspace, String tag) + private void clearKeyspaceSnapshot(String keyspace, String tag, long olderThanTimestamp) { Set snapshotsToClear = new SnapshotLoader().loadSnapshots(keyspace) .stream() - .filter(TableSnapshot.shouldClearSnapshot(tag)) + .filter(TableSnapshot.shouldClearSnapshot(tag, olderThanTimestamp)) .collect(Collectors.toSet()); for (TableSnapshot snapshot : snapshotsToClear) snapshotManager.clearSnapshot(snapshot); diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 10efb02039..be594fc950 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -270,9 +270,23 @@ public interface StorageServiceMBean extends NotificationEmitter /** * Remove the snapshot with the given name from the given keyspaces. * If no tag is specified we will remove all snapshots. + * + * @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared + * @param keyspaceNames name of keyspaces to clear snapshots for */ + @Deprecated public void clearSnapshot(String tag, String... keyspaceNames) throws IOException; + /** + * Remove the snapshot with the given name from the given keyspaces. + * If no tag is specified we will remove all snapshots. + * + * @param options map of options for cleanup operation, consult nodetool's ClearSnapshot + * @param tag name of snapshot to clear, if null or empty string, all snapshots of given keyspace will be cleared + * @param keyspaceNames name of keyspaces to clear snapshots for + */ + public void clearSnapshot(Map options, String tag, String... keyspaceNames) throws IOException; + /** * Get the details of all the snapshot * @return A map of snapshotName to all its details in Tabular form. diff --git a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java index b562d9c59f..77de32ccbb 100644 --- a/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java +++ b/src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java @@ -325,7 +325,7 @@ public class TableSnapshot return new File(liveDir.toString(), snapshotFilePath.getFileName().toString()); } - public static Predicate shouldClearSnapshot(String tag) + public static Predicate shouldClearSnapshot(String tag, long olderThanTimestamp) { return ts -> { @@ -335,7 +335,18 @@ public class TableSnapshot logger.info("Skipping deletion of ephemeral snapshot '{}' in keyspace {}. " + "Ephemeral snapshots are not removable by a user.", tag, ts.keyspaceName); - return !ts.isEphemeral() && (clearAll || ts.tag.equals(tag)); + boolean notEphemeral = !ts.isEphemeral(); + boolean shouldClearTag = clearAll || ts.tag.equals(tag); + boolean byTimestamp = true; + + if (olderThanTimestamp > 0L) + { + Instant createdAt = ts.getCreatedAt(); + if (createdAt != null) + byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp)); + } + + return notEphemeral && shouldClearTag && byTimestamp; }; } diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 60edad3d1f..b1336ddd5d 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -852,11 +852,31 @@ public class NodeProbe implements AutoCloseable } /** - * Remove all the existing snapshots. + * Remove all the existing snapshots of given tag for provided keyspaces. + * When no keyspaces are specified, take all keyspaces into account. When tag is not specified (null or empty string), + * take all tags into account. + * + * @param tag tag of snapshot to clear + * @param keyspaces keyspaces to clear snapshots for */ + @Deprecated public void clearSnapshot(String tag, String... keyspaces) throws IOException { - ssProxy.clearSnapshot(tag, keyspaces); + clearSnapshot(Collections.emptyMap(), tag, keyspaces); + } + + /** + * Remove all the existing snapshots of given tag for provided keyspaces. + * When no keyspaces are specified, take all keyspaces into account. When tag is not specified (null or empty string), + * take all tags into account. + * + * @param options options to supply for snapshot clearing + * @param tag tag of snapshot to clear + * @param keyspaces keyspaces to clear snapshots for + */ + public void clearSnapshot(Map options, String tag, String... keyspaces) throws IOException + { + ssProxy.clearSnapshot(options, tag, keyspaces); } public Map getSnapshotDetails(Map options) diff --git a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java index fb69b25306..866b85d3d5 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java +++ b/src/java/org/apache/cassandra/tools/nodetool/ClearSnapshot.java @@ -25,9 +25,14 @@ import io.airlift.airline.Command; import io.airlift.airline.Option; import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.tools.NodeProbe; import org.apache.cassandra.tools.NodeTool.NodeToolCmd; @@ -43,14 +48,45 @@ public class ClearSnapshot extends NodeToolCmd @Option(title = "clear_all_snapshots", name = "--all", description = "Removes all snapshots") private boolean clearAllSnapshots = false; + @Option(title = "older_than", name = "--older-than", description = "Clear snapshots older than specified time period.") + private String olderThan; + + @Option(title = "older_than_timestamp", name = "--older-than-timestamp", + description = "Clear snapshots older than specified timestamp. It has to be a string in ISO format, for example '2022-12-03T10:15:30Z'") + private String olderThanTimestamp; + @Override public void execute(NodeProbe probe) { - if(snapshotName.isEmpty() && !clearAllSnapshots) - throw new RuntimeException("Specify snapshot name or --all"); + if (snapshotName.isEmpty() && !clearAllSnapshots) + throw new IllegalArgumentException("Specify snapshot name or --all"); - if(!snapshotName.isEmpty() && clearAllSnapshots) - throw new RuntimeException("Specify only one of snapshot name or --all"); + if (!snapshotName.isEmpty() && clearAllSnapshots) + throw new IllegalArgumentException("Specify only one of snapshot name or --all"); + + if (olderThan != null && olderThanTimestamp != null) + throw new IllegalArgumentException("Specify only one of --older-than or --older-than-timestamp"); + + if (!snapshotName.isEmpty() && olderThan != null) + throw new IllegalArgumentException("Specifying snapshot name together with --older-than flag is not allowed"); + + if (!snapshotName.isEmpty() && olderThanTimestamp != null) + throw new IllegalArgumentException("Specifying snapshot name together with --older-than-timestamp flag is not allowed"); + + if (olderThanTimestamp != null) + try + { + Instant.parse(olderThanTimestamp); + } + catch (DateTimeParseException ex) + { + throw new IllegalArgumentException("Parameter --older-than-timestamp has to be a valid instant in ISO format."); + } + + Long olderThanInSeconds = null; + if (olderThan != null) + // fail fast when it is not valid + olderThanInSeconds = new DurationSpec.LongSecondsBound(olderThan).toSeconds(); StringBuilder sb = new StringBuilder(); @@ -59,18 +95,32 @@ public class ClearSnapshot extends NodeToolCmd if (keyspaces.isEmpty()) sb.append("[all keyspaces]"); else - sb.append("[").append(join(keyspaces, ", ")).append("]"); + sb.append('[').append(join(keyspaces, ", ")).append(']'); if (snapshotName.isEmpty()) sb.append(" with [all snapshots]"); else - sb.append(" with snapshot name [").append(snapshotName).append("]"); + sb.append(" with snapshot name [").append(snapshotName).append(']'); - probe.output().out.println(sb.toString()); + if (olderThanInSeconds != null) + sb.append(" older than ") + .append(olderThanInSeconds) + .append(" seconds."); + + if (olderThanTimestamp != null) + sb.append(" older than timestamp ").append(olderThanTimestamp); + + probe.output().out.println(sb); try { - probe.clearSnapshot(snapshotName, toArray(keyspaces, String.class)); + Map parameters = new HashMap<>(); + if (olderThan != null) + parameters.put("older_than", olderThan); + if (olderThanTimestamp != null) + parameters.put("older_than_timestamp", olderThanTimestamp); + + probe.clearSnapshot(parameters, snapshotName, toArray(keyspaces, String.class)); } catch (IOException e) { throw new RuntimeException("Error during clearing snapshots", e); diff --git a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java index db58fd6d49..0bade4bee8 100644 --- a/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java +++ b/test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java @@ -106,14 +106,14 @@ public class SystemKeyspaceTest // First, check that in the absence of any previous installed version, we don't create snapshots for (ColumnFamilyStore cfs : Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStores()) cfs.clearUnsafe(); - StorageService.instance.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); SystemKeyspace.snapshotOnVersionChange(); assertDeleted(); // now setup system.local as if we're upgrading from a previous version setupReleaseVersion(getOlderVersionString()); - StorageService.instance.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); assertDeleted(); // Compare versions again & verify that snapshots were created for all tables in the system ks @@ -126,7 +126,7 @@ public class SystemKeyspaceTest // clear out the snapshots & set the previous recorded version equal to the latest, we shouldn't // see any new snapshots created this time. - StorageService.instance.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); setupReleaseVersion(FBUtilities.getReleaseVersionString()); SystemKeyspace.snapshotOnVersionChange(); @@ -135,7 +135,7 @@ public class SystemKeyspaceTest // 10 files expected. assertDeleted(); - StorageService.instance.clearSnapshot(null, SchemaConstants.SYSTEM_KEYSPACE_NAME); + StorageService.instance.clearSnapshot(Collections.emptyMap(), null, SchemaConstants.SYSTEM_KEYSPACE_NAME); } @Test diff --git a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java index c1614dffdc..d592dd91d2 100644 --- a/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java +++ b/test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java @@ -21,8 +21,10 @@ package org.apache.cassandra.service.snapshot; import java.io.IOException; import java.nio.file.Paths; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -35,9 +37,11 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.utils.FBUtilities.now; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertTrue; public class TableSnapshotTest { @@ -50,15 +54,18 @@ public class TableSnapshotTest @ClassRule public static TemporaryFolder tempFolder = new TemporaryFolder(); - public static Set createFolders(TemporaryFolder temp) throws IOException { + public static Set createFolders(TemporaryFolder temp) throws IOException + { File folder = new File(temp.newFolder()); Set folders = new HashSet<>(); - for (String folderName : Arrays.asList("foo", "bar", "buzz")) { + for (String folderName : Arrays.asList("foo", "bar", "buzz")) + { File subfolder = new File(folder, folderName); subfolder.tryCreateDirectories(); assertThat(subfolder.exists()); folders.add(subfolder); - }; + } + ; return folders; } @@ -247,6 +254,74 @@ public class TableSnapshotTest assertThat(withoutCreatedAt.getCreatedAt()).isEqualTo(Instant.ofEpochMilli(folders.stream().mapToLong(f -> f.lastModified()).min().getAsLong())); } + @Test + public void testShouldClearSnapshot() throws Exception + { + // TableSnapshot variables -> ephemeral / true / false, createdAt -> null / notnull + + Instant now = Instant.now(); + + String keyspace = "ks"; + String table = "tbl"; + UUID id = UUID.randomUUID(); + String tag = "someTag"; + Instant snapshotCreation = now.minusSeconds(60); + Set folders = createFolders(tempFolder); + + List snapshots = new ArrayList<>(); + + for (boolean ephemeral : new boolean[]{ true, false }) + for (Instant createdAt : new Instant[]{ snapshotCreation, null }) + snapshots.add(new TableSnapshot(keyspace, + table, + id, + tag, + createdAt, // variable + null, + folders, + ephemeral)); // variable + + List> testingMethodInputs = new ArrayList<>(); + + for (String testingTag : new String[] {null, "", tag, "someothertag"}) + // 0 to deactive byTimestamp logic, now.toEpochMilli as true, snapshot minus 60s as false + for (long olderThanTimestamp : new long[] {0, now.toEpochMilli(), snapshotCreation.minusSeconds(60).toEpochMilli()}) + testingMethodInputs.add(Pair.create(testingTag, olderThanTimestamp)); + + for (Pair methodInput : testingMethodInputs) + { + String testingTag = methodInput.left(); + Long olderThanTimestamp = methodInput.right; + for (TableSnapshot snapshot : snapshots) + { + // if shouldClear method returns true, it is only in case + // 1. snapshot to clear is not ephemeral + // 2. tag to clear is null, empty, or it is equal to snapshot tag + // 3. byTimestamp is true + if (TableSnapshot.shouldClearSnapshot(testingTag, olderThanTimestamp).test(snapshot)) + { + // shouldClearTag = true + boolean shouldClearTag = (testingTag == null || testingTag.isEmpty()) || snapshot.getTag().equals(testingTag); + // notEphemeral + boolean notEphemeral = !snapshot.isEphemeral(); + // byTimestamp + boolean byTimestamp = true; + + if (olderThanTimestamp > 0L) + { + Instant createdAt = snapshot.getCreatedAt(); + if (createdAt != null) + byTimestamp = createdAt.isBefore(Instant.ofEpochMilli(olderThanTimestamp)); + } + + assertTrue(notEphemeral); + assertTrue(shouldClearTag); + assertTrue(byTimestamp); + } + } + } + } + @Test public void testGetLiveFileFromSnapshotFile() { diff --git a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java index 0e172f3bce..d58d4bb5da 100644 --- a/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java +++ b/test/unit/org/apache/cassandra/tools/nodetool/ClearSnapshotTest.java @@ -19,7 +19,14 @@ package org.apache.cassandra.tools.nodetool; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import javax.management.openmbean.TabularData; import org.junit.AfterClass; @@ -27,13 +34,27 @@ import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.snapshot.SnapshotManifest; import org.apache.cassandra.tools.NodeProbe; -import org.apache.cassandra.tools.ToolRunner; +import org.apache.cassandra.tools.ToolRunner.ToolResult; +import static java.lang.String.format; +import static java.time.temporal.ChronoUnit.HOURS; +import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Collections.emptyMap; +import static org.apache.cassandra.config.DatabaseDescriptor.getAllDataFileLocations; +import static org.apache.cassandra.tools.ToolRunner.invokeNodetool; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertTrue; public class ClearSnapshotTest extends CQLTester { + private static final Pattern DASH_PATTERN = Pattern.compile("-"); private static NodeProbe probe; @BeforeClass @@ -49,63 +70,255 @@ public class ClearSnapshotTest extends CQLTester probe.close(); } - @Test - public void testClearSnapshot_NoArgs() - { - ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clearsnapshot"); - assertThat(tool.getExitCode()).isEqualTo(2); - assertThat(tool.getCleanedStderr()).contains("Specify snapshot name or --all"); - - tool = ToolRunner.invokeNodetool("clearsnapshot", "--all"); - tool.assertOnCleanExit(); - } - - @Test - public void testClearSnapshot_AllAndName() - { - ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("clearsnapshot", "-t", "some-name", "--all"); - assertThat(tool.getExitCode()).isEqualTo(2); - assertThat(tool.getCleanedStderr()).contains("Specify only one of snapshot name or --all"); - } - @Test public void testClearSnapshot_RemoveByName() { - ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("snapshot","-t","some-name"); + ToolResult tool = invokeNodetool("snapshot", "-t", "some-name"); tool.assertOnCleanExit(); assertThat(tool.getStdout()).isNotEmpty(); - - Map snapshots_before = probe.getSnapshotDetails(); + + Map snapshots_before = probe.getSnapshotDetails(emptyMap()); assertThat(snapshots_before).containsKey("some-name"); - tool = ToolRunner.invokeNodetool("clearsnapshot","-t","some-name"); + tool = invokeNodetool("clearsnapshot", "-t", "some-name"); tool.assertOnCleanExit(); assertThat(tool.getStdout()).isNotEmpty(); - - Map snapshots_after = probe.getSnapshotDetails(); + + Map snapshots_after = probe.getSnapshotDetails(emptyMap()); assertThat(snapshots_after).doesNotContainKey("some-name"); } @Test public void testClearSnapshot_RemoveMultiple() { - ToolRunner.ToolResult tool = ToolRunner.invokeNodetool("snapshot","-t","some-name"); + ToolResult tool = invokeNodetool("snapshot", "-t", "some-name"); tool.assertOnCleanExit(); assertThat(tool.getStdout()).isNotEmpty(); - tool = ToolRunner.invokeNodetool("snapshot","-t","some-other-name"); + tool = invokeNodetool("snapshot", "-t", "some-other-name"); tool.assertOnCleanExit(); assertThat(tool.getStdout()).isNotEmpty(); - Map snapshots_before = probe.getSnapshotDetails(); + Map snapshots_before = probe.getSnapshotDetails(emptyMap()); assertThat(snapshots_before).hasSize(2); - tool = ToolRunner.invokeNodetool("clearsnapshot","--all"); + tool = invokeNodetool("clearsnapshot", "--all"); tool.assertOnCleanExit(); assertThat(tool.getStdout()).isNotEmpty(); - - Map snapshots_after = probe.getSnapshotDetails(); + + Map snapshots_after = probe.getSnapshotDetails(emptyMap()); assertThat(snapshots_after).isEmpty(); } - + + @Test + public void testClearSnapshotWithOlderThanFlag() throws Throwable + { + Instant start = Instant.ofEpochMilli(currentTimeMillis()); + prepareData(start); + + // wait 10 seconds for the sake of the test + await().timeout(15, TimeUnit.SECONDS).until(() -> Instant.now().isAfter(start.plusSeconds(10))); + + // clear all snapshots for specific keyspace older than 3 hours for a specific keyspace + invokeNodetool("clearsnapshot", "--older-than", "3h", "--all", "--", KEYSPACE).assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + return !output.contains("snapshot-to-clear-ks1-tb1") && + output.contains("some-other-snapshot-ks1-tb1") && + output.contains("last-snapshot-ks1-tb1") && + output.contains("snapshot-to-clear-ks2-tb2") && + output.contains("some-other-snapshot-ks2-tb2") && + output.contains("last-snapshot-ks2-tb2"); + }); + + // clear all snapshots older than 2 hours for all keyspaces + invokeNodetool("clearsnapshot", "--older-than", "2h", "--all").assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + + return !output.contains("some-other-snapshot-ks1-tb1") && + output.contains("last-snapshot-ks1-tb1") && + !output.contains("snapshot-to-clear-ks2-tb2") && + !output.contains("some-other-snapshot-ks2-tb2") && + output.contains("last-snapshot-ks2-tb2"); + }); + + // clear all snapshosts older than 1 second + invokeNodetool("clearsnapshot", "--older-than", "1s", "--all", "--", currentKeyspace()).assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + return output.contains("last-snapshot-ks1-tb1") && + !output.contains("last-snapshot-ks2-tb2"); + }); + + invokeNodetool("clearsnapshot", "--older-than", "1s", "--all").assertOnCleanExit(); + await().until(() -> !invokeNodetool("listsnapshots").getStdout().contains("last-snapshot-ks1-tb1")); + } + + + @Test + public void testClearSnapshotWithOlderThanTimestampFlag() throws Throwable + { + Instant start = Instant.ofEpochMilli(currentTimeMillis()); + prepareData(start); + + // wait 10 seconds for the sake of the test + await().timeout(15, TimeUnit.SECONDS).until(() -> Instant.now().isAfter(start.plusSeconds(10))); + + // clear all snapshots for specific keyspace older than 3 hours for a specific keyspace + invokeNodetool("clearsnapshot", "--older-than-timestamp", + Instant.now().minus(3, HOURS).toString(), + "--all", "--", KEYSPACE).assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + return !output.contains("snapshot-to-clear-ks1-tb1") && + output.contains("some-other-snapshot-ks1-tb1") && + output.contains("last-snapshot-ks1-tb1") && + output.contains("snapshot-to-clear-ks2-tb2") && + output.contains("some-other-snapshot-ks2-tb2") && + output.contains("last-snapshot-ks2-tb2"); + }); + + // clear all snapshots older than 2 hours for all keyspaces + invokeNodetool("clearsnapshot", "--older-than-timestamp", + Instant.now().minus(2, HOURS).toString(), + "--all").assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + + return !output.contains("some-other-snapshot-ks1-tb1") && + output.contains("last-snapshot-ks1-tb1") && + !output.contains("snapshot-to-clear-ks2-tb2") && + !output.contains("some-other-snapshot-ks2-tb2") && + output.contains("last-snapshot-ks2-tb2"); + }); + + // clear all snapshots older than now for all keyspaces + invokeNodetool("clearsnapshot", "--older-than-timestamp", + Instant.now().toString(), + "--all").assertOnCleanExit(); + + await().until(() -> { + String output = invokeNodetool("listsnapshots").getStdout(); + return !output.contains("last-snapshot-ks1-tb1") && + !output.contains("last-snapshot-ks2-tb2"); + }); + } + + @Test + public void testIncompatibleFlags() + { + ToolResult invalidCommand1 = invokeNodetool("clearsnapshot", + "--older-than-timestamp", Instant.now().toString(), + "--older-than", "3h", + "--all"); + invalidCommand1.asserts().failure(); + assertTrue(invalidCommand1.getStdout().contains("Specify only one of --older-than or --older-than-timestamp")); + + ToolResult invalidCommand2 = invokeNodetool("clearsnapshot", "-t", "some-snapshot-tag", "--all"); + invalidCommand2.asserts().failure(); + assertTrue(invalidCommand2.getStdout().contains("Specify only one of snapshot name or --all")); + + ToolResult invalidCommand3 = invokeNodetool("clearsnapshot", "--", "keyspace"); + invalidCommand3.asserts().failure(); + assertTrue(invalidCommand3.getStdout().contains("Specify snapshot name or --all")); + + ToolResult invalidCommand4 = invokeNodetool("clearsnapshot", + "--older-than-timestamp", Instant.now().toString(), + "-t", "some-snapshot-tag"); + invalidCommand4.asserts().failure(); + assertTrue(invalidCommand4.getStdout().contains("Specifying snapshot name together with --older-than-timestamp flag is not allowed")); + + ToolResult invalidCommand5 = invokeNodetool("clearsnapshot", + "--older-than", "3h", + "-t", "some-snapshot-tag"); + invalidCommand5.asserts().failure(); + assertTrue(invalidCommand5.getStdout().contains("Specifying snapshot name together with --older-than flag is not allowed")); + + ToolResult invalidCommand6 = invokeNodetool("clearsnapshot", + "--older-than-timestamp", "123", + "--all", "--", "somekeyspace"); + invalidCommand6.asserts().failure(); + assertTrue(invalidCommand6.getStdout().contains("Parameter --older-than-timestamp has to be a valid instant in ISO format.")); + + ToolResult invalidCommand7 = invokeNodetool("clearsnapshot", + "--older-than", "3k", + "--all", "--", "somekeyspace"); + invalidCommand7.asserts().failure(); + assertTrue(invalidCommand7.getStdout().contains("Invalid duration: 3k")); + } + + private void rewriteManifest(String tableId, + String[] dataDirs, + String keyspace, + String tableName, + String snapshotName, + Instant createdAt) throws Exception + { + Path manifestPath = findManifest(dataDirs, keyspace, tableId, tableName, snapshotName); + SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(new File(manifestPath)); + SnapshotManifest manifestWithEphemeralFlag = new SnapshotManifest(manifest.files, null, createdAt, false); + manifestWithEphemeralFlag.serializeToJsonFile(new File(manifestPath)); + } + + private Path findManifest(String[] dataDirs, String keyspace, String tableId, String tableName, String snapshotName) + { + for (String dataDir : dataDirs) + { + Path manifest = Paths.get(dataDir) + .resolve(keyspace) + .resolve(format("%s-%s", tableName, tableId)) + .resolve("snapshots") + .resolve(snapshotName) + .resolve("manifest.json"); + + if (Files.exists(manifest)) + { + return manifest; + } + } + + throw new IllegalStateException("Unable to find manifest!"); + } + + private void prepareData(Instant start) throws Throwable + { + String tableName = createTable(KEYSPACE, "CREATE TABLE %s (id int primary key)"); + execute("INSERT INTO %s (id) VALUES (?)", 1); + flush(KEYSPACE); + + String keyspace2 = createKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}"); + String tableName2 = createTable(keyspace2, "CREATE TABLE %s (id int primary key)"); + execute(formatQuery(keyspace2, "INSERT INTO %s (id) VALUES (?)"), 1); + flush(keyspace2); + + invokeNodetool("snapshot", "-t", "snapshot-to-clear-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit(); + invokeNodetool("snapshot", "-t", "some-other-snapshot-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit(); + invokeNodetool("snapshot", "-t", "last-snapshot-ks1-tb1", "-cf", tableName, "--", KEYSPACE).assertOnCleanExit(); + + invokeNodetool("snapshot", "-t", "snapshot-to-clear-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit(); + invokeNodetool("snapshot", "-t", "some-other-snapshot-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit(); + invokeNodetool("snapshot", "-t", "last-snapshot-ks2-tb2", "-cf", tableName2, "--", keyspace2).assertOnCleanExit(); + + Optional tableMetadata = Keyspace.open(KEYSPACE).getMetadata().tables.get(tableName); + Optional tableMetadata2 = Keyspace.open(keyspace2).getMetadata().tables.get(tableName2); + + String tableId = DASH_PATTERN.matcher(tableMetadata.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", KEYSPACE, tableName))) + .id.asUUID().toString()).replaceAll(""); + + String tableId2 = DASH_PATTERN.matcher(tableMetadata2.orElseThrow(() -> new IllegalStateException(format("no metadata found for %s.%s", keyspace2, tableName2))) + .id.asUUID().toString()).replaceAll(""); + + rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "snapshot-to-clear-ks1-tb1", start.minus(5, HOURS)); + rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "some-other-snapshot-ks1-tb1", start.minus(2, HOURS)); + rewriteManifest(tableId, getAllDataFileLocations(), KEYSPACE, tableName, "last-snapshot-ks1-tb1", start.minus(1, SECONDS)); + rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "snapshot-to-clear-ks2-tb2", start.minus(5, HOURS)); + rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "some-other-snapshot-ks2-tb2", start.minus(2, HOURS)); + rewriteManifest(tableId2, getAllDataFileLocations(), keyspace2, tableName2, "last-snapshot-ks2-tb2", start.minus(1, SECONDS)); + } }