natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove ephemeral snapshot marker file and introduce a flag to SnapshotManifest 

patch by Stefan Miklosovic; reviewed by Paulo Motta for CASSANDRA-16911
This commit is contained in:
Stefan Miklosovic 2022-06-24 12:42:12 +02:00
parent 1c2cd30125
commit 55f094a6d2
19 changed files with 425 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.2
* Remove ephemeral snapshot marker file and introduce a flag to SnapshotManifest (CASSANDRA-16911)
* Add a virtual table that exposes currently running queries (CASSANDRA-15241)
* Allow sstableloader to specify table without relying on path (CASSANDRA-16584)
* Fix TestGossipingPropertyFileSnitch.test_prefer_local_reconnect_on_listen_address (CASSANDRA-17700)

5
NEWS.txt
View File

@ -70,9 +70,14 @@ New features
- Whether ALTER TABLE commands are allowed to mutate columns
- Whether SimpleStrategy is allowed on keyspace creation or alteration
- Maximum replication factor
- It is possible to list ephemeral snapshots by nodetool listsnaphots command when flag "-e" is specified.
Upgrading
---------
- Emphemeral marker files for snapshots done by repairs are not created anymore,
there is a dedicated flag in snapshot manifest instead. On upgrade of a node to version 4.2, on node's start, in case there
are such ephemeral snapshots on disk, they will be deleted (same behaviour as before) and any new ephemeral snapshots
will stop to create ephemeral marker files as flag in a snapshot manifest was introduced instead.
Deprecation
-----------

53
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -22,7 +22,7 @@ import java.io.PrintStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@ -155,6 +155,7 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.TablePaxosRepairHistory;
import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.service.snapshot.TableSnapshot;
import org.apache.cassandra.streaming.TableStreamManager;
@ -1698,9 +1699,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
/**
* Rewrites all SSTables according to specified parameters
*
* @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link BigFormat#latestVersion})
* @param skipIfCurrentVersion - if {@link true}, will rewrite only SSTables that have version older than the current one ({@link org.apache.cassandra.io.sstable.format.big.BigFormat#latestVersion})
* @param skipIfNewerThanTimestamp - max timestamp (local creation time) for SSTable; SSTables created _after_ this timestamp will be excluded from compaction
* @param skipIfCompressionMatches - if {@link true}, will rewrite only SSTables whose compression parameters are different from {@link CFMetaData#compressionParams()}
* @param skipIfCompressionMatches - if {@link true}, will rewrite only SSTables whose compression parameters are different from {@link TableMetadata#params#getCompressionParameters()} ()}
* @param jobs number of jobs for parallel execution
*/
public CompactionManager.AllSSTableOpStatus sstablesRewrite(final boolean skipIfCurrentVersion,
@ -2039,7 +2040,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
.collect(Collectors.toCollection(HashSet::new));
// Create and write snapshot manifest
SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime);
SnapshotManifest manifest = new SnapshotManifest(mapToDataFilenames(sstables), ttl, creationTime, ephemeral);
File manifestFile = getDirectories().getSnapshotManifestFile(tag);
writeSnapshotManifest(manifest, manifestFile);
snapshotDirs.add(manifestFile.parent().toAbsolute()); // manifest may create empty snapshot dir
@ -2052,16 +2053,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
snapshotDirs.add(schemaFile.parent().toAbsolute()); // schema may create empty snapshot dir
}
// Maybe create ephemeral marker
if (ephemeral)
{
File ephemeralSnapshotMarker = getDirectories().getNewEphemeralSnapshotMarkerFile(tag);
createEphemeralSnapshotMarkerFile(tag, ephemeralSnapshotMarker);
snapshotDirs.add(ephemeralSnapshotMarker.parent().toAbsolute()); // marker may create empty snapshot dir
}
TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag,
manifest.createdAt, manifest.expiresAt, snapshotDirs);
TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(),
tag, manifest.createdAt, manifest.expiresAt, snapshotDirs,
manifest.ephemeral);
StorageService.instance.addSnapshot(snapshot);
return snapshot;
@ -2106,34 +2100,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
}
}
private void createEphemeralSnapshotMarkerFile(final String snapshot, File ephemeralSnapshotMarker)
{
try
{
if (!ephemeralSnapshotMarker.parent().exists())
ephemeralSnapshotMarker.parent().tryCreateDirectories();
Files.createFile(ephemeralSnapshotMarker.toPath());
if (logger.isTraceEnabled())
logger.trace("Created ephemeral snapshot marker file on {}.", ephemeralSnapshotMarker.absolutePath());
}
catch (IOException e)
{
logger.warn(String.format("Could not create marker file %s for ephemeral snapshot %s. " +
"In case there is a failure in the operation that created " +
"this snapshot, you may need to clean it manually afterwards.",
ephemeralSnapshotMarker.absolutePath(), snapshot), e);
}
}
protected static void clearEphemeralSnapshots(Directories directories)
{
RateLimiter clearSnapshotRateLimiter = DatabaseDescriptor.getSnapshotRateLimiter();
for (String ephemeralSnapshot : directories.listEphemeralSnapshots())
List<TableSnapshot> ephemeralSnapshots = new SnapshotLoader(directories).loadSnapshots()
.stream()
.filter(TableSnapshot::isEphemeral)
.collect(Collectors.toList());
for (TableSnapshot ephemeralSnapshot : ephemeralSnapshots)
{
logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot);
Directories.clearSnapshot(ephemeralSnapshot, directories.getCFDirectories(), clearSnapshotRateLimiter);
logger.trace("Clearing ephemeral snapshot {} leftover from previous session.", ephemeralSnapshot.getId());
Directories.clearSnapshot(ephemeralSnapshot.getTag(), directories.getCFDirectories(), clearSnapshotRateLimiter);
}
}

60
src/java/org/apache/cassandra/db/Directories.java
View File

@ -572,17 +572,6 @@ public class Directories
return new File(snapshotDir, "schema.cql");
}
public File getNewEphemeralSnapshotMarkerFile(String snapshotName)
{
File snapshotDir = new File(getWriteableLocationAsFile(1L), join(SNAPSHOT_SUBDIR, snapshotName));
return getEphemeralSnapshotMarkerFile(snapshotDir);
}
private static File getEphemeralSnapshotMarkerFile(File snapshotDirectory)
{
return new File(snapshotDirectory, "ephemeral.snapshot");
}
public static File getBackupsDirectory(Descriptor desc)
{
return getBackupsDirectory(desc.directory);
@ -983,18 +972,25 @@ public class Directories
return snapshots;
}
protected TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs) {
private TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs)
{
boolean ephemeral = manifest != null ? manifest.isEphemeral() : isLegacyEphemeralSnapshot(snapshotDirs);
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
return new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag, createdAt, expiresAt,
snapshotDirs);
snapshotDirs, ephemeral);
}
private static boolean isLegacyEphemeralSnapshot(Set<File> snapshotDirs)
{
return snapshotDirs.stream().map(d -> new File(d, "ephemeral.snapshot")).anyMatch(File::exists);
}
@VisibleForTesting
protected static SnapshotManifest maybeLoadManifest(String keyspace, String table, String tag, Set<File> snapshotDirs)
{
List<File> manifests = snapshotDirs.stream().map(d -> new File(d, "manifest.json"))
.filter(d -> d.exists()).collect(Collectors.toList());
.filter(File::exists).collect(Collectors.toList());
if (manifests.isEmpty())
{
@ -1018,42 +1014,6 @@ public class Directories
return null;
}
public List<String> listEphemeralSnapshots()
{
final List<String> ephemeralSnapshots = new LinkedList<>();
for (File snapshot : listAllSnapshots())
{
if (getEphemeralSnapshotMarkerFile(snapshot).exists())
ephemeralSnapshots.add(snapshot.name());
}
return ephemeralSnapshots;
}
private List<File> listAllSnapshots()
{
final List<File> snapshots = new LinkedList<>();
for (final File dir : dataPaths)
{
File snapshotDir = isSecondaryIndexFolder(dir)
? new File(dir.parentPath(), SNAPSHOT_SUBDIR)
: new File(dir, SNAPSHOT_SUBDIR);
if (snapshotDir.exists() && snapshotDir.isDirectory())
{
final File[] snapshotDirs = snapshotDir.tryList();
if (snapshotDirs != null)
{
for (final File snapshot : snapshotDirs)
{
if (snapshot.isDirectory())
snapshots.add(snapshot);
}
}
}
}
return snapshots;
}
@VisibleForTesting
protected Map<String, Set<File>> listSnapshotDirsByTag()
{

11
src/java/org/apache/cassandra/db/SnapshotDetailsTabularData.java
View File

@ -32,7 +32,8 @@ public class SnapshotDetailsTabularData
"True size",
"Size on disk",
"Creation time",
"Expiration time",};
"Expiration time",
"Ephemeral"};
private static final String[] ITEM_DESCS = new String[]{"snapshot_name",
"keyspace_name",
@ -40,7 +41,8 @@ public class SnapshotDetailsTabularData
"TrueDiskSpaceUsed",
"TotalDiskSpaceUsed",
"created_at",
"expires_at",};
"expires_at",
"ephemeral"};
private static final String TYPE_NAME = "SnapshotDetails";
@ -56,7 +58,7 @@ public class SnapshotDetailsTabularData
{
try
{
ITEM_TYPES = new OpenType[]{ SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING };
ITEM_TYPES = new OpenType[]{ SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, SimpleType.STRING };
COMPOSITE_TYPE = new CompositeType(TYPE_NAME, ROW_DESC, ITEM_NAMES, ITEM_DESCS, ITEM_TYPES);
@ -77,8 +79,9 @@ public class SnapshotDetailsTabularData
final String liveSize = FileUtils.stringifyFileSize(details.computeTrueSizeBytes());
String createdAt = safeToString(details.getCreatedAt());
String expiresAt = safeToString(details.getExpiresAt());
String ephemeral = Boolean.toString(details.isEphemeral());
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt }));
new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt, ephemeral }));
}
catch (OpenDataException e)
{

39
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -291,24 +291,13 @@ public class CassandraDaemon
SSTableHeaderFix.fixNonFrozenUDTIfUpgradeFrom30();
// clean up debris in the rest of the keyspaces
for (String keyspaceName : Schema.instance.getKeyspaces())
try
{
// Skip system as we've already cleaned it
if (keyspaceName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
continue;
for (TableMetadata cfm : Schema.instance.getTablesAndViews(keyspaceName))
{
try
{
ColumnFamilyStore.scrubDataDirectories(cfm);
}
catch (StartupException e)
{
exitOrFail(e.returnCode, e.getMessage(), e.getCause());
}
}
scrubDataDirectories();
}
catch (StartupException e)
{
exitOrFail(e.returnCode, e.getMessage(), e.getCause());
}
Keyspace.setInitialized();
@ -579,6 +568,22 @@ public class CassandraDaemon
VirtualKeyspaceRegistry.instance.register(SystemViewsKeyspace.instance);
}
public void scrubDataDirectories() throws StartupException
{
// clean up debris in the rest of the keyspaces
for (String keyspaceName : Schema.instance.getKeyspaces())
{
// Skip system as we've already cleaned it
if (keyspaceName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME))
continue;
for (TableMetadata cfm : Schema.instance.getTablesAndViews(keyspaceName))
{
ColumnFamilyStore.scrubDataDirectories(cfm);
}
}
}
public synchronized void initializeClientTransports()
{
// Native transport

3
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -4136,6 +4136,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
{
boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));
boolean includeEphemeral = options != null && Boolean.parseBoolean(options.getOrDefault("include_ephemeral", "false"));
SnapshotLoader loader = new SnapshotLoader();
Map<String, TabularData> snapshotMap = new HashMap<>();
@ -4144,6 +4145,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
if (skipExpiring && snapshot.isExpiring())
continue;
if (!includeEphemeral && snapshot.isEphemeral())
continue;
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag());
if (data == null)

5
src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
View File

@ -74,6 +74,11 @@ public class SnapshotLoader extends SimpleFileVisitor<Path>
this.dataDirectories = dataDirs;
}
public SnapshotLoader(Directories directories)
{
this(directories.getCFDirectories().stream().map(File::toPath).collect(Collectors.toList()));
}
public Set<TableSnapshot> loadSnapshots()
{
for (Path dataDir : dataDirectories)

22
src/java/org/apache/cassandra/service/snapshot/SnapshotManifest.java
View File

@ -46,19 +46,25 @@ public class SnapshotManifest
@JsonProperty("expires_at")
public final Instant expiresAt;
@JsonProperty("ephemeral")
public final boolean ephemeral;
/** needed for jackson serialization */
@SuppressWarnings("unused")
private SnapshotManifest() {
private SnapshotManifest()
{
this.files = null;
this.createdAt = null;
this.expiresAt = null;
this.ephemeral = false;
}
public SnapshotManifest(List<String> files, DurationSpec.IntSecondsBound ttl, Instant creationTime)
public SnapshotManifest(List<String> files, DurationSpec.IntSecondsBound ttl, Instant creationTime, boolean ephemeral)
{
this.files = files;
this.createdAt = creationTime;
this.expiresAt = ttl == null ? null : createdAt.plusSeconds(ttl.toSeconds());
this.ephemeral = ephemeral;
}
public List<String> getFiles()
@ -76,6 +82,11 @@ public class SnapshotManifest
return expiresAt;
}
public boolean isEphemeral()
{
return ephemeral;
}
public void serializeToJsonFile(File outputFile) throws IOException
{
FBUtilities.serializeToJsonFile(this, outputFile);
@ -92,12 +103,15 @@ public class SnapshotManifest
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SnapshotManifest manifest = (SnapshotManifest) o;
return Objects.equals(files, manifest.files) && Objects.equals(createdAt, manifest.createdAt) && Objects.equals(expiresAt, manifest.expiresAt);
return Objects.equals(files, manifest.files)
&& Objects.equals(createdAt, manifest.createdAt)
&& Objects.equals(expiresAt, manifest.expiresAt)
&& Objects.equals(ephemeral, manifest.ephemeral);
}
@Override
public int hashCode()
{
return Objects.hash(files, createdAt, expiresAt);
return Objects.hash(files, createdAt, expiresAt, ephemeral);
}
}

33
src/java/org/apache/cassandra/service/snapshot/TableSnapshot.java
View File

@ -45,6 +45,7 @@ public class TableSnapshot
private final String tableName;
private final UUID tableId;
private final String tag;
private final boolean ephemeral;
private final Instant createdAt;
private final Instant expiresAt;
@ -53,7 +54,7 @@ public class TableSnapshot
public TableSnapshot(String keyspaceName, String tableName, UUID tableId,
String tag, Instant createdAt, Instant expiresAt,
Set<File> snapshotDirs)
Set<File> snapshotDirs, boolean ephemeral)
{
this.keyspaceName = keyspaceName;
this.tableName = tableName;
@ -62,6 +63,7 @@ public class TableSnapshot
this.createdAt = createdAt;
this.expiresAt = expiresAt;
this.snapshotDirs = snapshotDirs;
this.ephemeral = ephemeral;
}
/**
@ -124,6 +126,11 @@ public class TableSnapshot
return snapshotDirs.stream().anyMatch(File::exists);
}
public boolean isEphemeral()
{
return ephemeral;
}
public boolean isExpiring()
{
return expiresAt != null;
@ -193,13 +200,13 @@ public class TableSnapshot
return Objects.equals(keyspaceName, snapshot.keyspaceName) && Objects.equals(tableName, snapshot.tableName) &&
Objects.equals(tableId, snapshot.tableId) && Objects.equals(tag, snapshot.tag) &&
Objects.equals(createdAt, snapshot.createdAt) && Objects.equals(expiresAt, snapshot.expiresAt) &&
Objects.equals(snapshotDirs, snapshot.snapshotDirs);
Objects.equals(snapshotDirs, snapshot.snapshotDirs) && Objects.equals(ephemeral, snapshot.ephemeral);
}
@Override
public int hashCode()
{
return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs);
return Objects.hash(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral);
}
@Override
@ -213,6 +220,7 @@ public class TableSnapshot
", createdAt=" + createdAt +
", expiresAt=" + expiresAt +
", snapshotDirs=" + snapshotDirs +
", ephemeral=" + ephemeral +
'}';
}
@ -224,6 +232,7 @@ public class TableSnapshot
private Instant createdAt = null;
private Instant expiresAt = null;
private boolean ephemeral;
private final Set<File> snapshotDirs = new HashSet<>();
@ -239,12 +248,17 @@ public class TableSnapshot
{
snapshotDirs.add(snapshotDir);
File manifestFile = new File(snapshotDir, "manifest.json");
if (manifestFile.exists() && createdAt == null && expiresAt == null) {
loadTimestampsFromManifest(manifestFile);
}
if (manifestFile.exists() && createdAt == null && expiresAt == null)
loadMetadataFromManifest(manifestFile);
// check if an ephemeral marker file exists only in case it is not already ephemeral
// by reading it from manifest
// TODO remove this on Cassandra 4.3 release, see CASSANDRA-16911
if (!ephemeral && new File(snapshotDir, "ephemeral.snapshot").exists())
ephemeral = true;
}
private void loadTimestampsFromManifest(File manifestFile)
private void loadMetadataFromManifest(File manifestFile)
{
try
{
@ -252,6 +266,9 @@ public class TableSnapshot
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(manifestFile);
createdAt = manifest.createdAt;
expiresAt = manifest.expiresAt;
// a snapshot may be ephemeral when it has a marker file (old way) or flag in manifest (new way)
if (!ephemeral)
ephemeral = manifest.ephemeral;
}
catch (IOException e)
{
@ -261,7 +278,7 @@ public class TableSnapshot
TableSnapshot build()
{
return new TableSnapshot(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs);
return new TableSnapshot(keyspaceName, tableName, tableId, tag, createdAt, expiresAt, snapshotDirs, ephemeral);
}
}

19
src/java/org/apache/cassandra/tools/nodetool/ListSnapshots.java
View File

@ -40,6 +40,11 @@ public class ListSnapshots extends NodeToolCmd
description = "Skip snapshots with TTL")
private boolean noTTL = false;
@Option(title = "ephemeral",
name = { "-e", "--ephemeral" },
description = "Include ephememeral snapshots")
private boolean includeEphemeral = false;
@Override
public void execute(NodeProbe probe)
{
@ -50,6 +55,7 @@ public class ListSnapshots extends NodeToolCmd
Map<String, String> options = new HashMap<>();
options.put("no_ttl", Boolean.toString(noTTL));
options.put("include_ephemeral", Boolean.toString(includeEphemeral));
final Map<String, TabularData> snapshotDetails = probe.getSnapshotDetails(options);
if (snapshotDetails.isEmpty())
@ -62,7 +68,11 @@ public class ListSnapshots extends NodeToolCmd
TableBuilder table = new TableBuilder();
// display column names only once
final List<String> indexNames = snapshotDetails.entrySet().iterator().next().getValue().getTabularType().getIndexNames();
table.add(indexNames.toArray(new String[indexNames.size()]));
if (includeEphemeral)
table.add(indexNames.toArray(new String[indexNames.size()]));
else
table.add(indexNames.subList(0, indexNames.size() - 1).toArray(new String[indexNames.size() - 1]));
for (final Map.Entry<String, TabularData> snapshotDetail : snapshotDetails.entrySet())
{
@ -70,12 +80,15 @@ public class ListSnapshots extends NodeToolCmd
for (Object eachValue : values)
{
final List<?> value = (List<?>) eachValue;
table.add(value.toArray(new String[value.size()]));
if (includeEphemeral)
table.add(value.toArray(new String[value.size()]));
else
table.add(value.subList(0, value.size() - 1).toArray(new String[value.size() - 1]));
}
}
table.printTo(out);
out.println("\nTotal TrueDiskSpaceUsed: " + FileUtils.stringifyFileSize(trueSnapshotsSize) + "\n");
out.println("\nTotal TrueDiskSpaceUsed: " + FileUtils.stringifyFileSize(trueSnapshotsSize) + '\n');
}
catch (Exception e)
{

4
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -114,6 +114,7 @@ import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.MigrationCoordinator;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.service.ClientState;
@ -619,6 +620,9 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
// Start up virtual table support
CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces();
// clean up debris in data directories
CassandraDaemon.getInstanceForTesting().scrubDataDirectories();
Keyspace.setInitialized();
// Replay any CommitLogSegments found on disk

164
test/distributed/org/apache/cassandra/distributed/test/EphemeralSnapshotTest.java Normal file
View File

@ -0,0 +1,164 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import com.google.common.util.concurrent.Futures;
import org.junit.Test;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.service.snapshot.SnapshotManifest;
import org.apache.cassandra.utils.Pair;
import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class EphemeralSnapshotTest extends TestBaseImpl
{
private static final String snapshotName = "snapshotname";
private static final String tableName = "city";
@Test
public void testStartupRemovesEphemeralSnapshotOnEphemeralFlagInManifest() throws Exception
{
try (Cluster c = init(builder().withNodes(1)
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
.start()))
{
Pair<String, String[]> initialisationData = initialise(c);
String tableId = initialisationData.left;
String[] dataDirs = initialisationData.right;
// rewrite manifest, pretend that it is ephemeral
Path manifestPath = findManifest(dataDirs, tableId);
SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(new File(manifestPath));
SnapshotManifest manifestWithEphemeralFlag = new SnapshotManifest(manifest.files, null, manifest.createdAt, true);
manifestWithEphemeralFlag.serializeToJsonFile(new File(manifestPath));
verify(c.get(1));
}
}
// TODO this test might be deleted once we get rid of ephemeral marker file for good in 4.3
@Test
public void testStartupRemovesEphemeralSnapshotOnMarkerFile() throws Exception
{
try (Cluster c = init(builder().withNodes(1)
.withConfig(config -> config.with(GOSSIP, NETWORK, NATIVE_PROTOCOL))
.start()))
{
Pair<String, String[]> initialisationData = initialise(c);
String tableId = initialisationData.left;
String[] dataDirs = initialisationData.right;
// place ephemeral marker file into snapshot directory pretending it was created as ephemeral
Path ephemeralMarkerFile = Paths.get(dataDirs[0])
.resolve(KEYSPACE)
.resolve(format("%s-%s", tableName, tableId))
.resolve("snapshots")
.resolve(snapshotName)
.resolve("ephemeral.snapshot");
Files.createFile(ephemeralMarkerFile);
verify(c.get(1));
}
}
private Pair<String, String[]> initialise(Cluster c)
{
c.schemaChange(withKeyspace("CREATE TABLE IF NOT EXISTS %s." + tableName + " (cityid int PRIMARY KEY, name text)"));
c.coordinator(1).execute(withKeyspace("INSERT INTO %s." + tableName + "(cityid, name) VALUES (1, 'Canberra');"), ONE);
IInvokableInstance instance = c.get(1);
instance.flush(KEYSPACE);
assertEquals(0, instance.nodetool("snapshot", "-kt", withKeyspace("%s." + tableName), "-t", snapshotName));
waitForSnapshot(instance, snapshotName);
String tableId = instance.callOnInstance((IIsolatedExecutor.SerializableCallable<String>) () -> {
return Keyspace.open(KEYSPACE).getMetadata().tables.get(tableName).get().id.asUUID().toString().replaceAll("-", "");
});
String[] dataDirs = (String[]) instance.config().get("data_file_directories");
return Pair.create(tableId, dataDirs);
}
private void verify(IInvokableInstance instance)
{
// by default, we do not see ephemerals
assertFalse(instance.nodetoolResult("listsnapshots").getStdout().contains("snapshotname"));
// we see them via -e flag
assertTrue(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains("snapshotname"));
Futures.getUnchecked(instance.shutdown());
// startup should remove ephemeral marker file
instance.startup();
assertFalse(instance.nodetoolResult("listsnapshots", "-e").getStdout().contains("snapshotname"));
}
private void waitForSnapshot(IInvokableInstance instance, String snapshotName)
{
await().timeout(20, SECONDS)
.pollInterval(1, SECONDS)
.until(() -> instance.nodetoolResult("listsnapshots", "-e").getStdout().contains(snapshotName));
}
private Path findManifest(String[] dataDirs, String tableId)
{
for (String dataDir : dataDirs)
{
Path manifest = Paths.get(dataDir)
.resolve(KEYSPACE)
.resolve(format("%s-%s", tableName, tableId))
.resolve("snapshots")
.resolve(snapshotName)
.resolve("manifest.json");
if (Files.exists(manifest))
{
return manifest;
}
}
throw new IllegalStateException("Unable to find manifest!");
}
}

4
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
View File

@ -170,7 +170,7 @@ public class ColumnFamilyStoreTest
}
@Test
public void testDeleteStandardRowSticksAfterFlush() throws Throwable
public void testDeleteStandardRowSticksAfterFlush()
{
// test to make sure flushing after a delete doesn't resurrect delted cols.
String keyspaceName = KEYSPACE1;
@ -228,7 +228,7 @@ public class ColumnFamilyStoreTest
}
@Test
public void testClearEphemeralSnapshots() throws Throwable
public void testClearEphemeralSnapshots()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_INDEX1);

72
test/unit/org/apache/cassandra/db/DirectoriesTest.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@ -97,6 +98,7 @@ public class DirectoriesTest
public static final String TABLE_NAME = "FakeTable";
public static final String SNAPSHOT1 = "snapshot1";
public static final String SNAPSHOT2 = "snapshot2";
public static final String SNAPSHOT3 = "snapshot3";
public static final String LEGACY_SNAPSHOT_NAME = "42";
private static File tempDataDir;
@ -105,7 +107,7 @@ public class DirectoriesTest
private static Set<TableMetadata> CFM;
private static Map<String, List<File>> sstablesByTableName;
@Parameterized.Parameter(0)
@Parameterized.Parameter
public SSTableId.Builder<? extends SSTableId> idBuilder;
@Parameterized.Parameter(1)
@ -151,7 +153,7 @@ public class DirectoriesTest
@AfterClass
public static void afterClass()
{
FileUtils.deleteRecursive(tempDataDir);
tempDataDir.deleteRecursive();
}
private static DataDirectory[] toDataDirectories(File location)
@ -159,7 +161,7 @@ public class DirectoriesTest
return new DataDirectory[] { new DataDirectory(location) };
}
private void createTestFiles() throws IOException
private void createTestFiles()
{
for (TableMetadata cfm : CFM)
{
@ -181,25 +183,27 @@ public class DirectoriesTest
}
}
class FakeSnapshot {
static class FakeSnapshot {
final TableMetadata table;
final String tag;
final File snapshotDir;
final SnapshotManifest manifest;
final boolean ephemeral;
FakeSnapshot(TableMetadata table, String tag, File snapshotDir, SnapshotManifest manifest)
FakeSnapshot(TableMetadata table, String tag, File snapshotDir, SnapshotManifest manifest, boolean ephemeral)
{
this.table = table;
this.tag = tag;
this.snapshotDir = snapshotDir;
this.manifest = manifest;
this.ephemeral = ephemeral;
}
public TableSnapshot asTableSnapshot()
{
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
return new TableSnapshot(table.keyspace, table.name, table.id.asUUID(), tag, createdAt, expiresAt, Collections.singleton(snapshotDir));
return new TableSnapshot(table.keyspace, table.name, table.id.asUUID(), tag, createdAt, expiresAt, Collections.singleton(snapshotDir), ephemeral);
}
}
@ -211,7 +215,7 @@ public class DirectoriesTest
.build();
}
public FakeSnapshot createFakeSnapshot(TableMetadata table, String tag, boolean createManifest) throws IOException
public FakeSnapshot createFakeSnapshot(TableMetadata table, String tag, boolean createManifest, boolean ephemeral) throws IOException
{
File tableDir = cfDir(table);
tableDir.tryCreateDirectories();
@ -225,11 +229,15 @@ public class DirectoriesTest
if (createManifest)
{
File manifestFile = Directories.getSnapshotManifestFile(snapshotDir);
manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.filenameFor(Component.DATA)), new DurationSpec.IntSecondsBound("1m"), now());
manifest = new SnapshotManifest(Collections.singletonList(sstableDesc.filenameFor(Component.DATA)), new DurationSpec.IntSecondsBound("1m"), now(), ephemeral);
manifest.serializeToJsonFile(manifestFile);
}
else if (ephemeral)
{
Files.createFile(snapshotDir.toPath().resolve("ephemeral.snapshot"));
}
return new FakeSnapshot(table, tag, snapshotDir, manifest);
return new FakeSnapshot(table, tag, snapshotDir, manifest, ephemeral);
}
private List<File> createFakeSSTable(File dir, String cf, int gen)
@ -269,7 +277,7 @@ public class DirectoriesTest
}
@Test
public void testStandardDirs() throws IOException
public void testStandardDirs()
{
for (TableMetadata cfm : CFM)
{
@ -296,22 +304,27 @@ public class DirectoriesTest
assertThat(directories.listSnapshots()).isEmpty();
// Create snapshot with and without manifest
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true);
FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false);
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false);
// ephemeral without manifst
FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, TableSnapshot> snapshots = directories.listSnapshots();
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2));
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT1)).isEqualTo(snapshot1.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot());
// Now remove snapshot1
FileUtils.deleteRecursive(snapshot1.snapshotDir);
snapshot1.snapshotDir.deleteRecursive();
// Only snapshot 2 should be present
// Only snapshot 2 and 3 should be present
snapshots = directories.listSnapshots();
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2));
assertThat(snapshots.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
assertThat(snapshots.get(SNAPSHOT2)).isEqualTo(snapshot2.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT3)).isEqualTo(snapshot3.asTableSnapshot());
assertThat(snapshots.get(SNAPSHOT3).isEphemeral()).isTrue();
}
@Test
@ -322,21 +335,23 @@ public class DirectoriesTest
assertThat(directories.listSnapshotDirsByTag()).isEmpty();
// Create snapshot with and without manifest
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true);
FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false);
FakeSnapshot snapshot1 = createFakeSnapshot(fakeTable, SNAPSHOT1, true, false);
FakeSnapshot snapshot2 = createFakeSnapshot(fakeTable, SNAPSHOT2, false, false);
FakeSnapshot snapshot3 = createFakeSnapshot(fakeTable, SNAPSHOT3, false, true);
// Both snapshots should be present
Map<String, Set<File>> snapshotDirs = directories.listSnapshotDirsByTag();
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2));
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT1, SNAPSHOT2, SNAPSHOT3));
assertThat(snapshotDirs.get(SNAPSHOT1)).allMatch(snapshotDir -> snapshotDir.equals(snapshot1.snapshotDir));
assertThat(snapshotDirs.get(SNAPSHOT2)).allMatch(snapshotDir -> snapshotDir.equals(snapshot2.snapshotDir));
assertThat(snapshotDirs.get(SNAPSHOT3)).allMatch(snapshotDir -> snapshotDir.equals(snapshot3.snapshotDir));
// Now remove snapshot1
FileUtils.deleteRecursive(snapshot1.snapshotDir);
snapshot1.snapshotDir.deleteRecursive();
// Only snapshot 2 should be present
// Only snapshot 2 and 3 should be present
snapshotDirs = directories.listSnapshotDirsByTag();
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2));
assertThat(snapshotDirs.keySet()).isEqualTo(Sets.newHashSet(SNAPSHOT2, SNAPSHOT3));
}
@Test
@ -353,7 +368,7 @@ public class DirectoriesTest
File manifestFile = directories.getSnapshotManifestFile(tag);
SnapshotManifest manifest = new SnapshotManifest(files, new DurationSpec.IntSecondsBound("1m"), now());
SnapshotManifest manifest = new SnapshotManifest(files, new DurationSpec.IntSecondsBound("1m"), now(), false);
manifest.serializeToJsonFile(manifestFile);
Set<File> dirs = new HashSet<>();
@ -488,7 +503,7 @@ public class DirectoriesTest
}
@Test
public void testTemporaryFile() throws IOException
public void testTemporaryFile()
{
for (TableMetadata cfm : CFM)
{
@ -552,11 +567,10 @@ public class DirectoriesTest
final Directories directories = new Directories(cfm, toDataDirectories(tempDataDir));
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
final String n = Long.toString(nanoTime());
Callable<File> directoryGetter = new Callable<File>() {
public File call() throws Exception {
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
return Directories.getSnapshotDirectory(desc, n);
}
Callable<File> directoryGetter = () ->
{
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.name, sstableId(1), SSTableFormat.Type.BIG);
return Directories.getSnapshotDirectory(desc, n);
};
List<Future<File>> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter));
for(Future<File> fut:invoked) {

51
test/unit/org/apache/cassandra/service/snapshot/SnapshotLoaderTest.java
View File

@ -27,6 +27,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@ -123,9 +124,40 @@ public class SnapshotLoaderTest
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, null, null, tag2Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, null, null, tag3Files, false));
}
@Test
public void testEphemeralSnapshotWithoutManifest() throws IOException
{
Set<File> tag1Files = new HashSet<>();
// Create one snapshot per table - without manifests:
// - ks1.t1 : tag1
File baseDir = new File(tmpDir.newFolder());
boolean ephemeralFileCreated = false;
for (String dataDir : DATA_DIRS)
{
File dir = createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1);
tag1Files.add(dir);
if (!ephemeralFileCreated)
{
createEphemeralMarkerFile(dir);
ephemeralFileCreated = true;
}
}
// Verify snapshot is found correctly from data directories
SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
Paths.get(baseDir.toString(), DATA_DIR_2),
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(1);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, null, null, tag1Files, true));
Assert.assertTrue(snapshots.stream().findFirst().get().isEphemeral());
}
@Test
@ -169,9 +201,9 @@ public class SnapshotLoaderTest
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files, false));
}
@Test
@ -208,7 +240,7 @@ public class SnapshotLoaderTest
private void writeManifest(File snapshotDir, Instant creationTime, DurationSpec.IntSecondsBound ttl) throws IOException
{
SnapshotManifest manifest = new SnapshotManifest(Lists.newArrayList("f1", "f2", "f3"), ttl, creationTime);
SnapshotManifest manifest = new SnapshotManifest(Lists.newArrayList("f1", "f2", "f3"), ttl, creationTime, false);
manifest.serializeToJsonFile(getManifestFile(snapshotDir));
}
@ -219,6 +251,11 @@ public class SnapshotLoaderTest
return file;
}
private static void createEphemeralMarkerFile(File dir)
{
Assert.assertTrue(new File(dir, "ephemeral.snapshot").createFileIfNotExists());
}
static String tableDirName(String tableName, UUID tableId)
{
return String.format("%s-%s", tableName, removeDashes(tableId));

23
test/unit/org/apache/cassandra/service/snapshot/SnapshotManagerTest.java
View File

@ -50,7 +50,7 @@ public class SnapshotManagerTest
@ClassRule
public static TemporaryFolder temporaryFolder = new TemporaryFolder();
private TableSnapshot generateSnapshotDetails(String tag, Instant expiration) throws Exception {
private TableSnapshot generateSnapshotDetails(String tag, Instant expiration, boolean ephemeral) throws Exception {
return new TableSnapshot(
"ks",
"tbl",
@ -58,15 +58,16 @@ public class SnapshotManagerTest
tag,
Instant.EPOCH,
expiration,
createFolders(temporaryFolder)
createFolders(temporaryFolder),
ephemeral
);
}
@Test
public void testLoadSnapshots() throws Exception {
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS));
TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null);
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusSeconds(ONE_DAY_SECS), false);
TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false);
List<TableSnapshot> snapshots = Arrays.asList(expired, nonExpired, nonExpiring);
// Create SnapshotManager with 3 snapshots: expired, non-expired and non-expiring
@ -84,9 +85,9 @@ public class SnapshotManagerTest
SnapshotManager manager = new SnapshotManager(3, 3);
// Add 3 snapshots: expired, non-expired and non-expiring
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS));
TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null);
TableSnapshot expired = generateSnapshotDetails("expired", Instant.EPOCH, false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
TableSnapshot nonExpiring = generateSnapshotDetails("non-expiring", null, false);
manager.addSnapshot(expired);
manager.addSnapshot(nonExpired);
manager.addSnapshot(nonExpiring);
@ -118,8 +119,8 @@ public class SnapshotManagerTest
// Add 2 expiring snapshots: one to expire in 2 seconds, another in 1 day
int TTL_SECS = 2;
TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS));
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS));
TableSnapshot toExpire = generateSnapshotDetails("to-expire", now().plusSeconds(TTL_SECS), false);
TableSnapshot nonExpired = generateSnapshotDetails("non-expired", now().plusMillis(ONE_DAY_SECS), false);
manager.addSnapshot(toExpire);
manager.addSnapshot(nonExpired);
@ -150,7 +151,7 @@ public class SnapshotManagerTest
{
// Given
SnapshotManager manager = new SnapshotManager(1, 3);
TableSnapshot expiringSnapshot = generateSnapshotDetails("snapshot", now().plusMillis(50000));
TableSnapshot expiringSnapshot = generateSnapshotDetails("snapshot", now().plusMillis(50000), false);
manager.addSnapshot(expiringSnapshot);
assertThat(manager.getExpiringSnapshots()).contains(expiringSnapshot);
assertThat(expiringSnapshot.exists()).isTrue();

2
test/unit/org/apache/cassandra/service/snapshot/SnapshotManifestTest.java
View File

@ -108,7 +108,7 @@ public class SnapshotManifestTest
@Test
public void testSerializeAndDeserialize() throws Exception {
SnapshotManifest manifest = new SnapshotManifest(Arrays.asList("db1", "db2", "db3"), new DurationSpec.IntSecondsBound("2m"), Instant.ofEpochMilli(currentTimeMillis()));
SnapshotManifest manifest = new SnapshotManifest(Arrays.asList("db1", "db2", "db3"), new DurationSpec.IntSecondsBound("2m"), Instant.ofEpochMilli(currentTimeMillis()), false);
File manifestFile = new File(tempFolder.newFile("manifest.json"));
manifest.serializeToJsonFile(manifestFile);

36
test/unit/org/apache/cassandra/service/snapshot/TableSnapshotTest.java
View File

@ -74,7 +74,9 @@ public class TableSnapshotTest
"some",
null,
null,
folders);
folders,
false
);
assertThat(snapshot.exists()).isTrue();
@ -95,7 +97,9 @@ public class TableSnapshotTest
"some",
null,
null,
folders);
folders,
false
);
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@ -107,7 +111,9 @@ public class TableSnapshotTest
"some",
now(),
null,
folders);
folders,
false
);
assertThat(snapshot.isExpiring()).isFalse();
assertThat(snapshot.isExpired(now())).isFalse();
@ -119,7 +125,9 @@ public class TableSnapshotTest
"some",
now(),
now().plusSeconds(1000),
folders);
folders,
false
);
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isFalse();
@ -131,7 +139,8 @@ public class TableSnapshotTest
"some",
now(),
now().minusSeconds(1000),
folders);
folders,
false);
assertThat(snapshot.isExpiring()).isTrue();
assertThat(snapshot.isExpired(now())).isTrue();
@ -159,7 +168,8 @@ public class TableSnapshotTest
"some",
null,
null,
folders);
folders,
false);
Long res = 0L;
@ -185,7 +195,9 @@ public class TableSnapshotTest
"some",
null,
null,
folders);
folders,
false
);
Long res = 0L;
@ -214,7 +226,10 @@ public class TableSnapshotTest
"some1",
createdAt,
null,
folders);
folders,
false
);
assertThat(withCreatedAt.getCreatedAt()).isEqualTo(createdAt);
// When createdAt is null, it should return the snapshot folder minimum update time
@ -225,7 +240,10 @@ public class TableSnapshotTest
"some1",
null,
null,
folders);
folders,
false
);
assertThat(withoutCreatedAt.getCreatedAt()).isEqualTo(Instant.ofEpochMilli(folders.stream().mapToLong(f -> f.lastModified()).min().getAsLong()));
}