diff --git a/CHANGES.txt b/CHANGES.txt index e9b5b439b5..86eae2b536 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,7 @@ * Streaming tasks handle empty SSTables correctly (CASSANDRA-16349) * Prevent SSTableLoader from doing unnecessary work (CASSANDRA-16349) Merged from 3.11: + * Fix snapshot true size calculation (CASSANDRA-17267) * dropping of a materialized view creates a snapshot with dropped- prefix (CASSANDRA-17415) * Validate existence of DCs when repairing (CASSANDRA-17407) Merged from 3.0: diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java index aa2881f5e8..f09cdae3a6 100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@ -22,6 +22,8 @@ import java.nio.file.*; import java.util.*; import java.util.concurrent.ThreadLocalRandom; import java.util.function.BiPredicate; +import java.util.stream.Collectors; + import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; @@ -1156,11 +1158,11 @@ public class Directories private class SSTableSizeSummer extends DirectorySizeCalculator { - private final HashSet toSkip; + private final Set toSkip; SSTableSizeSummer(File path, List files) { super(path); - toSkip = new HashSet<>(files); + toSkip = files.stream().map(f -> f.getName()).collect(Collectors.toSet()); } @Override @@ -1171,7 +1173,7 @@ public class Directories return desc != null && desc.ksname.equals(metadata.keyspace) && desc.cfname.equals(metadata.name) - && !toSkip.contains(file); + && !toSkip.contains(file.getName()); } } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index df8837405e..266b37d86a 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -32,10 +32,13 @@ import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.utils.Pair; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -254,6 +257,46 @@ public class ColumnFamilyStoreTest cfs.clearSnapshot(""); } + @Test + public void testSnapshotSize() + { + // cleanup any previous test gargbage + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1); + cfs.clearSnapshot(""); + + // Add row + new RowUpdateBuilder(cfs.metadata(), 0, "key1") + .clustering("Column1") + .add("val", "asdf") + .build() + .applyUnsafe(); + cfs.forceBlockingFlush(); + + // snapshot + cfs.snapshot("basic", null, false, false); + + // check snapshot was created + Map snapshotDetails = cfs.getSnapshotDetails(); + assertThat(snapshotDetails).hasSize(1); + assertThat(snapshotDetails).containsKey("basic"); + + // check that sizeOnDisk > trueSize = 0 + Directories.SnapshotSizeDetails details = snapshotDetails.get("basic"); + assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes); + assertThat(details.dataSizeBytes).isZero(); + + // compact base table to make trueSize > 0 + cfs.forceMajorCompaction(); + LifecycleTransaction.waitForDeletions(); + + // sizeOnDisk > trueSize because trueSize does not include manifest.json + // Check that truesize now is > 0 + snapshotDetails = cfs.getSnapshotDetails(); + details = snapshotDetails.get("basic"); + assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes); + assertThat(details.dataSizeBytes).isPositive(); + } + @Test public void testBackupAfterFlush() throws Throwable { diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index 364acdb87e..945a5e9f1f 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -42,6 +42,7 @@ import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.index.Index; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.Schema; @@ -151,7 +152,6 @@ public class SASIIndexTest data.put(UUID.randomUUID().toString(), Pair.create(UUID.randomUUID().toString(), r.nextInt())); ColumnFamilyStore store = loadData(data, true); - store.forceMajorCompaction(); Set ssTableReaders = store.getLiveSSTables(); Set sasiComponents = new HashSet<>(); @@ -165,6 +165,10 @@ public class SASIIndexTest try { store.snapshot(snapshotName); + // Compact to make true snapshot size != 0 + store.forceMajorCompaction(); + LifecycleTransaction.waitForDeletions(); + FileReader reader = new FileReader(store.getDirectories().getSnapshotManifestFile(snapshotName)); JSONObject manifest = (JSONObject) new JSONParser().parse(reader); JSONArray files = (JSONArray) manifest.get("files"); @@ -197,8 +201,7 @@ public class SASIIndexTest for (Component c : components) { - Path componentPath = Paths.get(sstable.descriptor + "-" + c.name); - long componentSize = Files.size(componentPath); + long componentSize = Files.size(Paths.get(snapshotSSTable.filenameFor(c))); if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX) indexSize += componentSize; else @@ -209,7 +212,7 @@ public class SASIIndexTest Map details = store.getSnapshotDetails(); // check that SASI components are included in the computation of snapshot size - Assert.assertEquals((long) details.get(snapshotName).dataSizeBytes, tableSize + indexSize); + Assert.assertEquals(tableSize + indexSize, (long) details.get(snapshotName).dataSizeBytes); } finally { @@ -2640,12 +2643,16 @@ public class SASIIndexTest private static Set getIndexed(ColumnFamilyStore store, ColumnFilter columnFilter, int maxResults, Expression... expressions) { - return getKeys(getIndexed(store, columnFilter, null, maxResults, expressions)); + ReadCommand command = getIndexReadCommand(store, columnFilter, null, maxResults, expressions); + try (ReadExecutionController controller = command.executionController(); + UnfilteredPartitionIterator rows = command.executeLocally(controller)) + { + return getKeys(rows); + } } private static Set getPaged(ColumnFamilyStore store, int pageSize, Expression... expressions) { - UnfilteredPartitionIterator currentPage; Set uniqueKeys = new TreeSet<>(); DecoratedKey lastKey = null; @@ -2654,28 +2661,32 @@ public class SASIIndexTest do { count = 0; - currentPage = getIndexed(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions); - if (currentPage == null) - break; + ReadCommand command = getIndexReadCommand(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions); - while (currentPage.hasNext()) + try (ReadExecutionController controller = command.executionController(); + UnfilteredPartitionIterator currentPage = command.executeLocally(controller)) { - try (UnfilteredRowIterator row = currentPage.next()) + if (currentPage == null) + break; + + while (currentPage.hasNext()) { - uniqueKeys.add(row.partitionKey()); - lastKey = row.partitionKey(); - count++; + try (UnfilteredRowIterator row = currentPage.next()) + { + uniqueKeys.add(row.partitionKey()); + lastKey = row.partitionKey(); + count++; + } } } - currentPage.close(); } while (count == pageSize); return uniqueKeys; } - private static UnfilteredPartitionIterator getIndexed(ColumnFamilyStore store, ColumnFilter columnFilter, DecoratedKey startKey, int maxResults, Expression... expressions) + private static ReadCommand getIndexReadCommand(ColumnFamilyStore store, ColumnFilter columnFilter, DecoratedKey startKey, int maxResults, Expression[] expressions) { DataRange range = (startKey == null) ? DataRange.allData(PARTITIONER) @@ -2692,8 +2703,7 @@ public class SASIIndexTest filter, DataLimits.cqlLimits(maxResults), range); - - return command.executeLocally(command.executionController()); + return command; } private static Mutation newMutation(String key, String firstName, String lastName, int age, long timestamp)