natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.11' into cassandra-4.0

This commit is contained in:
Paulo Motta 2022-03-14 17:18:15 -03:00
parent d9bd03552d 95a6223057
commit 5f50c797bc
4 changed files with 77 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -5,6 +5,7 @@
* Streaming tasks handle empty SSTables correctly (CASSANDRA-16349)
* Prevent SSTableLoader from doing unnecessary work (CASSANDRA-16349)
Merged from 3.11:
* Fix snapshot true size calculation (CASSANDRA-17267)
* dropping of a materialized view creates a snapshot with dropped- prefix (CASSANDRA-17415)
* Validate existence of DCs when repairing (CASSANDRA-17407)
Merged from 3.0:

8
src/java/org/apache/cassandra/db/Directories.java
View File

@ -22,6 +22,8 @@ import java.nio.file.*;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
@ -1156,11 +1158,11 @@ public class Directories
private class SSTableSizeSummer extends DirectorySizeCalculator
{
private final HashSet<File> toSkip;
private final Set<String> toSkip;
SSTableSizeSummer(File path, List<File> files)
{
super(path);
toSkip = new HashSet<>(files);
toSkip = files.stream().map(f -> f.getName()).collect(Collectors.toSet());
}
@Override
@ -1171,7 +1173,7 @@ public class Directories
return desc != null
&& desc.ksname.equals(metadata.keyspace)
&& desc.cfname.equals(metadata.name)
&& !toSkip.contains(file);
&& !toSkip.contains(file.getName());
}
}

43
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
View File

@ -32,10 +32,13 @@ import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.utils.Pair;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -254,6 +257,46 @@ public class ColumnFamilyStoreTest
cfs.clearSnapshot("");
}
@Test
public void testSnapshotSize()
{
// cleanup any previous test gargbage
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
cfs.clearSnapshot("");
// Add row
new RowUpdateBuilder(cfs.metadata(), 0, "key1")
.clustering("Column1")
.add("val", "asdf")
.build()
.applyUnsafe();
cfs.forceBlockingFlush();
// snapshot
cfs.snapshot("basic", null, false, false);
// check snapshot was created
Map<String, Directories.SnapshotSizeDetails> snapshotDetails = cfs.getSnapshotDetails();
assertThat(snapshotDetails).hasSize(1);
assertThat(snapshotDetails).containsKey("basic");
// check that sizeOnDisk > trueSize = 0
Directories.SnapshotSizeDetails details = snapshotDetails.get("basic");
assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes);
assertThat(details.dataSizeBytes).isZero();
// compact base table to make trueSize > 0
cfs.forceMajorCompaction();
LifecycleTransaction.waitForDeletions();
// sizeOnDisk > trueSize because trueSize does not include manifest.json
// Check that truesize now is > 0
snapshotDetails = cfs.getSnapshotDetails();
details = snapshotDetails.get("basic");
assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes);
assertThat(details.dataSizeBytes).isPositive();
}
@Test
public void testBackupAfterFlush() throws Throwable
{

46
test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
View File

@ -42,6 +42,7 @@ import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
@ -151,7 +152,6 @@ public class SASIIndexTest
data.put(UUID.randomUUID().toString(), Pair.create(UUID.randomUUID().toString(), r.nextInt()));
ColumnFamilyStore store = loadData(data, true);
store.forceMajorCompaction();
Set<SSTableReader> ssTableReaders = store.getLiveSSTables();
Set<Component> sasiComponents = new HashSet<>();
@ -165,6 +165,10 @@ public class SASIIndexTest
try
{
store.snapshot(snapshotName);
// Compact to make true snapshot size != 0
store.forceMajorCompaction();
LifecycleTransaction.waitForDeletions();
FileReader reader = new FileReader(store.getDirectories().getSnapshotManifestFile(snapshotName));
JSONObject manifest = (JSONObject) new JSONParser().parse(reader);
JSONArray files = (JSONArray) manifest.get("files");
@ -197,8 +201,7 @@ public class SASIIndexTest
for (Component c : components)
{
Path componentPath = Paths.get(sstable.descriptor + "-" + c.name);
long componentSize = Files.size(componentPath);
long componentSize = Files.size(Paths.get(snapshotSSTable.filenameFor(c)));
if (Component.Type.fromRepresentation(c.name) == Component.Type.SECONDARY_INDEX)
indexSize += componentSize;
else
@ -209,7 +212,7 @@ public class SASIIndexTest
Map<String, Directories.SnapshotSizeDetails> details = store.getSnapshotDetails();
// check that SASI components are included in the computation of snapshot size
Assert.assertEquals((long) details.get(snapshotName).dataSizeBytes, tableSize + indexSize);
Assert.assertEquals(tableSize + indexSize, (long) details.get(snapshotName).dataSizeBytes);
}
finally
{
@ -2640,12 +2643,16 @@ public class SASIIndexTest
private static Set<String> getIndexed(ColumnFamilyStore store, ColumnFilter columnFilter, int maxResults, Expression... expressions)
{
return getKeys(getIndexed(store, columnFilter, null, maxResults, expressions));
ReadCommand command = getIndexReadCommand(store, columnFilter, null, maxResults, expressions);
try (ReadExecutionController controller = command.executionController();
UnfilteredPartitionIterator rows = command.executeLocally(controller))
{
return getKeys(rows);
}
}
private static Set<DecoratedKey> getPaged(ColumnFamilyStore store, int pageSize, Expression... expressions)
{
UnfilteredPartitionIterator currentPage;
Set<DecoratedKey> uniqueKeys = new TreeSet<>();
DecoratedKey lastKey = null;
@ -2654,28 +2661,32 @@ public class SASIIndexTest
do
{
count = 0;
currentPage = getIndexed(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions);
if (currentPage == null)
break;
ReadCommand command = getIndexReadCommand(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions);
while (currentPage.hasNext())
try (ReadExecutionController controller = command.executionController();
UnfilteredPartitionIterator currentPage = command.executeLocally(controller))
{
try (UnfilteredRowIterator row = currentPage.next())
if (currentPage == null)
break;
while (currentPage.hasNext())
{
uniqueKeys.add(row.partitionKey());
lastKey = row.partitionKey();
count++;
try (UnfilteredRowIterator row = currentPage.next())
{
uniqueKeys.add(row.partitionKey());
lastKey = row.partitionKey();
count++;
}
}
}
currentPage.close();
}
while (count == pageSize);
return uniqueKeys;
}
private static UnfilteredPartitionIterator getIndexed(ColumnFamilyStore store, ColumnFilter columnFilter, DecoratedKey startKey, int maxResults, Expression... expressions)
private static ReadCommand getIndexReadCommand(ColumnFamilyStore store, ColumnFilter columnFilter, DecoratedKey startKey, int maxResults, Expression[] expressions)
{
DataRange range = (startKey == null)
? DataRange.allData(PARTITIONER)
@ -2692,8 +2703,7 @@ public class SASIIndexTest
filter,
DataLimits.cqlLimits(maxResults),
range);
return command.executeLocally(command.executionController());
return command;
}
private static Mutation newMutation(String key, String firstName, String lastName, int age, long timestamp)