natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-2.1' into trunk 

Conflicts:
	test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
This commit is contained in:
Marcus Eriksson 2014-11-14 11:10:09 +01:00
parent 3b6edc6afd abbcfc5f72
commit 41a35ec74e
3 changed files with 60 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -33,6 +33,7 @@
* improve concurrency of repair (CASSANDRA-6455, 8208)
2.1.3
* Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
* Add more log info if readMeter is null (CASSANDRA-8238)
* add check of the system wall clock time at startup (CASSANDRA-8305)
* Support for frozen collections (CASSANDRA-7859)

20
src/java/org/apache/cassandra/db/compaction/CompactionController.java
View File

@ -92,12 +92,11 @@ public class CompactionController implements AutoCloseable
* Finds expired sstables
*
* works something like this;
* 1. find "global" minTimestamp of overlapping sstables (excluding the possibly droppable ones)
* 2. build a list of candidates to be dropped
* 3. sort the candidate list, biggest maxTimestamp first in list
* 4. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp) and it is included in the compaction
* - if not droppable, update global minTimestamp and remove from candidates
* 5. return candidates.
* 1. find "global" minTimestamp of overlapping sstables and compacting sstables containing any non-expired data
* 2. build a list of fully expired candidates
* 3. check if the candidates to be dropped actually can be dropped (maxTimestamp < global minTimestamp)
* - if not droppable, remove from candidates
* 4. return candidates.
*
* @param cfStore
* @param compacting we take the drop-candidates from this set, it is usually the sstables included in the compaction
@ -127,10 +126,10 @@ public class CompactionController implements AutoCloseable
minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp());
}
// we still need to keep candidates that might shadow something in a
// non-candidate sstable. And if we remove a sstable from the candidates, we
// must take it's timestamp into account (hence the sorting below).
Collections.sort(candidates, SSTableReader.maxTimestampComparator);
// At this point, minTimestamp denotes the lowest timestamp of any relevant
// SSTable that contains a constructive value. candidates contains all the
// candidates with no constructive values. The ones out of these that have
// (getMaxTimestamp() < minTimestamp) serve no purpose anymore.
Iterator<SSTableReader> iterator = candidates.iterator();
while (iterator.hasNext())
@ -138,7 +137,6 @@ public class CompactionController implements AutoCloseable
SSTableReader candidate = iterator.next();
if (candidate.getMaxTimestamp() >= minTimestamp)
{
minTimestamp = Math.min(candidate.getMinTimestamp(), minTimestamp);
iterator.remove();
}
else

50
test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
View File

@ -22,6 +22,7 @@ package org.apache.cassandra.db.compaction;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import com.google.common.collect.Sets;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -34,6 +35,10 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.utils.ByteBufferUtil;
import java.util.Collections;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -53,6 +58,51 @@ public class TTLExpiryTest
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1));
}
@Test
public void testAggressiveFullyExpired()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
DecoratedKey ttlKey = Util.dk("ttl");
Mutation rm = new Mutation(KEYSPACE1, ttlKey.getKey());
rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new Mutation(KEYSPACE1, ttlKey.getKey());
rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new Mutation(KEYSPACE1, ttlKey.getKey());
rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new Mutation(KEYSPACE1, ttlKey.getKey());
rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
int now = (int)(System.currentTimeMillis() / 1000);
int gcBefore = now + 2;
Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
cfs,
sstables,
Collections.EMPTY_SET,
gcBefore);
assertEquals(2, expired.size());
cfs.clearUnsafe();
}
@Test
public void testSimpleExpire() throws InterruptedException
{