natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add ability to disable purgable tombstone check via startup flag 

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11831
This commit is contained in:
Marcus Eriksson 2016-05-20 10:37:34 +02:00
parent 03180469d6
commit 5dc7414970
3 changed files with 149 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
View File

@ -359,7 +359,7 @@ public abstract class AbstractCompactionStrategy
*/
protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore)
{
if (disableTombstoneCompactions)
if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES)
return false;
// since we use estimations to calculate, there is a chance that compaction will not drop tombstones actually.
// if that happens we will end up in infinite compaction loop, so first we check enough if enough time has

26
src/java/org/apache/cassandra/db/compaction/CompactionController.java
View File

@ -41,8 +41,13 @@ import org.apache.cassandra.utils.concurrent.Refs;
public class CompactionController implements AutoCloseable
{
private static final Logger logger = LoggerFactory.getLogger(CompactionController.class);
static final boolean NEVER_PURGE_TOMBSTONES = Boolean.getBoolean("cassandra.never_purge_tombstones");
public final ColumnFamilyStore cfs;
// note that overlappingTree and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES is set - this is a
// good thing so that noone starts using them and thinks that if overlappingSSTables is empty, there
// is no overlap.
private DataTracker.SSTableIntervalTree overlappingTree;
private Refs<SSTableReader> overlappingSSTables;
private final Iterable<SSTableReader> compacting;
@ -54,17 +59,25 @@ public class CompactionController implements AutoCloseable
this(cfs, null, maxValue);
}
public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, int gcBefore)
public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting, int gcBefore)
{
assert cfs != null;
this.cfs = cfs;
this.gcBefore = gcBefore;
this.compacting = compacting;
refreshOverlaps();
if (NEVER_PURGE_TOMBSTONES)
logger.warn("You are running with -Dcassandra.never_purge_tombstones=true, this is dangerous!");
}
void maybeRefreshOverlaps()
{
if (NEVER_PURGE_TOMBSTONES)
{
logger.debug("not refreshing overlaps - running with -Dcassandra.never_purge_tombstones=true");
return;
}
for (SSTableReader reader : overlappingSSTables)
{
if (reader.isMarkedCompacted())
@ -77,6 +90,9 @@ public class CompactionController implements AutoCloseable
private void refreshOverlaps()
{
if (NEVER_PURGE_TOMBSTONES)
return;
if (this.overlappingSSTables != null)
overlappingSSTables.release();
@ -112,7 +128,7 @@ public class CompactionController implements AutoCloseable
{
logger.debug("Checking droppable sstables in {}", cfStore);
if (compacting == null)
if (compacting == null || NEVER_PURGE_TOMBSTONES)
return Collections.<SSTableReader>emptySet();
List<SSTableReader> candidates = new ArrayList<SSTableReader>();
@ -175,6 +191,9 @@ public class CompactionController implements AutoCloseable
*/
public long maxPurgeableTimestamp(DecoratedKey key)
{
if (NEVER_PURGE_TOMBSTONES)
return Long.MIN_VALUE;
List<SSTableReader> filteredSSTables = overlappingTree.search(key);
long min = Long.MAX_VALUE;
for (SSTableReader sstable : filteredSSTables)
@ -191,6 +210,7 @@ public class CompactionController implements AutoCloseable
public void close()
{
overlappingSSTables.release();
if (overlappingSSTables != null)
overlappingSSTables.release();
}
}

125
test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java Normal file
View File

@ -0,0 +1,125 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.compaction;
import java.util.Collection;
import org.junit.Test;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DeletedCell;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.OnDiskAtom;
import org.apache.cassandra.db.RangeTombstone;
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.SSTableReader;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class NeverPurgeTest extends CQLTester
{
static
{
System.setProperty("cassandra.never_purge_tombstones", "true");
}
@Test
public void neverPurgeCellTombstoneTest() throws Throwable
{
testHelper("UPDATE %s SET c = null WHERE a=1 AND b=2");
}
@Test
public void neverPurgeRangeTombstoneTest() throws Throwable
{
testHelper("DELETE FROM %s WHERE a=1 AND b=2");
}
@Test
public void neverPurgePartitionTombstoneTest() throws Throwable
{
testHelper("DELETE FROM %s WHERE a=1");
}
@Test
public void minorNeverPurgeTombstonesTest() throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 0");
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
cfs.disableAutoCompaction();
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < 1000; j++)
{
execute("INSERT INTO %s (a, b, c) VALUES (" + j + ", 2, '3')");
}
cfs.forceBlockingFlush();
}
execute("UPDATE %s SET c = null WHERE a=1 AND b=2");
execute("DELETE FROM %s WHERE a=2 AND b=2");
execute("DELETE FROM %s WHERE a=3");
cfs.forceBlockingFlush();
cfs.enableAutoCompaction();
while (cfs.getSSTables().size() > 1)
Thread.sleep(100);
verifyContainsTombstones(cfs.getSSTables(), 3);
}
private void testHelper(String deletionStatement) throws Throwable
{
createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 0");
ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
execute("INSERT INTO %s (a, b, c) VALUES (1, 2, '3')");
execute(deletionStatement);
Thread.sleep(1000);
cfs.forceBlockingFlush();
cfs.forceMajorCompaction();
verifyContainsTombstones(cfs.getSSTables(), 1);
}
private void verifyContainsTombstones(Collection<SSTableReader> sstables, int expectedTombstoneCount) throws Exception
{
assertTrue(sstables.size() == 1); // always run a major compaction before calling this
SSTableReader sstable = sstables.iterator().next();
int tombstoneCount = 0;
try (ISSTableScanner scanner = sstable.getScanner())
{
while (scanner.hasNext())
{
try (OnDiskAtomIterator iter = scanner.next())
{
if (iter.getColumnFamily().deletionInfo().getTopLevelDeletion().localDeletionTime < Integer.MAX_VALUE)
tombstoneCount++;
while (iter.hasNext())
{
OnDiskAtom atom = iter.next();
if (atom instanceof DeletedCell || atom instanceof RangeTombstone)
tombstoneCount++;
}
}
}
}
assertEquals(tombstoneCount, expectedTombstoneCount);
}
}