natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add option to change how often we check for expired sstables with DTCS 

Patch by marcuse; reviewed by yukim for CASSANDRA-9882
This commit is contained in:
Marcus Eriksson 2015-08-17 09:50:37 +02:00
parent f53aacba80
commit 17042f19b0
3 changed files with 25 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
View File

@ -38,8 +38,7 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
protected DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
@VisibleForTesting
long lastExpiredCheck;
private long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
{
@ -87,8 +86,8 @@ public class DateTieredCompactionStrategy extends AbstractCompactionStrategy
Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
Set<SSTableReader> expired = Collections.emptySet();
// we only check for expired sstables every 10 minutes due to it being an expensive operation
if (System.currentTimeMillis() - lastExpiredCheck > TimeUnit.MINUTES.toMillis(10))
// we only check for expired sstables every 10 minutes (by default) due to it being an expensive operation
if (System.currentTimeMillis() - lastExpiredCheck > options.expiredSSTableCheckFrequency)
{
// Find fully expired SSTables. Those will be included no matter what.
expired = CompactionController.getFullyExpiredSSTables(cfs, uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);

21
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyOptions.java
View File

@ -27,12 +27,15 @@ public final class DateTieredCompactionStrategyOptions
protected static final TimeUnit DEFAULT_TIMESTAMP_RESOLUTION = TimeUnit.MICROSECONDS;
protected static final double DEFAULT_MAX_SSTABLE_AGE_DAYS = 365;
protected static final long DEFAULT_BASE_TIME_SECONDS = 60;
protected static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
protected static final String TIMESTAMP_RESOLUTION_KEY = "timestamp_resolution";
protected static final String MAX_SSTABLE_AGE_KEY = "max_sstable_age_days";
protected static final String BASE_TIME_KEY = "base_time_seconds";
protected static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY = "expired_sstable_check_frequency_seconds";
protected final long maxSSTableAge;
protected final long baseTime;
protected final long expiredSSTableCheckFrequency;
public DateTieredCompactionStrategyOptions(Map<String, String> options)
{
@ -43,12 +46,15 @@ public final class DateTieredCompactionStrategyOptions
maxSSTableAge = Math.round(fractionalDays * timestampResolution.convert(1, TimeUnit.DAYS));
optionValue = options.get(BASE_TIME_KEY);
baseTime = timestampResolution.convert(optionValue == null ? DEFAULT_BASE_TIME_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue), TimeUnit.SECONDS);
}
public DateTieredCompactionStrategyOptions()
{
maxSSTableAge = Math.round(DEFAULT_MAX_SSTABLE_AGE_DAYS * DEFAULT_TIMESTAMP_RESOLUTION.convert(1, TimeUnit.DAYS));
baseTime = DEFAULT_TIMESTAMP_RESOLUTION.convert(DEFAULT_BASE_TIME_SECONDS, TimeUnit.SECONDS);
expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS, TimeUnit.SECONDS);
}
public static Map<String, String> validateOptions(Map<String, String> options, Map<String, String> uncheckedOptions) throws ConfigurationException
@ -92,9 +98,24 @@ public final class DateTieredCompactionStrategyOptions
throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, BASE_TIME_KEY), e);
}
optionValue = options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
try
{
long expiredCheckFrequency = optionValue == null ? DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS : Long.parseLong(optionValue);
if (expiredCheckFrequency < 0)
{
throw new ConfigurationException(String.format("%s must not be negative, but was %d", EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, expiredCheckFrequency));
}
}
catch (NumberFormatException e)
{
throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s", optionValue, EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY), e);
}
uncheckedOptions.remove(MAX_SSTABLE_AGE_KEY);
uncheckedOptions.remove(BASE_TIME_KEY);
uncheckedOptions.remove(TIMESTAMP_RESOLUTION_KEY);
uncheckedOptions.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY);
return uncheckedOptions;
}

2
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
View File

@ -305,10 +305,10 @@ public class DateTieredCompactionStrategyTest extends SchemaLoader
options.put(DateTieredCompactionStrategyOptions.BASE_TIME_KEY, "30");
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY, "MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY, Double.toString((1d / (24 * 60 * 60))));
options.put(DateTieredCompactionStrategyOptions.EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_KEY, "0");
DateTieredCompactionStrategy dtcs = new DateTieredCompactionStrategy(cfs, options);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int) (System.currentTimeMillis() / 1000)));
dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int) (System.currentTimeMillis()/1000));
assertNotNull(t);