natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove memory emergency pressure valve logic 

patch by Aleksey Yeschenko; reviewed by Jonathan Ellis for
CASSANDRA-3534
This commit is contained in:
Aleksey Yeschenko 2013-03-11 19:24:27 +03:00
parent 0f72806c3c
commit e79d9fbf84
12 changed files with 9 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -13,6 +13,7 @@
* Change order of directory searching for c*.in.sh (CASSANDRA-3983)
* Add tool to reset SSTable level (CASSANDRA-5271)
* Allow custom configuration loader (CASSANDRA-5045)
* Remove memory emergency pressure valve logic (CASSANDRA-3534)
1.2.3

6
NEWS.txt
View File

@ -15,7 +15,11 @@ Upgrading
---------
- Replication and strategy options do not accept unknown options anymore.
This was already the case for CQL3 in 1.2 but this is now the case for
thrift to.
thrift too.
- reduce_cache_sizes_at, reduce_cache_capacity_to, and
flush_largest_memtables_at options have been removed from cassandra.yaml.
- CacheServiceMBean.reduceCacheSizes() has been removed.
Use CacheServiceMBean.set{Key,Row}CacheCapacityInMB() instead.
1.2.3

25
conf/cassandra.yaml
View File

@ -237,31 +237,6 @@ seed_provider:
# Ex: "<ip1>,<ip2>,<ip3>"
- seeds: "127.0.0.1"
# emergency pressure valve: each time heap usage after a full (CMS)
# garbage collection is above this fraction of the max, Cassandra will
# flush the largest memtables.
#
# Set to 1.0 to disable. Setting this lower than
# CMSInitiatingOccupancyFraction is not likely to be useful.
#
# RELYING ON THIS AS YOUR PRIMARY TUNING MECHANISM WILL WORK POORLY:
# it is most effective under light to moderate load, or read-heavy
# workloads; under truly massive write load, it will often be too
# little, too late.
flush_largest_memtables_at: 0.75
# emergency pressure valve #2: the first time heap usage after a full
# (CMS) garbage collection is above this fraction of the max,
# Cassandra will reduce cache maximum _capacity_ to the given fraction
# of the current _size_. Should usually be set substantially above
# flush_largest_memtables_at, since that will have less long-term
# impact on the system.
#
# Set to 1.0 to disable. Setting this lower than
# CMSInitiatingOccupancyFraction is not likely to be useful.
reduce_cache_sizes_at: 0.85
reduce_cache_capacity_to: 0.6
# For workloads with more data than can fit in memory, Cassandra's
# bottleneck will be reads that need to fetch data from
# disk. "concurrent_reads" should be set to (16 * number_of_drives) in

13
src/java/org/apache/cassandra/cache/AutoSavingCache.java vendored
View File

@ -176,19 +176,6 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
return CompactionManager.instance.submitCacheWrite(getWriter(keysToSave));
}
public void reduceCacheSize()
{
if (getCapacity() > 0)
{
int newCapacity = (int) (DatabaseDescriptor.getReduceCacheCapacityTo() * weightedSize());
logger.warn(String.format("Reducing %s capacity from %d to %s to reduce memory pressure",
cacheType, getCapacity(), newCapacity));
setCapacity(newCapacity);
}
}
public class Writer extends CompactionInfo.Holder
{
private final Set<K> keys;

3
src/java/org/apache/cassandra/config/Config.java
View File

@ -144,9 +144,6 @@ public class Config
public InternodeCompression internode_compression = InternodeCompression.none;
public Double flush_largest_memtables_at = 1.0;
public Double reduce_cache_sizes_at = 1.0;
public double reduce_cache_capacity_to = 0.6;
public int hinted_handoff_throttle_in_kb = 1024;
public int max_hints_delivery_threads = 1;
public boolean compaction_preheat_key_cache = true;

16
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -37,7 +37,6 @@ import org.apache.cassandra.config.EncryptionOptions.ServerEncryptionOptions;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DefsTable;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.FSWriteError;
@ -1107,21 +1106,6 @@ public class DatabaseDescriptor
return conf.client_encryption_options;
}
public static double getFlushLargestMemtablesAt()
{
return conf.flush_largest_memtables_at;
}
public static double getReduceCacheSizesAt()
{
return conf.reduce_cache_sizes_at;
}
public static double getReduceCacheCapacityTo()
{
return conf.reduce_cache_capacity_to;
}
public static int getHintedHandoffThrottleInKB()
{
return conf.hinted_handoff_throttle_in_kb;

16
src/java/org/apache/cassandra/service/CacheService.java
View File

@ -296,22 +296,6 @@ public class CacheService implements CacheServiceMBean
return keyCache.size();
}
public void reduceCacheSizes()
{
reduceRowCacheSize();
reduceKeyCacheSize();
}
public void reduceRowCacheSize()
{
rowCache.reduceCacheSize();
}
public void reduceKeyCacheSize()
{
keyCache.reduceCacheSize();
}
public void saveCaches() throws ExecutionException, InterruptedException
{
List<Future<?>> futures = new ArrayList<Future<?>>(2);

5
src/java/org/apache/cassandra/service/CacheServiceMBean.java
View File

@ -41,11 +41,6 @@ public interface CacheServiceMBean
public void setKeyCacheCapacityInMB(long capacity);
/**
* sets each cache's maximum capacity to "reduce_cache_capacity_to" of its current size
*/
public void reduceCacheSizes();
/**
* save row and key caches
*

20
src/java/org/apache/cassandra/service/GCInspector.java
View File

@ -31,7 +31,6 @@ import javax.management.ObjectName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.utils.StatusLogger;
@ -50,8 +49,6 @@ public class GCInspector
final List<GarbageCollectorMXBean> beans = new ArrayList<GarbageCollectorMXBean>();
final MemoryMXBean membean = ManagementFactory.getMemoryMXBean();
private volatile boolean cacheSizesReduced;
public GCInspector()
{
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
@ -125,24 +122,7 @@ public class GCInspector
// if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
if (gc.getName().equals("ConcurrentMarkSweep"))
{
SSTableDeletingTask.rescheduleFailedTasks();
double usage = (double) memoryUsed / memoryMax;
if (memoryUsed > DatabaseDescriptor.getReduceCacheSizesAt() * memoryMax && !cacheSizesReduced)
{
cacheSizesReduced = true;
logger.warn("Heap is " + usage + " full. You may need to reduce memtable and/or cache sizes. Cassandra is now reducing cache sizes to free up memory. Adjust reduce_cache_sizes_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
CacheService.instance.reduceCacheSizes();
}
if (memoryUsed > DatabaseDescriptor.getFlushLargestMemtablesAt() * memoryMax)
{
logger.warn("Heap is " + usage + " full. You may need to reduce memtable and/or cache sizes. Cassandra will now flush up to the two largest memtables to free up memory. Adjust flush_largest_memtables_at threshold in cassandra.yaml if you don't want Cassandra to do this automatically");
StorageService.instance.flushLargestMemtables();
}
}
}
}
}

26
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -3516,32 +3516,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
((DynamicEndpointSnitch)oldSnitch).unregisterMBean();
}
/**
* Flushes the two largest memtables by ops and by throughput
*/
public void flushLargestMemtables()
{
ColumnFamilyStore largest = null;
for (ColumnFamilyStore cfs : ColumnFamilyStore.all())
{
long total = cfs.getTotalMemtableLiveSize();
if (total > 0 && (largest == null || total > largest.getTotalMemtableLiveSize()))
{
logger.debug(total + " estimated memtable size for " + cfs);
largest = cfs;
}
}
if (largest == null)
{
logger.info("Unable to reduce heap usage since there are no dirty column families");
return;
}
logger.warn("Flushing " + largest + " to relieve memory pressure");
largest.forceFlush();
}
/**
* Seed data to the endpoints that will be responsible for it at the future
*

1
test/conf/cassandra.yaml
View File

@ -31,5 +31,4 @@ server_encryption_options:
truststore: conf/.truststore
truststore_password: cassandra
incremental_backups: true
flush_largest_memtables_at: 1.0
compaction_throughput_mb_per_sec: 0

9
test/unit/org/apache/cassandra/db/RowCacheTest.java
View File

@ -114,7 +114,7 @@ public class RowCacheTest extends SchemaLoader
public void testRowCacheLoad() throws Exception
{
CacheService.instance.setRowCacheCapacityInMB(1);
rowCacheLoad(100, Integer.MAX_VALUE, false);
rowCacheLoad(100, Integer.MAX_VALUE);
CacheService.instance.setRowCacheCapacityInMB(0);
}
@ -122,11 +122,11 @@ public class RowCacheTest extends SchemaLoader
public void testRowCachePartialLoad() throws Exception
{
CacheService.instance.setRowCacheCapacityInMB(1);
rowCacheLoad(100, 50, true);
rowCacheLoad(100, 50);
CacheService.instance.setRowCacheCapacityInMB(0);
}
public void rowCacheLoad(int totalKeys, int keysToSave, boolean reduceLoadCapacity) throws Exception
public void rowCacheLoad(int totalKeys, int keysToSave) throws Exception
{
CompactionManager.instance.disableAutoCompaction();
@ -144,9 +144,6 @@ public class RowCacheTest extends SchemaLoader
// force the cache to disk
CacheService.instance.rowCache.submitWrite(keysToSave).get();
if (reduceLoadCapacity)
CacheService.instance.reduceRowCacheSize();
// empty the cache again to make sure values came from disk
CacheService.instance.invalidateRowCache();
assert CacheService.instance.rowCache.size() == 0;