Revert 7486, make CMS default GC, move gc opts to conf/jvm.options

Patch by pmotta; reviewed by jmckenzie for CASSANDRA-10403
This commit is contained in:
Paulo Motta 2015-10-14 14:36:52 -04:00 committed by Joshua McKenzie
parent ebb2b1beda
commit 1415fa512a
7 changed files with 280 additions and 87 deletions

View File

@ -1,4 +1,6 @@
3.0-rc2
* Revert CASSANDRA-7486, make CMS default GC, move GC config to
conf/jvm.options (CASSANDRA-10403)
* Fix TeeingAppender causing some logs to be truncated/empty (CASSANDRA-10447)
* Allow EACH_QUORUM for reads (CASSANDRA-9602)
* Fix potential ClassCastException while upgrading (CASSANDRA-10468)

View File

@ -54,7 +54,7 @@ Upgrading
- Max mutation size is now configurable via max_mutation_size_in_kb setting in
cassandra.yaml; the default is half the size commitlog_segment_size_in_mb * 1024.
- 3.0 requires Java 8u40 or later.
- The default JVM GC has been changed to G1GC.
- Garbage collection options were moved from cassandra-env to jvm.options file.
- New transaction log files have been introduced to replace the compactions_in_progress
system table, temporary file markers (tmp and tmplink) and sstable ancerstors.
Therefore, compaction metadata no longer contains ancestors. Transaction log files

View File

@ -128,16 +128,18 @@ Function CalculateHeapSizes
}
# Validate that we need to run this function and that our config is good
if ($env:MAX_HEAP_SIZE)
if (($env:MAX_HEAP_SIZE -and !$env:HEAP_NEWSIZE) -or (!$env:MAX_HEAP_SIZE -and $env:HEAP_NEWSIZE))
{
return
echo "Please set or unset MAX_HEAP_SIZE and HEAP_NEWSIZE in pairs. Aborting startup."
exit 1
}
$memObject = Get-WMIObject -class win32_physicalmemory
if ($memObject -eq $null)
{
echo "WARNING! Could not determine system memory. Defaulting to 2G heap. Manually override in conf\cassandra-env.ps1 for different heap values."
echo "WARNING! Could not determine system memory. Defaulting to 2G heap, 512M newgen. Manually override in conf\jvm.options for different heap values."
$env:MAX_HEAP_SIZE = "2048M"
$env:HEAP_NEWSIZE = "512M"
return
}
@ -174,6 +176,20 @@ Function CalculateHeapSizes
$maxHeapMB = $quarterMem
}
$env:MAX_HEAP_SIZE = [System.Convert]::ToString($maxHeapMB) + "M"
# Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4
$maxYGPerCore = 100
$maxYGTotal = $maxYGPerCore * $systemCores
$desiredYG = [Math]::Truncate($maxHeapMB / 4)
if ($desiredYG -gt $maxYGTotal)
{
$env:HEAP_NEWSIZE = [System.Convert]::ToString($maxYGTotal) + "M"
}
else
{
$env:HEAP_NEWSIZE = [System.Convert]::ToString($desiredYG) + "M"
}
}
#-----------------------------------------------------------------------------
@ -294,12 +310,71 @@ Function SetCassandraEnvironment
# Override these to set the amount of memory to allocate to the JVM at
# start-up. For production use you may wish to adjust this for your
# environment. MAX_HEAP_SIZE is the total amount of memory dedicated
# to the Java heap.
# to the Java heap. HEAP_NEWSIZE refers to the size of the young
# generation. Both MAX_HEAP_SIZE and HEAP_NEWSIZE should be either set
# or not (if you set one, set the other).
#
# The main trade-off for the young generation is that the larger it
# is, the longer GC pause times will be. The shorter it is, the more
# expensive GC will be (usually).
#
# The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent
# times. If in doubt, and if you do not particularly want to tweak, go
# 100 MB per physical CPU core.
#$env:MAX_HEAP_SIZE="4096M"
#$env:HEAP_NEWSIZE="800M"
CalculateHeapSizes
ParseJVMInfo
# Read user-defined JVM options from jvm.options file
$content = Get-Content "$env:CASSANDRA_CONF\jvm.options"
for ($i = 0; $i -lt $content.Count; $i++)
{
$line = $content[$i]
if ($line.StartsWith("-"))
{
$env:JVM_OPTS = "$env:JVM_OPTS $line"
}
}
$defined_xmn = $env:JVM_OPTS -like '*Xmn*'
$defined_xmx = $env:JVM_OPTS -like '*Xmx*'
$defined_xms = $env:JVM_OPTS -like '*Xms*'
$using_cms = $env:JVM_OPTS -like '*UseConcMarkSweepGC*'
# We only set -Xms and -Xmx if they were not defined on jvm.options file
# If defined, both Xmx and Xms should be defined together.
if (($defined_xmx -eq $false) -and ($defined_xms -eq $false))
{
$env:JVM_OPTS="$env:JVM_OPTS -Xms$env:MAX_HEAP_SIZE"
$env:JVM_OPTS="$env:JVM_OPTS -Xmx$env:MAX_HEAP_SIZE"
}
elseif (($defined_xmx -eq $false) -or ($defined_xms -eq $false))
{
echo "Please set or unset -Xmx and -Xms flags in pairs on jvm.options file."
exit
}
# We only set -Xmn flag if it was not defined in jvm.options file
# and if the CMS GC is being used
# If defined, both Xmn and Xmx should be defined together.
if (($defined_xmn -eq $true) -and ($defined_xmx -eq $false))
{
echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm.options file."
exit
}
elseif (($defined_xmn -eq $false) -and ($using_cms -eq $true))
{
$env:JVM_OPTS="$env:JVM_OPTS -Xmn$env:HEAP_NEWSIZE"
}
if (($env:JVM_ARCH -eq "64-Bit") -and ($using_cms -eq $true))
{
$env:JVM_OPTS="$env:JVM_OPTS -XX:+UseCondCardMark"
}
# Add sigar env - see Cassandra-7838
$env:JVM_OPTS = "$env:JVM_OPTS -Djava.library.path=""$env:CASSANDRA_HOME\lib\sigar-bin"""
@ -355,10 +430,6 @@ Function SetCassandraEnvironment
# see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar
$env:JVM_OPTS="$env:JVM_OPTS -XX:ThreadPriorityPolicy=42"
# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize.
$env:JVM_OPTS="$env:JVM_OPTS -Xms$env:MAX_HEAP_SIZE"
$env:JVM_OPTS="$env:JVM_OPTS -Xmx$env:MAX_HEAP_SIZE"
$env:JVM_OPTS="$env:JVM_OPTS -XX:+HeapDumpOnOutOfMemoryError"
# Per-thread stack size.
@ -367,28 +438,6 @@ Function SetCassandraEnvironment
# Larger interned string table, for gossip's benefit (CASSANDRA-6410)
$env:JVM_OPTS="$env:JVM_OPTS -XX:StringTableSize=1000003"
# GC tuning options
# Use the Hotspot garbage-first collector.
$env:JVM_OPTS="$env:JVM_OPTS -XX:+UseG1GC"
# Have the JVM do less remembered set work during STW, instead
# preferring concurrent GC. Reduces p99.9 latency.
$env:JVM_OPTS="$env:JVM_OPTS -XX:G1RSetUpdatingPauseTimePercent=5"
# The JVM maximum is 8 PGC threads and 1/4 of that for ConcGC.
# Machines with > 10 cores may need additional threads. Increase to <= full cores.
#$env:JVM_OPTS="$env:JVM_OPTS -XX:ParallelGCThreads=16"
#$env:JVM_OPTS="$env:JVM_OPTS -XX:ConcGCThreads=16"
# Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
# 200ms is the JVM default and lowest viable setting
# 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
$env:JVM_OPTS="$env:JVM_OPTS -XX:MaxGCPauseMillis=500"
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
#$env:JVM_OPTS="$env:JVM_OPTS -XX:InitiatingHeapOccupancyPercent=70"
# Make sure all memory is faulted and zeroed on startup.
# This helps prevent soft faults in containers and makes
# transparent hugepage allocation more effective.
@ -404,19 +453,6 @@ Function SetCassandraEnvironment
# http://www.evanjones.ca/jvm-mmap-pause.html
$env:JVM_OPTS="$env:JVM_OPTS -XX:+PerfDisableSharedMem"
# GC logging options -- uncomment to enable
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintGCDetails"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintGCDateStamps"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintHeapAtGC"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintTenuringDistribution"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintGCApplicationStoppedTime"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+PrintPromotionFailure"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:PrintFLSStatistics=1"
# $env:JVM_OPTS="$env:JVM_OPTS -Xloggc:$env:CASSANDRA_HOME/logs/gc.log"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseGCLogFileRotation"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:NumberOfGCLogFiles=10"
# $env:JVM_OPTS="$env:JVM_OPTS -XX:GCLogFileSize=10M"
# Configure the following for JEMallocAllocator and if jemalloc is not available in the system
# library path.
# set LD_LIBRARY_PATH=<JEMALLOC_HOME>/lib/

View File

@ -71,6 +71,19 @@ calculate_heap_sizes()
max_heap_size_in_mb="$quarter_system_memory_in_mb"
fi
MAX_HEAP_SIZE="${max_heap_size_in_mb}M"
# Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap size)
max_sensible_yg_per_core_in_mb="100"
max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*" $system_cpu_cores`
desired_yg_in_mb=`expr $max_heap_size_in_mb / 4`
if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ]
then
HEAP_NEWSIZE="${max_sensible_yg_in_mb}M"
else
HEAP_NEWSIZE="${desired_yg_in_mb}M"
fi
}
# Determine the sort of JVM we'll be running on.
@ -89,19 +102,54 @@ if [ "$JVM_VERSION" \< "1.8" ] && [ "$JVM_PATCH_VERSION" \< "40" ] ; then
exit 1;
fi
jvm=`echo "$java_ver_output" | grep -A 1 'java version' | awk 'NR==2 {print $1}'`
case "$jvm" in
OpenJDK)
JVM_VENDOR=OpenJDK
# this will be "64-Bit" or "32-Bit"
JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
;;
"Java(TM)")
JVM_VENDOR=Oracle
# this will be "64-Bit" or "32-Bit"
JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
;;
*)
# Help fill in other JVM values
JVM_VENDOR=other
JVM_ARCH=unknown
;;
esac
# Override these to set the amount of memory to allocate to the JVM at
# start-up. For production use you may wish to adjust this for your
# environment. MAX_HEAP_SIZE is the total amount of memory dedicated
# to the Java heap.
# to the Java heap. HEAP_NEWSIZE refers to the size of the young
# generation. Both MAX_HEAP_SIZE and HEAP_NEWSIZE should be either set
# or not (if you set one, set the other).
#
# The main trade-off for the young generation is that the larger it
# is, the longer GC pause times will be. The shorter it is, the more
# expensive GC will be (usually).
#
# The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause
# times. If in doubt, and if you do not particularly want to tweak, go with
# 100 MB per physical CPU core.
#MAX_HEAP_SIZE="4G"
#HEAP_NEWSIZE="800M"
# Set this to control the amount of arenas per-thread in glibc
#export MALLOC_ARENA_MAX=4
# only calculate the size if it's not set manually
if [ "x$MAX_HEAP_SIZE" = "x" ] ; then
if [ "x$MAX_HEAP_SIZE" = "x" ] && [ "x$HEAP_NEWSIZE" = "x" ]; then
calculate_heap_sizes
else
if [ "x$MAX_HEAP_SIZE" = "x" ] || [ "x$HEAP_NEWSIZE" = "x" ]; then
echo "please set or unset MAX_HEAP_SIZE and HEAP_NEWSIZE in pairs (see cassandra-env.sh)"
exit 1
fi
fi
if [ "x$MALLOC_ARENA_MAX" = "x" ] ; then
@ -158,41 +206,54 @@ esac
# Here we create the arguments that will get passed to the jvm when
# starting cassandra.
# Read user-defined JVM options from jvm.options file
JVM_OPTS_FILE=$CASSANDRA_CONF/jvm.options
for opt in `grep "^-" $JVM_OPTS_FILE`
do
JVM_OPTS="$JVM_OPTS $opt"
done
# Check what parameters were defined on jvm.options file to avoid conflicts
echo $JVM_OPTS | grep -q Xmn
DEFINED_XMN=$?
echo $JVM_OPTS | grep -q Xmx
DEFINED_XMX=$?
echo $JVM_OPTS | grep -q Xms
DEFINED_XMS=$?
echo $JVM_OPTS | grep -q UseConcMarkSweepGC
USING_CMS=$?
# We only set -Xms and -Xmx if they were not defined on jvm.options file
# If defined, both Xmx and Xms should be defined together.
if [ $DEFINED_XMX -ne 0 ] && [ $DEFINED_XMS -ne 0 ]; then
JVM_OPTS="$JVM_OPTS -Xms${MAX_HEAP_SIZE}"
JVM_OPTS="$JVM_OPTS -Xmx${MAX_HEAP_SIZE}"
elif [ $DEFINED_XMX -ne 0 ] || [ $DEFINED_XMS -ne 0 ]; then
echo "Please set or unset -Xmx and -Xms flags in pairs on jvm.options file."
exit 1
fi
# We only set -Xmn flag if it was not defined in jvm.options file
# and if the CMS GC is being used
# If defined, both Xmn and Xmx should be defined together.
if [ $DEFINED_XMN -eq 0 ] && [ $DEFINED_XMX -ne 0 ]; then
echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm.options file."
exit 1
elif [ $DEFINED_XMN -ne 0 ] && [ $USING_CMS -eq 0 ]; then
JVM_OPTS="$JVM_OPTS -Xmn${HEAP_NEWSIZE}"
fi
if [ "$JVM_ARCH" = "64-Bit" ] && [ $USING_CMS -eq 0 ]; then
JVM_OPTS="$JVM_OPTS -XX:+UseCondCardMark"
fi
# enable assertions. disabling this in production will give a modest
# performance benefit (around 5%).
JVM_OPTS="$JVM_OPTS -ea"
# min and max heap sizes should be set to the same value to avoid
# stop-the-world GC pauses during resize, and so that we can lock the
# heap in memory on startup to prevent any of it from being swapped
# out.
JVM_OPTS="$JVM_OPTS -Xms${MAX_HEAP_SIZE}"
JVM_OPTS="$JVM_OPTS -Xmx${MAX_HEAP_SIZE}"
# Per-thread stack size.
JVM_OPTS="$JVM_OPTS -Xss256k"
# Use the Hotspot garbage-first collector.
JVM_OPTS="$JVM_OPTS -XX:+UseG1GC"
# Have the JVM do less remembered set work during STW, instead
# preferring concurrent GC. Reduces p99.9 latency.
JVM_OPTS="$JVM_OPTS -XX:G1RSetUpdatingPauseTimePercent=5"
# The JVM maximum is 8 PGC threads and 1/4 of that for ConcGC.
# Machines with > 10 cores may need additional threads. Increase to <= full cores.
#JVM_OPTS="$JVM_OPTS -XX:ParallelGCThreads=16"
#JVM_OPTS="$JVM_OPTS -XX:ConcGCThreads=16"
# Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
# 200ms is the JVM default and lowest viable setting
# 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
JVM_OPTS="$JVM_OPTS -XX:MaxGCPauseMillis=500"
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
#JVM_OPTS="$JVM_OPTS -XX:InitiatingHeapOccupancyPercent=70"
# Make sure all memory is faulted and zeroed on startup.
# This helps prevent soft faults in containers and makes
# transparent hugepage allocation more effective.
@ -230,19 +291,6 @@ if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then
JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=$CASSANDRA_HEAPDUMP_DIR/cassandra-`date +%s`-pid$$.hprof"
fi
# GC logging options -- uncomment to enable
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCDetails"
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCDateStamps"
# JVM_OPTS="$JVM_OPTS -XX:+PrintHeapAtGC"
# JVM_OPTS="$JVM_OPTS -XX:+PrintTenuringDistribution"
# JVM_OPTS="$JVM_OPTS -XX:+PrintGCApplicationStoppedTime"
# JVM_OPTS="$JVM_OPTS -XX:+PrintPromotionFailure"
# JVM_OPTS="$JVM_OPTS -XX:PrintFLSStatistics=1"
# JVM_OPTS="$JVM_OPTS -Xloggc:/var/log/cassandra/gc.log"
# JVM_OPTS="$JVM_OPTS -XX:+UseGCLogFileRotation"
# JVM_OPTS="$JVM_OPTS -XX:NumberOfGCLogFiles=10"
# JVM_OPTS="$JVM_OPTS -XX:GCLogFileSize=10M"
# Configure the following for JEMallocAllocator and if jemalloc is not available in the system
# library path (Example: /usr/local/lib/). Usually "make install" will do the right thing.
# export LD_LIBRARY_PATH=<JEMALLOC_HOME>/lib/

105
conf/jvm.options Normal file
View File

@ -0,0 +1,105 @@
###########################################################################
# jvm.options #
# #
# - all flags defined here will be used by cassandra to startup the JVM #
# - one flag should be specified per line #
# - lines that do not start with '-' will be ignored #
# - only static flags are accepted (no variables or parameters) #
# - dynamic flags will be appended to these on cassandra-env #
###########################################################################
#################
# HEAP SETTINGS #
#################
# Heap size is automatically calculated by cassandra-env based on this
# formula: max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB))
# That is:
# - calculate 1/2 ram and cap to 1024MB
# - calculate 1/4 ram and cap to 8192MB
# - pick the max
#
# For production use you may wish to adjust this for your environment.
# If that's the case, uncomment the -Xmx and Xms options below to override the
# automatic calculation of JVM heap memory.
#
# It is recommended to set min (-Xms) and max (-Xmx) heap sizes to
# the same value to avoid stop-the-world GC pauses during resize, and
# so that we can lock the heap in memory on startup to prevent any
# of it from being swapped out.
#-Xms4G
#-Xmx4G
# Young generation size is automatically calculated by cassandra-env
# based on this formula: min(100 * num_cores, 1/4 * heap size)
#
# The main trade-off for the young generation is that the larger it
# is, the longer GC pause times will be. The shorter it is, the more
# expensive GC will be (usually).
#
# It is not recommended to set the young generation size if using the
# G1 GC, since that will override the target pause-time goal.
# More info: http://www.oracle.com/technetwork/articles/java/g1gc-1984535.html
#
# The example below assumes a modern 8-core+ machine for decent
# times. If in doubt, and if you do not particularly want to tweak, go
# 100 MB per physical CPU core.
#-Xmn800M
#################
# GC SETTINGS #
#################
### CMS Settings
-XX:+UseParNewGC
-XX:+UseConcMarkSweepGC
-XX:+CMSParallelRemarkEnabled
-XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=1
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-XX:CMSWaitDuration=10000
-XX:+CMSParallelInitialMarkEnabled
-XX:+CMSEdenChunksRecordAlways
# some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541
-XX:+CMSClassUnloadingEnabled
### G1 Settings (experimental, comment previous section and uncomment section below to enable)
## Use the Hotspot garbage-first collector.
#-XX:+UseG1GC
#
## Have the JVM do less remembered set work during STW, instead
## preferring concurrent GC. Reduces p99.9 latency.
#-XX:G1RSetUpdatingPauseTimePercent=5
#
## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
## 200ms is the JVM default and lowest viable setting
## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
#-XX:MaxGCPauseMillis=500
## Optional G1 Settings
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
#-XX:InitiatingHeapOccupancyPercent=70
# The JVM maximum is 8 PGC threads and 1/4 of that for ConcGC.
# Machines with > 10 cores may need additional threads. Increase to <= full cores.
#-XX:ParallelGCThreads=16
#-XX:ConcGCThreads=16
### GC logging options -- uncomment to enable
#-XX:+PrintGCDetails
#-XX:+PrintGCDateStamps
#-XX:+PrintHeapAtGC
#-XX:+PrintTenuringDistribution
#-XX:+PrintGCApplicationStoppedTime
#-XX:+PrintPromotionFailure
#-XX:PrintFLSStatistics=1
#-Xloggc:/var/log/cassandra/gc.log
#-XX:+UseGCLogFileRotation
#-XX:NumberOfGCLogFiles=10
#-XX:GCLogFileSize=10M

View File

@ -5,6 +5,7 @@ conf/commitlog_archiving.properties etc/cassandra
conf/cassandra-topology.properties etc/cassandra
conf/logback.xml etc/cassandra
conf/logback-tools.xml etc/cassandra
conf/jvm.options etc/cassandra
conf/triggers/* etc/cassandra/triggers
debian/cassandra.in.sh usr/share/cassandra
debian/cassandra.conf etc/security/limits.d

1
debian/init vendored
View File

@ -21,6 +21,7 @@ SCRIPTNAME=/etc/init.d/$NAME
CONFDIR=/etc/cassandra
WAIT_FOR_START=10
CASSANDRA_HOME=/usr/share/cassandra
CASSANDRA_CONF=$CONFDIR
FD_LIMIT=100000
[ -e /usr/share/cassandra/apache-cassandra.jar ] || exit 0