diff --git a/CHANGES.txt b/CHANGES.txt index 8cff71690d..20ccc4b27b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -17,6 +17,7 @@ * Mishandling of cells for removed/dropped columns when reading legacy files (CASSANDRA-13939) * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922) Merged from 2.2: + * Rely on the JVM to handle OutOfMemoryErrors (CASSANDRA-13006) * Grab refs during scrub/index redistribution/cleanup (CASSANDRA-13873) diff --git a/NEWS.txt b/NEWS.txt index 621866bc7c..1bbe1aa301 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -18,8 +18,13 @@ using the provided 'sstableupgrade' tool. Upgrading --------- - - Nothing specific to this release, but please see previous upgrading sections, - especially if you are upgrading from 2.2. + - Cassandra is now relying on the JVM options to properly shutdown on OutOfMemoryError. By default it will + rely on the OnOutOfMemoryError option as the ExitOnOutOfMemoryError and CrashOnOutOfMemoryError options + are not supported by the older 1.7 and 1.8 JVMs. A warning will be logged at startup if none of those JVM + options are used. See CASSANDRA-13006 for more details. + - Cassandra is not logging anymore by default an Heap histogram on OutOfMemoryError. To enable that behavior + set the 'cassandra.printHeapHistogramOnOutOfMemoryError' System property to 'true'. See CASSANDRA-13006 + for more details. Materialized Views ------------------- diff --git a/bin/cassandra b/bin/cassandra index ae48ba4837..1305db9e28 100755 --- a/bin/cassandra +++ b/bin/cassandra @@ -28,6 +28,7 @@ # # CLASSPATH -- A Java classpath containing everything necessary to run. # JVM_OPTS -- Additional arguments to the JVM for heap size, etc +# JVM_ON_OUT_OF_MEMORY_ERROR_OPT -- The OnOutOfMemoryError JVM option if specified # CASSANDRA_CONF -- Directory containing Cassandra configuration files. # # As a convenience, a fragment of shell is sourced in order to set one or @@ -199,12 +200,22 @@ launch_service() # to close stdout/stderr, but it's up to us not to background. if [ "x$foreground" != "x" ]; then cassandra_parms="$cassandra_parms -Dcassandra-foreground=yes" - exec $NUMACTL "$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" $props "$class" + if [ "x$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" != "x" ]; then + exec $NUMACTL "$JAVA" $JVM_OPTS "$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" $cassandra_parms -cp "$CLASSPATH" $props "$class" + else + exec $NUMACTL "$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" $props "$class" + fi # Startup CassandraDaemon, background it, and write the pid. else - exec $NUMACTL "$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" $props "$class" <&- & - [ ! -z "$pidpath" ] && printf "%d" $! > "$pidpath" - true + if [ "x$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" != "x" ]; then + exec $NUMACTL "$JAVA" $JVM_OPTS "$JVM_ON_OUT_OF_MEMORY_ERROR_OPT" $cassandra_parms -cp "$CLASSPATH" $props "$class" <&- & + [ ! -z "$pidpath" ] && printf "%d" $! > "$pidpath" + true + else + exec $NUMACTL "$JAVA" $JVM_OPTS $cassandra_parms -cp "$CLASSPATH" $props "$class" <&- & + [ ! -z "$pidpath" ] && printf "%d" $! > "$pidpath" + true + fi fi return $? diff --git a/conf/cassandra-env.ps1 b/conf/cassandra-env.ps1 index d3a0a87f06..405ed92e0b 100644 --- a/conf/cassandra-env.ps1 +++ b/conf/cassandra-env.ps1 @@ -1,391 +1,391 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: All param tuning can be done in the SetCassandraEnvironment Function below - -#----------------------------------------------------------------------------- -Function SetCassandraHome() -{ - if (! $env:CASSANDRA_HOME) - { - $cwd = [System.IO.Directory]::GetCurrentDirectory() - $cwd = Split-Path $cwd -parent - $env:CASSANDRA_HOME = $cwd -replace "\\", "/" - } -} - -#----------------------------------------------------------------------------- -Function SetCassandraMain() -{ - if (! $env:CASSANDRA_MAIN) - { - $env:CASSANDRA_MAIN="org.apache.cassandra.service.CassandraDaemon" - } -} - -#----------------------------------------------------------------------------- -Function BuildClassPath -{ - $cp = """$env:CASSANDRA_HOME\conf""" - foreach ($file in Get-ChildItem "$env:CASSANDRA_HOME\lib\*.jar") - { - $file = $file -replace "\\", "/" - $cp = $cp + ";" + """$file""" - } - - # Add build/classes/main so it works in development - $cp = $cp + ";" + """$env:CASSANDRA_HOME\build\classes\main"";""$env:CASSANDRA_HOME\build\classes\thrift""" - $env:CLASSPATH=$cp -} - -#----------------------------------------------------------------------------- -Function CalculateHeapSizes -{ - # Check if swapping is enabled on the host and warn if so - reference CASSANDRA-7316 - - $osInfo = Get-WmiObject -class "Win32_computersystem" - $autoPage = $osInfo.AutomaticManagedPageFile - - if ($autoPage) - { - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - echo "" - echo " WARNING! Automatic page file configuration detected." - echo " It is recommended that you disable swap when running Cassandra" - echo " for performance and stability reasons." - echo "" - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - } - else - { - $pageFileInfo = Get-WmiObject -class "Win32_PageFileSetting" -EnableAllPrivileges - $pageFileCount = $PageFileInfo.Count - if ($pageFileInfo) - { - $files = @() - $sizes = @() - $hasSizes = $FALSE - - # PageFileCount isn't populated and obj comes back as single if there's only 1 - if ([string]::IsNullOrEmpty($PageFileCount)) - { - $PageFileCount = 1 - $files += $PageFileInfo.Name - if ($PageFileInfo.MaximumSize -ne 0) - { - $hasSizes = $TRUE - $sizes += $PageFileInfo.MaximumSize - } - } - else - { - for ($i = 0; $i -le $PageFileCount; $i++) - { - $files += $PageFileInfo[$i].Name - if ($PageFileInfo[$i].MaximumSize -ne 0) - { - $hasSizes = $TRUE - $sizes += $PageFileInfo[$i].MaximumSize - } - } - } - - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - echo "" - echo " WARNING! $PageFileCount swap file(s) detected" - for ($i = 0; $i -lt $PageFileCount; $i++) - { - $toPrint = " Name: " + $files[$i] - if ($hasSizes) - { - $toPrint = $toPrint + " Size: " + $sizes[$i] - $toPrint = $toPrint -replace [Environment]::NewLine, "" - } - echo $toPrint - } - echo " It is recommended that you disable swap when running Cassandra" - echo " for performance and stability reasons." - echo "" - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - } - } - - # Validate that we need to run this function and that our config is good - if ($env:MAX_HEAP_SIZE -and $env:HEAP_NEWSIZE) - { - return - } - - if ((($env:MAX_HEAP_SIZE -and !$env:HEAP_NEWSIZE) -or (!$env:MAX_HEAP_SIZE -and $env:HEAP_NEWSIZE)) -and ($using_cms -eq $true)) - { - echo "Please set or unset MAX_HEAP_SIZE and HEAP_NEWSIZE in pairs. Aborting startup." - exit 1 - } - - $memObject = Get-WMIObject -class win32_physicalmemory - if ($memObject -eq $null) - { - echo "WARNING! Could not determine system memory. Defaulting to 2G heap, 512M newgen. Manually override in conf\jvm.options for different heap values." - $env:MAX_HEAP_SIZE = "2048M" - $env:HEAP_NEWSIZE = "512M" - return - } - - $memory = ($memObject | Measure-Object Capacity -Sum).sum - $memoryMB = [Math]::Truncate($memory / (1024*1024)) - - $cpu = gwmi Win32_ComputerSystem | Select-Object NumberOfLogicalProcessors - $systemCores = $cpu.NumberOfLogicalProcessors - - # set max heap size based on the following - # max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB)) - # calculate 1/2 ram and cap to 1024MB - # calculate 1/4 ram and cap to 8192MB - # pick the max - $halfMem = [Math]::Truncate($memoryMB / 2) - $quarterMem = [Math]::Truncate($halfMem / 2) - - if ($halfMem -gt 1024) - { - $halfMem = 1024 - } - if ($quarterMem -gt 8192) - { - $quarterMem = 8192 - } - - $maxHeapMB = "" - if ($halfMem -gt $quarterMem) - { - $maxHeapMB = $halfMem - } - else - { - $maxHeapMB = $quarterMem - } - $env:MAX_HEAP_SIZE = [System.Convert]::ToString($maxHeapMB) + "M" - - # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 - $maxYGPerCore = 100 - $maxYGTotal = $maxYGPerCore * $systemCores - $desiredYG = [Math]::Truncate($maxHeapMB / 4) - - if ($desiredYG -gt $maxYGTotal) - { - $env:HEAP_NEWSIZE = [System.Convert]::ToString($maxYGTotal) + "M" - } - else - { - $env:HEAP_NEWSIZE = [System.Convert]::ToString($desiredYG) + "M" - } -} - -#----------------------------------------------------------------------------- -Function ParseJVMInfo -{ - # grab info about the JVM - $pinfo = New-Object System.Diagnostics.ProcessStartInfo - $pinfo.FileName = "$env:JAVA_BIN" - $pinfo.RedirectStandardError = $true - $pinfo.RedirectStandardOutput = $true - $pinfo.UseShellExecute = $false - $pinfo.Arguments = "-d64 -version" - $p = New-Object System.Diagnostics.Process - $p.StartInfo = $pinfo - $p.Start() | Out-Null - $p.WaitForExit() - $stderr = $p.StandardError.ReadToEnd() - - $env:JVM_ARCH = "64-bit" - - if ($stderr.Contains("Error")) - { - # 32-bit JVM. re-run w/out -d64 - echo "Failed 64-bit check. Re-running to get version from 32-bit" - $pinfo.Arguments = "-version" - $p = New-Object System.Diagnostics.Process - $p.StartInfo = $pinfo - $p.Start() | Out-Null - $p.WaitForExit() - $stderr = $p.StandardError.ReadToEnd() - $env:JVM_ARCH = "32-bit" - } - - $sa = $stderr.Split("""") - $env:JVM_VERSION = $sa[1] - - if ($stderr.Contains("OpenJDK")) - { - $env:JVM_VENDOR = "OpenJDK" - } - elseif ($stderr.Contains("Java(TM)")) - { - $env:JVM_VENDOR = "Oracle" - } - else - { - $JVM_VENDOR = "other" - } - - $pa = $sa[1].Split("_") - $subVersion = $pa[1] - # Deal with -b (build) versions - if ($subVersion -contains '-') - { - $patchAndBuild = $subVersion.Split("-") - $subVersion = $patchAndBuild[0] - } - $env:JVM_PATCH_VERSION = $subVersion -} - -#----------------------------------------------------------------------------- -Function SetCassandraEnvironment -{ - if (Test-Path Env:\JAVA_HOME) - { - $env:JAVA_BIN = "$env:JAVA_HOME\bin\java.exe" - } - elseif (Get-Command "java.exe") - { - $env:JAVA_BIN = "java.exe" - } - else - { - echo "ERROR! No JAVA_HOME set and could not find java.exe in the path." - exit - } - SetCassandraHome - $env:CASSANDRA_CONF = "$env:CASSANDRA_HOME\conf" - $env:CASSANDRA_PARAMS="-Dcassandra -Dlogback.configurationFile=logback.xml" - - $logdir = "$env:CASSANDRA_HOME\logs" - $storagedir = "$env:CASSANDRA_HOME\data" - $env:CASSANDRA_PARAMS = $env:CASSANDRA_PARAMS + " -Dcassandra.logdir=""$logdir"" -Dcassandra.storagedir=""$storagedir""" - - SetCassandraMain - BuildClassPath - - # Override these to set the amount of memory to allocate to the JVM at - # start-up. For production use you may wish to adjust this for your - # environment. MAX_HEAP_SIZE is the total amount of memory dedicated - # to the Java heap. HEAP_NEWSIZE refers to the size of the young - # generation. Both MAX_HEAP_SIZE and HEAP_NEWSIZE should be either set - # or not (if you set one, set the other). - # - # The main trade-off for the young generation is that the larger it - # is, the longer GC pause times will be. The shorter it is, the more - # expensive GC will be (usually). - # - # The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent - # times. If in doubt, and if you do not particularly want to tweak, go - # 100 MB per physical CPU core. - - #GC log path has to be defined here since it needs to find CASSANDRA_HOME - $env:JVM_OPTS="$env:JVM_OPTS -Xloggc:""$env:CASSANDRA_HOME/logs/gc.log""" - - # Read user-defined JVM options from jvm.options file - $content = Get-Content "$env:CASSANDRA_CONF\jvm.options" - for ($i = 0; $i -lt $content.Count; $i++) - { - $line = $content[$i] - if ($line.StartsWith("-")) - { - $env:JVM_OPTS = "$env:JVM_OPTS $line" - } - } - - $defined_xmn = $env:JVM_OPTS -like '*Xmn*' - $defined_xmx = $env:JVM_OPTS -like '*Xmx*' - $defined_xms = $env:JVM_OPTS -like '*Xms*' - $using_cms = $env:JVM_OPTS -like '*UseConcMarkSweepGC*' - - #$env:MAX_HEAP_SIZE="4096M" - #$env:HEAP_NEWSIZE="800M" - CalculateHeapSizes - - ParseJVMInfo - - # We only set -Xms and -Xmx if they were not defined on jvm.options file - # If defined, both Xmx and Xms should be defined together. - if (($defined_xmx -eq $false) -and ($defined_xms -eq $false)) - { - $env:JVM_OPTS="$env:JVM_OPTS -Xms$env:MAX_HEAP_SIZE" - $env:JVM_OPTS="$env:JVM_OPTS -Xmx$env:MAX_HEAP_SIZE" - } - elseif (($defined_xmx -eq $false) -or ($defined_xms -eq $false)) - { - echo "Please set or unset -Xmx and -Xms flags in pairs on jvm.options file." - exit - } - - # We only set -Xmn flag if it was not defined in jvm.options file - # and if the CMS GC is being used - # If defined, both Xmn and Xmx should be defined together. - if (($defined_xmn -eq $true) -and ($defined_xmx -eq $false)) - { - echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm.options file." - exit - } - elseif (($defined_xmn -eq $false) -and ($using_cms -eq $true)) - { - $env:JVM_OPTS="$env:JVM_OPTS -Xmn$env:HEAP_NEWSIZE" - } - - if (($env:JVM_ARCH -eq "64-Bit") -and ($using_cms -eq $true)) - { - $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseCondCardMark" - } - - # Add sigar env - see Cassandra-7838 - $env:JVM_OPTS = "$env:JVM_OPTS -Djava.library.path=""$env:CASSANDRA_HOME\lib\sigar-bin""" - - # Confirm we're on high performance power plan, warn if not - # Change to $true to suppress this warning - $suppressPowerWarning = $false - if (!$suppressPowerWarning) - { - $currentProfile = powercfg /GETACTIVESCHEME - if (!$currentProfile.Contains("High performance")) - { - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - echo "" - echo " WARNING! Detected a power profile other than High Performance." - echo " Performance of this node will suffer." - echo " Modify conf\cassandra.env.ps1 to suppress this warning." - echo "" - echo "*---------------------------------------------------------------------*" - echo "*---------------------------------------------------------------------*" - } - } - +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: All param tuning can be done in the SetCassandraEnvironment Function below + +#----------------------------------------------------------------------------- +Function SetCassandraHome() +{ + if (! $env:CASSANDRA_HOME) + { + $cwd = [System.IO.Directory]::GetCurrentDirectory() + $cwd = Split-Path $cwd -parent + $env:CASSANDRA_HOME = $cwd -replace "\\", "/" + } +} + +#----------------------------------------------------------------------------- +Function SetCassandraMain() +{ + if (! $env:CASSANDRA_MAIN) + { + $env:CASSANDRA_MAIN="org.apache.cassandra.service.CassandraDaemon" + } +} + +#----------------------------------------------------------------------------- +Function BuildClassPath +{ + $cp = """$env:CASSANDRA_HOME\conf""" + foreach ($file in Get-ChildItem "$env:CASSANDRA_HOME\lib\*.jar") + { + $file = $file -replace "\\", "/" + $cp = $cp + ";" + """$file""" + } + + # Add build/classes/main so it works in development + $cp = $cp + ";" + """$env:CASSANDRA_HOME\build\classes\main"";""$env:CASSANDRA_HOME\build\classes\thrift""" + $env:CLASSPATH=$cp +} + +#----------------------------------------------------------------------------- +Function CalculateHeapSizes +{ + # Check if swapping is enabled on the host and warn if so - reference CASSANDRA-7316 + + $osInfo = Get-WmiObject -class "Win32_computersystem" + $autoPage = $osInfo.AutomaticManagedPageFile + + if ($autoPage) + { + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + echo "" + echo " WARNING! Automatic page file configuration detected." + echo " It is recommended that you disable swap when running Cassandra" + echo " for performance and stability reasons." + echo "" + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + } + else + { + $pageFileInfo = Get-WmiObject -class "Win32_PageFileSetting" -EnableAllPrivileges + $pageFileCount = $PageFileInfo.Count + if ($pageFileInfo) + { + $files = @() + $sizes = @() + $hasSizes = $FALSE + + # PageFileCount isn't populated and obj comes back as single if there's only 1 + if ([string]::IsNullOrEmpty($PageFileCount)) + { + $PageFileCount = 1 + $files += $PageFileInfo.Name + if ($PageFileInfo.MaximumSize -ne 0) + { + $hasSizes = $TRUE + $sizes += $PageFileInfo.MaximumSize + } + } + else + { + for ($i = 0; $i -le $PageFileCount; $i++) + { + $files += $PageFileInfo[$i].Name + if ($PageFileInfo[$i].MaximumSize -ne 0) + { + $hasSizes = $TRUE + $sizes += $PageFileInfo[$i].MaximumSize + } + } + } + + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + echo "" + echo " WARNING! $PageFileCount swap file(s) detected" + for ($i = 0; $i -lt $PageFileCount; $i++) + { + $toPrint = " Name: " + $files[$i] + if ($hasSizes) + { + $toPrint = $toPrint + " Size: " + $sizes[$i] + $toPrint = $toPrint -replace [Environment]::NewLine, "" + } + echo $toPrint + } + echo " It is recommended that you disable swap when running Cassandra" + echo " for performance and stability reasons." + echo "" + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + } + } + + # Validate that we need to run this function and that our config is good + if ($env:MAX_HEAP_SIZE -and $env:HEAP_NEWSIZE) + { + return + } + + if ((($env:MAX_HEAP_SIZE -and !$env:HEAP_NEWSIZE) -or (!$env:MAX_HEAP_SIZE -and $env:HEAP_NEWSIZE)) -and ($using_cms -eq $true)) + { + echo "Please set or unset MAX_HEAP_SIZE and HEAP_NEWSIZE in pairs. Aborting startup." + exit 1 + } + + $memObject = Get-WMIObject -class win32_physicalmemory + if ($memObject -eq $null) + { + echo "WARNING! Could not determine system memory. Defaulting to 2G heap, 512M newgen. Manually override in conf\jvm.options for different heap values." + $env:MAX_HEAP_SIZE = "2048M" + $env:HEAP_NEWSIZE = "512M" + return + } + + $memory = ($memObject | Measure-Object Capacity -Sum).sum + $memoryMB = [Math]::Truncate($memory / (1024*1024)) + + $cpu = gwmi Win32_ComputerSystem | Select-Object NumberOfLogicalProcessors + $systemCores = $cpu.NumberOfLogicalProcessors + + # set max heap size based on the following + # max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB)) + # calculate 1/2 ram and cap to 1024MB + # calculate 1/4 ram and cap to 8192MB + # pick the max + $halfMem = [Math]::Truncate($memoryMB / 2) + $quarterMem = [Math]::Truncate($halfMem / 2) + + if ($halfMem -gt 1024) + { + $halfMem = 1024 + } + if ($quarterMem -gt 8192) + { + $quarterMem = 8192 + } + + $maxHeapMB = "" + if ($halfMem -gt $quarterMem) + { + $maxHeapMB = $halfMem + } + else + { + $maxHeapMB = $quarterMem + } + $env:MAX_HEAP_SIZE = [System.Convert]::ToString($maxHeapMB) + "M" + + # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 + $maxYGPerCore = 100 + $maxYGTotal = $maxYGPerCore * $systemCores + $desiredYG = [Math]::Truncate($maxHeapMB / 4) + + if ($desiredYG -gt $maxYGTotal) + { + $env:HEAP_NEWSIZE = [System.Convert]::ToString($maxYGTotal) + "M" + } + else + { + $env:HEAP_NEWSIZE = [System.Convert]::ToString($desiredYG) + "M" + } +} + +#----------------------------------------------------------------------------- +Function ParseJVMInfo +{ + # grab info about the JVM + $pinfo = New-Object System.Diagnostics.ProcessStartInfo + $pinfo.FileName = "$env:JAVA_BIN" + $pinfo.RedirectStandardError = $true + $pinfo.RedirectStandardOutput = $true + $pinfo.UseShellExecute = $false + $pinfo.Arguments = "-d64 -version" + $p = New-Object System.Diagnostics.Process + $p.StartInfo = $pinfo + $p.Start() | Out-Null + $p.WaitForExit() + $stderr = $p.StandardError.ReadToEnd() + + $env:JVM_ARCH = "64-bit" + + if ($stderr.Contains("Error")) + { + # 32-bit JVM. re-run w/out -d64 + echo "Failed 64-bit check. Re-running to get version from 32-bit" + $pinfo.Arguments = "-version" + $p = New-Object System.Diagnostics.Process + $p.StartInfo = $pinfo + $p.Start() | Out-Null + $p.WaitForExit() + $stderr = $p.StandardError.ReadToEnd() + $env:JVM_ARCH = "32-bit" + } + + $sa = $stderr.Split("""") + $env:JVM_VERSION = $sa[1] + + if ($stderr.Contains("OpenJDK")) + { + $env:JVM_VENDOR = "OpenJDK" + } + elseif ($stderr.Contains("Java(TM)")) + { + $env:JVM_VENDOR = "Oracle" + } + else + { + $JVM_VENDOR = "other" + } + + $pa = $sa[1].Split("_") + $subVersion = $pa[1] + # Deal with -b (build) versions + if ($subVersion -contains '-') + { + $patchAndBuild = $subVersion.Split("-") + $subVersion = $patchAndBuild[0] + } + $env:JVM_PATCH_VERSION = $subVersion +} + +#----------------------------------------------------------------------------- +Function SetCassandraEnvironment +{ + if (Test-Path Env:\JAVA_HOME) + { + $env:JAVA_BIN = "$env:JAVA_HOME\bin\java.exe" + } + elseif (Get-Command "java.exe") + { + $env:JAVA_BIN = "java.exe" + } + else + { + echo "ERROR! No JAVA_HOME set and could not find java.exe in the path." + exit + } + SetCassandraHome + $env:CASSANDRA_CONF = "$env:CASSANDRA_HOME\conf" + $env:CASSANDRA_PARAMS="-Dcassandra -Dlogback.configurationFile=logback.xml" + + $logdir = "$env:CASSANDRA_HOME\logs" + $storagedir = "$env:CASSANDRA_HOME\data" + $env:CASSANDRA_PARAMS = $env:CASSANDRA_PARAMS + " -Dcassandra.logdir=""$logdir"" -Dcassandra.storagedir=""$storagedir""" + + SetCassandraMain + BuildClassPath + + # Override these to set the amount of memory to allocate to the JVM at + # start-up. For production use you may wish to adjust this for your + # environment. MAX_HEAP_SIZE is the total amount of memory dedicated + # to the Java heap. HEAP_NEWSIZE refers to the size of the young + # generation. Both MAX_HEAP_SIZE and HEAP_NEWSIZE should be either set + # or not (if you set one, set the other). + # + # The main trade-off for the young generation is that the larger it + # is, the longer GC pause times will be. The shorter it is, the more + # expensive GC will be (usually). + # + # The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent + # times. If in doubt, and if you do not particularly want to tweak, go + # 100 MB per physical CPU core. + + #GC log path has to be defined here since it needs to find CASSANDRA_HOME + $env:JVM_OPTS="$env:JVM_OPTS -Xloggc:""$env:CASSANDRA_HOME/logs/gc.log""" + + # Read user-defined JVM options from jvm.options file + $content = Get-Content "$env:CASSANDRA_CONF\jvm.options" + for ($i = 0; $i -lt $content.Count; $i++) + { + $line = $content[$i] + if ($line.StartsWith("-")) + { + $env:JVM_OPTS = "$env:JVM_OPTS $line" + } + } + + $defined_xmn = $env:JVM_OPTS -like '*Xmn*' + $defined_xmx = $env:JVM_OPTS -like '*Xmx*' + $defined_xms = $env:JVM_OPTS -like '*Xms*' + $using_cms = $env:JVM_OPTS -like '*UseConcMarkSweepGC*' + + #$env:MAX_HEAP_SIZE="4096M" + #$env:HEAP_NEWSIZE="800M" + CalculateHeapSizes + + ParseJVMInfo + + # We only set -Xms and -Xmx if they were not defined on jvm.options file + # If defined, both Xmx and Xms should be defined together. + if (($defined_xmx -eq $false) -and ($defined_xms -eq $false)) + { + $env:JVM_OPTS="$env:JVM_OPTS -Xms$env:MAX_HEAP_SIZE" + $env:JVM_OPTS="$env:JVM_OPTS -Xmx$env:MAX_HEAP_SIZE" + } + elseif (($defined_xmx -eq $false) -or ($defined_xms -eq $false)) + { + echo "Please set or unset -Xmx and -Xms flags in pairs on jvm.options file." + exit + } + + # We only set -Xmn flag if it was not defined in jvm.options file + # and if the CMS GC is being used + # If defined, both Xmn and Xmx should be defined together. + if (($defined_xmn -eq $true) -and ($defined_xmx -eq $false)) + { + echo "Please set or unset -Xmx and -Xmn flags in pairs on jvm.options file." + exit + } + elseif (($defined_xmn -eq $false) -and ($using_cms -eq $true)) + { + $env:JVM_OPTS="$env:JVM_OPTS -Xmn$env:HEAP_NEWSIZE" + } + + if (($env:JVM_ARCH -eq "64-Bit") -and ($using_cms -eq $true)) + { + $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseCondCardMark" + } + + # Add sigar env - see Cassandra-7838 + $env:JVM_OPTS = "$env:JVM_OPTS -Djava.library.path=""$env:CASSANDRA_HOME\lib\sigar-bin""" + + # Confirm we're on high performance power plan, warn if not + # Change to $true to suppress this warning + $suppressPowerWarning = $false + if (!$suppressPowerWarning) + { + $currentProfile = powercfg /GETACTIVESCHEME + if (!$currentProfile.Contains("High performance")) + { + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + echo "" + echo " WARNING! Detected a power profile other than High Performance." + echo " Performance of this node will suffer." + echo " Modify conf\cassandra.env.ps1 to suppress this warning." + echo "" + echo "*---------------------------------------------------------------------*" + echo "*---------------------------------------------------------------------*" + } + } + # provides hints to the JIT compiler $env:JVM_OPTS = "$env:JVM_OPTS -XX:CompileCommandFile=$env:CASSANDRA_CONF\hotspot_compiler" - # add the jamm javaagent - if (($env:JVM_VENDOR -ne "OpenJDK") -or ($env:JVM_VERSION.CompareTo("1.6.0") -eq 1) -or - (($env:JVM_VERSION -eq "1.6.0") -and ($env:JVM_PATCH_VERSION.CompareTo("22") -eq 1))) - { - $env:JVM_OPTS = "$env:JVM_OPTS -javaagent:""$env:CASSANDRA_HOME\lib\jamm-0.3.0.jar""" - } - + # add the jamm javaagent + if (($env:JVM_VENDOR -ne "OpenJDK") -or ($env:JVM_VERSION.CompareTo("1.6.0") -eq 1) -or + (($env:JVM_VERSION -eq "1.6.0") -and ($env:JVM_PATCH_VERSION.CompareTo("22") -eq 1))) + { + $env:JVM_OPTS = "$env:JVM_OPTS -javaagent:""$env:CASSANDRA_HOME\lib\jamm-0.3.0.jar""" + } + # set jvm HeapDumpPath with CASSANDRA_HEAPDUMP_DIR if ($env:CASSANDRA_HEAPDUMP_DIR) { @@ -393,88 +393,98 @@ Function SetCassandraEnvironment $env:JVM_OPTS="$env:JVM_OPTS -XX:HeapDumpPath=$env:CASSANDRA_HEAPDUMP_DIR\cassandra-$unixTimestamp-pid$pid.hprof" } - if ($env:JVM_VERSION.CompareTo("1.8.0") -eq -1 -or [convert]::ToInt32($env:JVM_PATCH_VERSION) -lt 40) - { - echo "Cassandra 3.0 and later require Java 8u40 or later." - exit - } - - # enable assertions. disabling this in production will give a modest - # performance benefit (around 5%). - $env:JVM_OPTS = "$env:JVM_OPTS -ea" - - # Specifies the default port over which Cassandra will be available for - # JMX connections. - $JMX_PORT="7199" - - # store in env to check if it's avail in verification - $env:JMX_PORT=$JMX_PORT - - # enable thread priorities, primarily so we can give periodic tasks - # a lower priority to avoid interfering with client workload - $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseThreadPriorities" - # allows lowering thread priority without being root on linux - probably - # not necessary on Windows but doesn't harm anything. - # see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar - $env:JVM_OPTS="$env:JVM_OPTS -XX:ThreadPriorityPolicy=42" - - $env:JVM_OPTS="$env:JVM_OPTS -XX:+HeapDumpOnOutOfMemoryError" - - # Per-thread stack size. - $env:JVM_OPTS="$env:JVM_OPTS -Xss256k" - - # Larger interned string table, for gossip's benefit (CASSANDRA-6410) - $env:JVM_OPTS="$env:JVM_OPTS -XX:StringTableSize=1000003" - - # Make sure all memory is faulted and zeroed on startup. - # This helps prevent soft faults in containers and makes - # transparent hugepage allocation more effective. - #$env:JVM_OPTS="$env:JVM_OPTS -XX:+AlwaysPreTouch" - - # Biased locking does not benefit Cassandra. - $env:JVM_OPTS="$env:JVM_OPTS -XX:-UseBiasedLocking" - - # Enable thread-local allocation blocks and allow the JVM to automatically - # resize them at runtime. - $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseTLAB -XX:+ResizeTLAB" - - # http://www.evanjones.ca/jvm-mmap-pause.html - $env:JVM_OPTS="$env:JVM_OPTS -XX:+PerfDisableSharedMem" - - # Configure the following for JEMallocAllocator and if jemalloc is not available in the system - # library path. - # set LD_LIBRARY_PATH=/lib/ - # $env:JVM_OPTS="$env:JVM_OPTS -Djava.library.path=/lib/" - - # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 - # $env:JVM_OPTS="$env:JVM_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414" - - # Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See - # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version: - # comment out this entry to enable IPv6 support). - $env:JVM_OPTS="$env:JVM_OPTS -Djava.net.preferIPv4Stack=true" - - # jmx: metrics and administration interface - # - # add this if you're having trouble connecting: - # $env:JVM_OPTS="$env:JVM_OPTS -Djava.rmi.server.hostname=" - # - # see - # https://blogs.oracle.com/jmxetc/entry/troubleshooting_connection_problems_in_jconsole - # for more on configuring JMX through firewalls, etc. (Short version: - # get it working with no firewall first.) - # - # Due to potential security exploits, Cassandra ships with JMX accessible - # *only* from localhost. To enable remote JMX connections, uncomment lines below - # with authentication and ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity - # - #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT" - #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.ssl=false" - #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true" - #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.password.file=C:/jmxremote.password" - $env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT -XX:+DisableExplicitGC" - - $env:JVM_OPTS="$env:JVM_OPTS $env:JVM_EXTRA_OPTS" - - #$env:JVM_OPTS="$env:JVM_OPTS -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" -} + if ($env:JVM_VERSION.CompareTo("1.8.0") -eq -1 -or [convert]::ToInt32($env:JVM_PATCH_VERSION) -lt 40) + { + echo "Cassandra 3.0 and later require Java 8u40 or later." + exit + } + + # enable assertions. disabling this in production will give a modest + # performance benefit (around 5%). + $env:JVM_OPTS = "$env:JVM_OPTS -ea" + + # Specifies the default port over which Cassandra will be available for + # JMX connections. + $JMX_PORT="7199" + + # store in env to check if it's avail in verification + $env:JMX_PORT=$JMX_PORT + + # enable thread priorities, primarily so we can give periodic tasks + # a lower priority to avoid interfering with client workload + $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseThreadPriorities" + # allows lowering thread priority without being root on linux - probably + # not necessary on Windows but doesn't harm anything. + # see http://tech.stolsvik.com/2010/01/linux-java-thread-priorities-workar + $env:JVM_OPTS="$env:JVM_OPTS -XX:ThreadPriorityPolicy=42" + + $env:JVM_OPTS="$env:JVM_OPTS -XX:+HeapDumpOnOutOfMemoryError" + + # stop the jvm on OutOfMemoryError as it can result in some data corruption + # uncomment the preferred option + # ExitOnOutOfMemoryError and CrashOnOutOfMemoryError require a JRE greater or equals to 1.7 update 101 or 1.8 update 92 + # $env:JVM_OPTS="$env:JVM_OPTS -XX:+ExitOnOutOfMemoryError" + # $env:JVM_OPTS="$env:JVM_OPTS -XX:+CrashOnOutOfMemoryError" + $env:JVM_OPTS="$env:JVM_OPTS -XX:OnOutOfMemoryError=""taskkill /F /PID %p""" + + # print an heap histogram on OutOfMemoryError + # $env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.printHeapHistogramOnOutOfMemoryError=true" + + # Per-thread stack size. + $env:JVM_OPTS="$env:JVM_OPTS -Xss256k" + + # Larger interned string table, for gossip's benefit (CASSANDRA-6410) + $env:JVM_OPTS="$env:JVM_OPTS -XX:StringTableSize=1000003" + + # Make sure all memory is faulted and zeroed on startup. + # This helps prevent soft faults in containers and makes + # transparent hugepage allocation more effective. + #$env:JVM_OPTS="$env:JVM_OPTS -XX:+AlwaysPreTouch" + + # Biased locking does not benefit Cassandra. + $env:JVM_OPTS="$env:JVM_OPTS -XX:-UseBiasedLocking" + + # Enable thread-local allocation blocks and allow the JVM to automatically + # resize them at runtime. + $env:JVM_OPTS="$env:JVM_OPTS -XX:+UseTLAB -XX:+ResizeTLAB" + + # http://www.evanjones.ca/jvm-mmap-pause.html + $env:JVM_OPTS="$env:JVM_OPTS -XX:+PerfDisableSharedMem" + + # Configure the following for JEMallocAllocator and if jemalloc is not available in the system + # library path. + # set LD_LIBRARY_PATH=/lib/ + # $env:JVM_OPTS="$env:JVM_OPTS -Djava.library.path=/lib/" + + # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 + # $env:JVM_OPTS="$env:JVM_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414" + + # Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See + # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version: + # comment out this entry to enable IPv6 support). + $env:JVM_OPTS="$env:JVM_OPTS -Djava.net.preferIPv4Stack=true" + + # jmx: metrics and administration interface + # + # add this if you're having trouble connecting: + # $env:JVM_OPTS="$env:JVM_OPTS -Djava.rmi.server.hostname=" + # + # see + # https://blogs.oracle.com/jmxetc/entry/troubleshooting_connection_problems_in_jconsole + # for more on configuring JMX through firewalls, etc. (Short version: + # get it working with no firewall first.) + # + # Due to potential security exploits, Cassandra ships with JMX accessible + # *only* from localhost. To enable remote JMX connections, uncomment lines below + # with authentication and ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity + # + #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT" + #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.ssl=false" + #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.authenticate=true" + #$env:JVM_OPTS="$env:JVM_OPTS -Dcom.sun.management.jmxremote.password.file=C:/jmxremote.password" + $env:JVM_OPTS="$env:JVM_OPTS -Dcassandra.jmx.local.port=$JMX_PORT -XX:+DisableExplicitGC" + + $env:JVM_OPTS="$env:JVM_OPTS $env:JVM_EXTRA_OPTS" + + #$env:JVM_OPTS="$env:JVM_OPTS -XX:+UnlockCommercialFeatures -XX:+FlightRecorder" +} diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh index 44fe110b18..ffc773fea2 100644 --- a/conf/cassandra-env.sh +++ b/conf/cassandra-env.sh @@ -247,6 +247,18 @@ if [ "x$CASSANDRA_HEAPDUMP_DIR" != "x" ]; then JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=$CASSANDRA_HEAPDUMP_DIR/cassandra-`date +%s`-pid$$.hprof" fi +# stop the jvm on OutOfMemoryError as it can result in some data corruption +# uncomment the preferred option +# ExitOnOutOfMemoryError and CrashOnOutOfMemoryError require a JRE greater or equals to 1.7 update 101 or 1.8 update 92 +# For OnOutOfMemoryError we cannot use the JVM_OPTS variables because bash commands split words +# on white spaces without taking quotes into account +# JVM_OPTS="$JVM_OPTS -XX:+ExitOnOutOfMemoryError" +# JVM_OPTS="$JVM_OPTS -XX:+CrashOnOutOfMemoryError" +JVM_ON_OUT_OF_MEMORY_ERROR_OPT="-XX:OnOutOfMemoryError=kill -9 %p" + +# print an heap histogram on OutOfMemoryError +# JVM_OPTS="$JVM_OPTS -Dcassandra.printHeapHistogramOnOutOfMemoryError=true" + # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 # JVM_OPTS="$JVM_OPTS -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1414" diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java index e9f99ee776..03b362ac2e 100644 --- a/src/java/org/apache/cassandra/service/StartupChecks.java +++ b/src/java/org/apache/cassandra/service/StartupChecks.java @@ -20,6 +20,8 @@ package org.apache.cassandra.service; import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.*; @@ -189,6 +191,78 @@ public class StartupChecks { logger.warn("Non-Oracle JVM detected. Some features, such as immediate unmap of compacted SSTables, may not work as intended"); } + else + { + checkOutOfMemoryHandling(); + } + } + + /** + * Checks that the JVM is configured to handle OutOfMemoryError + */ + private void checkOutOfMemoryHandling() + { + int version = getJavaVersion(); + int update = getUpdate(); + // The ExitOnOutOfMemory and CrashOnOutOfMemory are supported since the version 7u101 and 8u92 + boolean jreSupportExitOnOutOfMemory = version > 8 + || (version == 7 && update >= 101) + || (version == 8 && update >= 92); + if (jreSupportExitOnOutOfMemory) + { + if (!jvmOptionsContainsOneOf("-XX:OnOutOfMemoryError=", "-XX:+ExitOnOutOfMemoryError", "-XX:+CrashOnOutOfMemoryError")) + logger.warn("The JVM is not configured to stop on OutOfMemoryError which can cause data corruption." + + " Use one of the following JVM options to configure the behavior on OutOfMemoryError: " + + " -XX:+ExitOnOutOfMemoryError, -XX:+CrashOnOutOfMemoryError, or -XX:OnOutOfMemoryError=\";\""); + } + else + { + if (!jvmOptionsContainsOneOf("-XX:OnOutOfMemoryError=")) + logger.warn("The JVM is not configured to stop on OutOfMemoryError which can cause data corruption." + + " Either upgrade your JRE to a version greater or equal to 8u92 and use -XX:+ExitOnOutOfMemoryError/-XX:+CrashOnOutOfMemoryError" + + " or use -XX:OnOutOfMemoryError=\";\" on your current JRE."); + } + } + + /** + * Returns the java version number for an Oracle JVM. + * @return the java version number + */ + private int getJavaVersion() + { + String jreVersion = System.getProperty("java.version"); + String version = jreVersion.startsWith("1.") ? jreVersion.substring(2, 3) // Pre 9 version + : jreVersion.substring(0, jreVersion.indexOf('.')); + return Integer.parseInt(version); + } + + /** + * Return the update number for an Oracle JVM. + * @return the update number + */ + private int getUpdate() + { + String jreVersion = System.getProperty("java.version"); + int updateSeparatorIndex = jreVersion.indexOf('_'); + return Integer.parseInt(jreVersion.substring(updateSeparatorIndex + 1)); + } + + /** + * Checks if one of the specified options is being used. + * @param optionNames The name of the options to check + * @return {@code true} if one of the specified options is being used, {@code false} otherwise. + */ + private boolean jvmOptionsContainsOneOf(String... optionNames) + { + RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); + List inputArguments = runtimeMxBean.getInputArguments(); + for (String argument : inputArguments) + { + for (String optionName : optionNames) + if (argument.startsWith(optionName)) + return true; + } + return false; } }; diff --git a/src/java/org/apache/cassandra/utils/HeapUtils.java b/src/java/org/apache/cassandra/utils/HeapUtils.java index 67d7d79b65..4c84f9b3e3 100644 --- a/src/java/org/apache/cassandra/utils/HeapUtils.java +++ b/src/java/org/apache/cassandra/utils/HeapUtils.java @@ -19,11 +19,6 @@ package org.apache.cassandra.utils; import java.io.*; import java.lang.management.ManagementFactory; -import java.lang.management.RuntimeMXBean; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.text.StrBuilder; @@ -32,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Utility to generate heap dumps. + * Utility to log heap histogram. * */ public final class HeapUtils @@ -40,65 +35,43 @@ public final class HeapUtils private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class); /** - * Generates a HEAP dump in the directory specified by the HeapDumpPath JVM option - * or in the CASSANDRA_HOME directory. + * Generates a HEAP histogram in the log file. */ - public static void generateHeapDump() + public static void logHeapHistogram() { - Long processId = getProcessId(); - if (processId == null) + try { - logger.error("The process ID could not be retrieved. Skipping heap dump generation."); - return; - } + logger.info("Trying to log the heap histogram using jcmd"); - String heapDumpPath = getHeapDumpPathOption(); - if (heapDumpPath == null) - { - String cassandraHome = System.getenv("CASSANDRA_HOME"); - if (cassandraHome == null) + Long processId = getProcessId(); + if (processId == null) { + logger.error("The process ID could not be retrieved. Skipping heap histogram generation."); return; } - heapDumpPath = cassandraHome; - } + String jcmdPath = getJcmdPath(); - Path dumpPath = FileSystems.getDefault().getPath(heapDumpPath); - if (Files.isDirectory(dumpPath)) - { - dumpPath = dumpPath.resolve("java_pid" + processId + ".hprof"); - } + // The jcmd file could not be found. In this case let's default to jcmd in the hope that it is in the path. + String jcmdCommand = jcmdPath == null ? "jcmd" : jcmdPath; - String jmapPath = getJmapPath(); + String[] histoCommands = new String[] {jcmdCommand, + processId.toString(), + "GC.class_histogram"}; - // The jmap file could not be found. In this case let's default to jmap in the hope that it is in the path. - String jmapCommand = jmapPath == null ? "jmap" : jmapPath; - - String[] dumpCommands = new String[] {jmapCommand, - "-dump:format=b,file=" + dumpPath, - processId.toString()}; - - // Lets also log the Heap histogram - String[] histoCommands = new String[] {jmapCommand, - "-histo", - processId.toString()}; - try - { - logProcessOutput(Runtime.getRuntime().exec(dumpCommands)); logProcessOutput(Runtime.getRuntime().exec(histoCommands)); } - catch (IOException e) + catch (Throwable e) { - logger.error("The heap dump could not be generated due to the following error: ", e); + logger.error("The heap histogram could not be generated due to the following error: ", e); } } /** - * Retrieve the path to the JMAP executable. - * @return the path to the JMAP executable or null if it cannot be found. + * Retrieve the path to the JCMD executable. + * @return the path to the JCMD executable or null if it cannot be found. */ - private static String getJmapPath() + private static String getJcmdPath() { // Searching in the JAVA_HOME is safer than searching into System.getProperty("java.home") as the Oracle // JVM might use the JRE which do not contains jmap. @@ -111,7 +84,7 @@ public final class HeapUtils { public boolean accept(File dir, String name) { - return name.startsWith("jmap"); + return name.startsWith("jcmd"); } }); return ArrayUtils.isEmpty(files) ? null : files[0].getPath(); @@ -136,32 +109,6 @@ public final class HeapUtils logger.info(builder.toString()); } - /** - * Retrieves the value of the HeapDumpPath JVM option. - * @return the value of the HeapDumpPath JVM option or null if the value has not been - * specified. - */ - private static String getHeapDumpPathOption() - { - RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); - List inputArguments = runtimeMxBean.getInputArguments(); - String heapDumpPathOption = null; - for (String argument : inputArguments) - { - if (argument.startsWith("-XX:HeapDumpPath=")) - { - heapDumpPathOption = argument; - // We do not break in case the option has been specified several times. - // In general it seems that JVMs use the right-most argument as the winner. - } - } - - if (heapDumpPathOption == null) - return null; - - return heapDumpPathOption.substring(17, heapDumpPathOption.length()); - } - /** * Retrieves the process ID or null if the process ID cannot be retrieved. * @return the process ID or null if the process ID cannot be retrieved. diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java index c06a97bc7d..89ef129793 100644 --- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java +++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.annotations.VisibleForTesting; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +42,8 @@ public final class JVMStabilityInspector private static final Logger logger = LoggerFactory.getLogger(JVMStabilityInspector.class); private static Killer killer = new Killer(); + private static Object lock = new Object(); + private static boolean printingHeapHistogram; private JVMStabilityInspector() {} @@ -55,8 +58,25 @@ public final class JVMStabilityInspector boolean isUnstable = false; if (t instanceof OutOfMemoryError) { - isUnstable = true; - HeapUtils.generateHeapDump(); + if (Boolean.getBoolean("cassandra.printHeapHistogramOnOutOfMemoryError")) + { + // We want to avoid printing multiple time the heap histogram if multiple OOM errors happen in a short + // time span. + synchronized(lock) + { + if (printingHeapHistogram) + return; + printingHeapHistogram = true; + } + HeapUtils.logHeapHistogram(); + } + + logger.error("OutOfMemory error letting the JVM handle the error:", t); + + StorageService.instance.removeShutdownHook(); + // We let the JVM handle the error. The startup checks should have warned the user if it did not configure + // the JVM behavior in case of OOM (CASSANDRA-13006). + throw (OutOfMemoryError) t; } if (DatabaseDescriptor.getDiskFailurePolicy() == Config.DiskFailurePolicy.die) diff --git a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java index eb8dbf96dd..00447dad31 100644 --- a/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java +++ b/test/unit/org/apache/cassandra/utils/JVMStabilityInspectorTest.java @@ -27,8 +27,10 @@ import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.FSReadError; +import static java.util.Arrays.asList; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class JVMStabilityInspectorTest { @@ -46,10 +48,6 @@ public class JVMStabilityInspectorTest JVMStabilityInspector.inspectThrowable(new IOException()); assertFalse(killerForTests.wasKilled()); - killerForTests.reset(); - JVMStabilityInspector.inspectThrowable(new OutOfMemoryError()); - assertTrue(killerForTests.wasKilled()); - DatabaseDescriptor.setDiskFailurePolicy(Config.DiskFailurePolicy.die); killerForTests.reset(); JVMStabilityInspector.inspectThrowable(new FSReadError(new IOException(), "blah")); @@ -63,11 +61,6 @@ public class JVMStabilityInspectorTest killerForTests.reset(); JVMStabilityInspector.inspectThrowable(new Exception(new IOException())); assertFalse(killerForTests.wasKilled()); - - killerForTests.reset(); - JVMStabilityInspector.inspectThrowable(new Exception(new OutOfMemoryError())); - assertTrue(killerForTests.wasKilled()); - } finally { @@ -77,6 +70,23 @@ public class JVMStabilityInspectorTest } } + @Test + public void testOutOfMemoryHandling() + { + for (Throwable oom : asList(new OutOfMemoryError(), new Exception(new OutOfMemoryError()))) + { + try + { + JVMStabilityInspector.inspectThrowable(oom); + fail("The JVMStabilityInspector should delegate the handling of OutOfMemoryErrors to the JVM"); + } + catch (OutOfMemoryError e) + { + assertTrue(true); + } + } + } + @Test public void fileHandleTest() {