natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CASSANDRA-15677 Add shutdown to JMX thread pool to avoid metaspace errors associated with thread leaks. There are still other thread shutdown issues remaining, but this seems to be the most serious one.

This commit is contained in:
bryn 2020-06-08 11:02:42 +01:00 committed by Brandon Williams
parent 58141308ba
commit 6dad660039
3 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/java/org/apache/cassandra/utils/progress/jmx/JMXBroadcastExecutor.java
View File

@ -18,9 +18,11 @@
package org.apache.cassandra.utils.progress.jmx;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.cassandra.concurrent.NamedThreadFactory;
/**
* Holds dedicated executor for JMX event handling. Events will internally queued by ArrayNotificationBuffer,
* synchronized by an exclusive write lock, which makes a shared single thread executor desirable.
@ -30,6 +32,6 @@ public final class JMXBroadcastExecutor
private JMXBroadcastExecutor() { }
public final static Executor executor = Executors.newSingleThreadExecutor();
public final static ExecutorService executor = Executors.newSingleThreadExecutor(new NamedThreadFactory("JMX"));
}

12
test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
View File

@ -667,9 +667,21 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster<I
Thread.setDefaultUncaughtExceptionHandler(previousHandler);
previousHandler = null;
//checkForThreadLeaks();
//withThreadLeakCheck(futures);
}
private void checkForThreadLeaks()
{
//This is an alternate version of the thread leak check that just checks to see if any threads are still alive
// with the context classloader.
Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
threadSet.stream().filter(t->t.getContextClassLoader() instanceof InstanceClassLoader).forEach(t->{
t.setContextClassLoader(null);
throw new RuntimeException("Unterminated thread detected " + t.getName() + " in group " + t.getThreadGroup().getName());
});
}
// We do not want this check to run every time until we fix problems with tread stops
private void withThreadLeakCheck(List<Future<?>> futures)
{

4
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -108,6 +108,7 @@ import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.memory.BufferPool;
import org.apache.cassandra.utils.progress.jmx.JMXBroadcastExecutor;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
@ -566,6 +567,9 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
() -> Stage.shutdownAndWait(1L, MINUTES),
() -> SharedExecutorPool.SHARED.shutdownAndWait(1L, MINUTES)
);
error = parallelRun(error, executor,
() -> shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor))
);
Throwables.maybeFail(error);
}).apply(isolatedExecutor);