diff --git a/build.xml b/build.xml index 3593d4bd70..8d9b8f5516 100644 --- a/build.xml +++ b/build.xml @@ -153,7 +153,7 @@ - + @@ -218,6 +218,7 @@ --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.transport=ALL-UNNAMED --add-exports java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED --add-exports java.sql/java.sql=ALL-UNNAMED @@ -2173,7 +2174,7 @@ - ]]> diff --git a/src/java/org/apache/cassandra/utils/RMIClientSocketFactoryImpl.java b/src/java/org/apache/cassandra/utils/RMIClientSocketFactoryImpl.java index 62ab88fb92..5caa92cce4 100644 --- a/src/java/org/apache/cassandra/utils/RMIClientSocketFactoryImpl.java +++ b/src/java/org/apache/cassandra/utils/RMIClientSocketFactoryImpl.java @@ -23,6 +23,8 @@ import java.io.Serializable; import java.net.InetAddress; import java.net.Socket; import java.rmi.server.RMIClientSocketFactory; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -32,6 +34,7 @@ import java.util.Objects; */ public class RMIClientSocketFactoryImpl implements RMIClientSocketFactory, Serializable { + List sockets = new ArrayList<>(); private final InetAddress localAddress; public RMIClientSocketFactoryImpl(InetAddress localAddress) @@ -42,7 +45,23 @@ public class RMIClientSocketFactoryImpl implements RMIClientSocketFactory, Seria @Override public Socket createSocket(String host, int port) throws IOException { - return new Socket(localAddress, port); + Socket socket = new Socket(localAddress, port); + sockets.add(socket); + return socket; + } + + public void close() throws IOException + { + for (Socket socket: sockets) { + try + { + socket.close(); + } + catch (IOException ignored) + { + // intentionally ignored + } + } } @Override diff --git a/src/java/org/apache/cassandra/utils/ReflectionUtils.java b/src/java/org/apache/cassandra/utils/ReflectionUtils.java index bd605db3bc..07802267f1 100644 --- a/src/java/org/apache/cassandra/utils/ReflectionUtils.java +++ b/src/java/org/apache/cassandra/utils/ReflectionUtils.java @@ -20,6 +20,9 @@ package org.apache.cassandra.utils; import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Predicate; public class ReflectionUtils { public static Field getField(Class clazz, String fieldName) throws NoSuchFieldException @@ -52,4 +55,44 @@ public class ReflectionUtils { throw e; } } + + /** + * Used by the in-jvm dtest framework to remove entries from private map fields that otherwise would prevent + * collection of classloaders (which causes metaspace OOMs) or otherwise interfere with instance restart. + * @param clazz The class which has the map field to clear + * @param instance an instance of the class to clear (pass null for a static member) + * @param mapName the name of the map field to clear + * @param shouldRemove a predicate which determines if the entry in question should be removed + * @param The type of the map key + * @param The type of the map value + */ + public static void clearMapField(Class clazz, Object instance, String mapName, Predicate> shouldRemove) { + try + { + Field mapField = getField(clazz, mapName); + mapField.setAccessible(true); + // noinspection unchecked + Map map = (Map) mapField.get(instance); + // Because multiple instances can be shutting down at once, + // synchronize on the map to avoid ConcurrentModificationException + synchronized (map) + { + // This could be done with a simple `map.entrySet.removeIf()` call + // but for debugging purposes it is much easier to keep it like this. + Iterator> it = map.entrySet().iterator(); + while (it.hasNext()) + { + Map.Entry entry = it.next(); + if (shouldRemove.test(entry)) + { + it.remove(); + } + } + } + } + catch (NoSuchFieldException | IllegalAccessException ex) + { + throw new RuntimeException(String.format("Could not clear map field %s in class %s", mapName, clazz), ex); + } + } } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index 9975aeb9eb..4b4276774b 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -755,17 +755,18 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance initialized = true; } - private void startJmx() + private synchronized void startJmx() { this.isolatedJmx = new IsolatedJmx(this, inInstancelogger); isolatedJmx.startJmx(); } - private void stopJmx() throws IllegalAccessException, NoSuchFieldException, InterruptedException + private synchronized void stopJmx() { if (config.has(JMX)) { isolatedJmx.stopJmx(); + isolatedJmx = null; } } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java index e19e29fdb3..2fd4ec9647 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java @@ -18,38 +18,40 @@ package org.apache.cassandra.distributed.impl; -import java.lang.reflect.Field; +import java.io.IOException; import java.net.InetAddress; -import java.net.MalformedURLException; import java.util.HashMap; -import java.util.Iterator; +import java.util.LinkedList; import java.util.Map; +import java.util.concurrent.TimeUnit; import javax.management.remote.JMXConnector; -import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXConnectorServer; import javax.management.remote.JMXServiceURL; import javax.management.remote.rmi.RMIConnectorServer; import javax.management.remote.rmi.RMIJRMPServerImpl; +import com.google.common.util.concurrent.Uninterruptibles; import org.slf4j.Logger; import org.apache.cassandra.distributed.api.IInstance; import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.shared.JMXUtil; import org.apache.cassandra.utils.JMXServerUtils; import org.apache.cassandra.utils.MBeanWrapper; import org.apache.cassandra.utils.RMIClientSocketFactoryImpl; -import org.apache.cassandra.utils.ReflectionUtils; import sun.rmi.transport.tcp.TCPEndpoint; import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_RMI_DGC_LEASE_VALUE_IN_JVM_DTEST; import static org.apache.cassandra.config.CassandraRelevantProperties.ORG_APACHE_CASSANDRA_DISABLE_MBEAN_REGISTRATION; import static org.apache.cassandra.config.CassandraRelevantProperties.SUN_RMI_TRANSPORT_TCP_THREADKEEPALIVETIME; import static org.apache.cassandra.distributed.api.Feature.JMX; +import static org.apache.cassandra.utils.ReflectionUtils.clearMapField; public class IsolatedJmx { - private static final int RMI_KEEPALIVE_TIME = 1000; + public static final int RMI_KEEPALIVE_TIME = 1000; + public static final String UNKNOWN_JMX_CONNECTION_ERROR = "Could not connect to JMX due to an unknown error"; private JMXConnectorServer jmxConnectorServer; private JMXServerUtils.JmxRegistry registry; @@ -127,7 +129,7 @@ public class IsolatedJmx registry.setRemoteServerStub(jmxRmiServer.toStub()); JMXServerUtils.logJmxServiceUrl(addr, jmxPort); - waitForJmxAvailability(hostname, jmxPort, env); + waitForJmxAvailability(env); } catch (Throwable t) { @@ -135,36 +137,20 @@ public class IsolatedJmx } } - private void waitForJmxAvailability(String hostname, int rmiPort, Map env) throws InterruptedException, MalformedURLException + private void waitForJmxAvailability(Map env) { - String url = String.format("service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi", hostname, rmiPort); - JMXServiceURL serviceURL = new JMXServiceURL(url); - int attempts = 0; - Throwable lastThrown = null; - while (attempts < 20) - { - attempts++; - try (JMXConnector ignored = JMXConnectorFactory.connect(serviceURL, env)) - { - inInstancelogger.info("Connected to JMX server at {} after {} attempt(s)", - url, attempts); - return; - } - catch (MalformedURLException e) - { - throw new RuntimeException(e); - } - catch (Throwable thrown) - { - lastThrown = thrown; - } - inInstancelogger.info("Could not connect to JMX on {} after {} attempts. Will retry.", url, attempts); - Thread.sleep(1000); + try (JMXConnector ignored = JMXUtil.getJmxConnector(config, 20, env)) { + // Do nothing - JMXUtil now retries + } + catch (IOException iex) + { + // If we land here, there's something more than a timeout + inInstancelogger.error(UNKNOWN_JMX_CONNECTION_ERROR, iex); + throw new RuntimeException(UNKNOWN_JMX_CONNECTION_ERROR, iex); } - throw new RuntimeException("Could not start JMX - unreachable after 20 attempts", lastThrown); } - public void stopJmx() throws IllegalAccessException, NoSuchFieldException, InterruptedException + public void stopJmx() { if (!config.has(JMX)) return; @@ -197,6 +183,14 @@ public class IsolatedJmx inInstancelogger.warn("failed to close registry.", e); } try + { + clientSocketFactory.close(); + } + catch (Throwable e) + { + inInstancelogger.warn("failed to close clientSocketFactory.", e); + } + try { serverSocketFactory.close(); } @@ -206,25 +200,18 @@ public class IsolatedJmx } // The TCPEndpoint class holds references to a class in the in-jvm dtest framework // which transitively has a reference to the InstanceClassLoader, so we need to - // make sure to remove the reference to them when the instance is shutting down - clearMapField(TCPEndpoint.class, null, "localEndpoints"); - Thread.sleep(2 * RMI_KEEPALIVE_TIME); // Double the keep-alive time to give Distributed GC some time to clean up + // make sure to remove the reference to them when the instance is shutting down. + // Additionally, we must make sure to only clear endpoints created by this instance + // As clearning the entire map can cause issues with starting and stopping nodes mid-test. + clearMapField(TCPEndpoint.class, null, "localEndpoints", this::endpointCreateByThisInstance); + Uninterruptibles.sleepUninterruptibly(2 * RMI_KEEPALIVE_TIME, TimeUnit.MILLISECONDS); // Double the keep-alive time to give Distributed GC some time to clean up } - private void clearMapField(Class clazz, Object instance, String mapName) - throws IllegalAccessException, NoSuchFieldException { - Field mapField = ReflectionUtils.getField(clazz, mapName); - mapField.setAccessible(true); - Map map = (Map) mapField.get(instance); - // Because multiple instances can be shutting down at once, - // synchronize on the map to avoid ConcurrentModificationException - synchronized (map) - { - for (Iterator> it = map.entrySet().iterator(); it.hasNext(); ) - { - it.next(); - it.remove(); - } - } + private boolean endpointCreateByThisInstance(Map.Entry> entry) + { + return entry.getValue() + .stream() + .anyMatch(ep -> ep.getServerSocketFactory() == this.serverSocketFactory && + ep.getClientSocketFactory() == this.clientSocketFactory); } } diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index dc280f3085..8cfe221b1e 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -425,6 +425,20 @@ public class ClusterUtils ring.stream().allMatch(ClusterUtils::isRingInstanceDetailsHealthy)); } + /** + * Wait for the ring to have the target instance with the provided status. + * + * @param instance instance to check on + * @param expectedInRing to look for + * @param status expected + * @return the ring + */ + public static List awaitRingStatus(IInstance instance, IInstance expectedInRing, String status) + { + return awaitInstanceMatching(instance, expectedInRing, d -> d.status.equals(status), + "Timeout waiting for " + expectedInRing + " to have status " + status); + } + /** * Wait for the ring to have the target instance with the provided state. * @@ -435,12 +449,20 @@ public class ClusterUtils */ public static List awaitRingState(IInstance instance, IInstance expectedInRing, String state) { - return awaitRing(instance, "Timeout waiting for " + expectedInRing + " to have state " + state, - ring -> - ring.stream() - .filter(d -> d.address.equals(getBroadcastAddressHostString(expectedInRing))) - .filter(d -> d.state.equals(state)) - .findAny().isPresent()); + return awaitInstanceMatching(instance, expectedInRing, d -> d.state.equals(state), + "Timeout waiting for " + expectedInRing + " to have state " + state); + } + + private static List awaitInstanceMatching(IInstance instance, + IInstance expectedInRing, + Predicate predicate, + String errorMessage) + { + return awaitRing(instance, + errorMessage, + ring -> ring.stream() + .filter(d -> d.address.equals(getBroadcastAddressHostString(expectedInRing))) + .anyMatch(predicate)); } /** diff --git a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java index 794873feea..7e4581c99c 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java @@ -47,6 +47,7 @@ import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.JMX; import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.test.jmx.JMXGetterCheckTest.testAllValidGetters; import static org.apache.cassandra.utils.FBUtilities.now; import static org.hamcrest.Matchers.startsWith; @@ -145,6 +146,30 @@ public class ResourceLeakTest extends TestBaseImpl } } + static void testJmx(Cluster cluster) + { + try + { + for (IInvokableInstance instance : cluster.get(1, cluster.size())) + { + IInstanceConfig config = instance.config(); + try (JMXConnector jmxc = JMXUtil.getJmxConnector(config, 5)) + { + MBeanServerConnection mbsc = jmxc.getMBeanServerConnection(); + // instances get their default domain set to their IP address, so us it + // to check that we are actually connecting to the correct instance + String defaultDomain = mbsc.getDefaultDomain(); + Assert.assertThat(defaultDomain, startsWith(JMXUtil.getJmxHost(config) + ":" + config.jmxPort())); + } + } + testAllValidGetters(cluster); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + void doTest(int numClusterNodes, Consumer updater) throws Throwable { doTest(numClusterNodes, updater, ignored -> {}); @@ -225,27 +250,7 @@ public class ResourceLeakTest extends TestBaseImpl @Test public void looperJmxTest() throws Throwable { - doTest(1, config -> config.with(JMX), cluster -> { - // NOTE: At some point, the hostname of the broadcastAddress can be resolved - // and then the `getHostString`, which would otherwise return the IP address, - // starts returning `localhost` - use `.getAddress().getHostAddress()` to work around this. - for (IInvokableInstance instance:cluster.get(1, cluster.size())) - { - IInstanceConfig config = instance.config(); - try (JMXConnector jmxc = JMXUtil.getJmxConnector(config)) - { - MBeanServerConnection mbsc = jmxc.getMBeanServerConnection(); - // instances get their default domain set to their IP address, so us it - // to check that we are actually connecting to the correct instance - String defaultDomain = mbsc.getDefaultDomain(); - Assert.assertThat(defaultDomain, startsWith(JMXUtil.getJmxHost(config) + ":" + config.jmxPort())); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - }); + doTest(2, config -> config.with(JMX), ResourceLeakTest::testJmx); if (forceCollection) { System.runFinalization(); @@ -258,7 +263,10 @@ public class ResourceLeakTest extends TestBaseImpl @Test public void looperEverythingTest() throws Throwable { - doTest(1, config -> config.with(Feature.values())); + doTest(2, config -> config.with(Feature.values()), + cluster -> { + testJmx(cluster); + }); if (forceCollection) { System.runFinalization(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java index 83a35e594d..a1576d6e36 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java @@ -18,9 +18,9 @@ package org.apache.cassandra.distributed.test.jmx; -import java.io.IOException; import java.util.HashSet; import java.util.Set; + import javax.management.MBeanServerConnection; import javax.management.remote.JMXConnector; @@ -31,10 +31,16 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.impl.INodeProvisionStrategy; +import org.apache.cassandra.distributed.shared.ClusterUtils; import org.apache.cassandra.distributed.shared.JMXUtil; import org.apache.cassandra.distributed.test.TestBaseImpl; +import static org.apache.cassandra.distributed.test.jmx.JMXGetterCheckTest.testAllValidGetters; +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; public class JMXFeatureTest extends TestBaseImpl @@ -46,59 +52,83 @@ public class JMXFeatureTest extends TestBaseImpl * - Test that when connecting, we get the correct MBeanServer by checking the default domain, which is set to the IP of the instance * - Run the test multiple times to ensure cleanup of the JMX servers is complete so the next test can run successfully using the same host/port. * - * @throws Exception + * @throws Exception it's a test that calls JMX endpoints - lots of different Jmx exceptions are possible */ @Test public void testMultipleNetworkInterfacesProvisioning() throws Exception { - int iterations = 2; // Make sure the JMX infrastructure all cleans up properly by running this multiple times. - Set allInstances = new HashSet<>(); - for (int i = 0; i < iterations; i++) - { - try (Cluster cluster = Cluster.build(2) - .withNodeProvisionStrategy(INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces) - .withConfig(c -> c.with(Feature.values())).start()) - { - Set instancesContacted = new HashSet<>(); - for (IInvokableInstance instance : cluster.get(1, 2)) - { - testInstance(instancesContacted, instance); - } - Assert.assertEquals("Should have connected with both JMX instances.", 2, instancesContacted.size()); - allInstances.addAll(instancesContacted); - } - } - Assert.assertEquals("Each instance from each cluster should have been unique", iterations * 2, allInstances.size()); + testJmxFeatures(INodeProvisionStrategy.Strategy.MultipleNetworkInterfaces); } @Test public void testOneNetworkInterfaceProvisioning() throws Exception { - int iterations = 2; // Make sure the JMX infrastructure all cleans up properly by running this multiple times. + testJmxFeatures(INodeProvisionStrategy.Strategy.OneNetworkInterface); + } + + private void testJmxFeatures(INodeProvisionStrategy.Strategy provisionStrategy) throws Exception + { Set allInstances = new HashSet<>(); + int iterations = 2; // Make sure the JMX infrastructure all cleans up properly by running this multiple times. for (int i = 0; i < iterations; i++) { try (Cluster cluster = Cluster.build(2) - .withNodeProvisionStrategy(INodeProvisionStrategy.Strategy.OneNetworkInterface) + .withNodeProvisionStrategy(provisionStrategy) .withConfig(c -> c.with(Feature.values())).start()) { Set instancesContacted = new HashSet<>(); - for (IInvokableInstance instance : cluster.get(1, 2)) + for (IInvokableInstance instance : cluster) { testInstance(instancesContacted, instance); } Assert.assertEquals("Should have connected with both JMX instances.", 2, instancesContacted.size()); allInstances.addAll(instancesContacted); + // Make sure we actually exercise the mbeans by testing a bunch of getters. + // Without this it's possible for the test to pass as we don't touch any mBeans that we register. + testAllValidGetters(cluster); } } Assert.assertEquals("Each instance from each cluster should have been unique", iterations * 2, allInstances.size()); } - private void testInstance(Set instancesContacted, IInvokableInstance instance) throws IOException + @Test + public void testShutDownAndRestartInstances() throws Exception + { + HashSet instances = new HashSet<>(); + try (Cluster cluster = Cluster.build(2).withConfig(c -> c.with(Feature.values())).start()) + { + IInvokableInstance instanceToStop = cluster.get(1); + IInvokableInstance otherInstance = cluster.get(2); + testInstance(instances, cluster.get(1)); + ClusterUtils.stopUnchecked(instanceToStop); + // NOTE: This would previously fail because we cleared everything from the TCPEndpoint map in IsolatedJmx. + // Now, we only clear the endpoints related to that instance, which prevents this code from + // breaking with a `java.net.BindException: Address already in use (Bind failed)` + ClusterUtils.awaitRingStatus(otherInstance, instanceToStop, "Down"); + NodeToolResult statusResult = cluster.get(2).nodetoolResult("status"); + Assert.assertEquals(0, statusResult.getRc()); + Assert.assertThat(statusResult.getStderr(), is(blankOrNullString())); + Assert.assertThat(statusResult.getStdout(), containsString("DN 127.0.0.1")); + testInstance(instances, cluster.get(2)); + ClusterUtils.start(instanceToStop, props -> { + }); + ClusterUtils.awaitRingState(otherInstance, instanceToStop, "Normal"); + ClusterUtils.awaitRingStatus(otherInstance, instanceToStop, "Up"); + statusResult = cluster.get(1).nodetoolResult("status"); + Assert.assertEquals(0, statusResult.getRc()); + Assert.assertThat(statusResult.getStderr(), is(blankOrNullString())); + Assert.assertThat(statusResult.getStdout(), containsString("UN 127.0.0.1")); + statusResult = cluster.get(2).nodetoolResult("status"); + Assert.assertEquals(0, statusResult.getRc()); + Assert.assertThat(statusResult.getStderr(), is(blankOrNullString())); + Assert.assertThat(statusResult.getStdout(), containsString("UN 127.0.0.1")); + testInstance(instances, cluster.get(1)); + testAllValidGetters(cluster); + } + } + + private void testInstance(Set instancesContacted, IInvokableInstance instance) { - // NOTE: At some point, the hostname of the broadcastAddress can be resolved - // and then the `getHostString`, which would otherwise return the IP address, - // starts returning `localhost` - use `.getAddress().getHostAddress()` to work around this. IInstanceConfig config = instance.config(); try (JMXConnector jmxc = JMXUtil.getJmxConnector(config)) { @@ -109,5 +139,9 @@ public class JMXFeatureTest extends TestBaseImpl instancesContacted.add(defaultDomain); Assert.assertThat(defaultDomain, startsWith(JMXUtil.getJmxHost(config) + ":" + config.jmxPort())); } + catch (Throwable t) + { + throw new RuntimeException("Could not connect to JMX", t); + } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java index ece6052a75..8221e039db 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/jmx/JMXGetterCheckTest.java @@ -29,42 +29,66 @@ import javax.management.MBeanOperationInfo; import javax.management.MBeanServerConnection; import javax.management.ObjectName; import javax.management.remote.JMXConnector; -import javax.management.remote.JMXConnectorFactory; -import javax.management.remote.JMXServiceURL; import com.google.common.collect.ImmutableSet; import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.shared.JMXUtil; import org.apache.cassandra.distributed.test.TestBaseImpl; public class JMXGetterCheckTest extends TestBaseImpl { private static final Set IGNORE_ATTRIBUTES = ImmutableSet.of( - "org.apache.cassandra.net:type=MessagingService:BackPressurePerHost" // throws unsupported saying the feature was removed... dropped in CASSANDRA-15375 + "org.apache.cassandra.net:type=MessagingService:BackPressurePerHost", // throws unsupported saying the feature was removed... dropped in CASSANDRA-15375 + "org.apache.cassandra.db:type=DynamicEndpointSnitch:Scores" // when running in multiple-port-one-IP mode, this fails + ); private static final Set IGNORE_OPERATIONS = ImmutableSet.of( "org.apache.cassandra.db:type=StorageService:stopDaemon", // halts the instance, which then causes the JVM to exit "org.apache.cassandra.db:type=StorageService:drain", // don't drain, it stops things which can cause other APIs to be unstable as we are in a stopped state "org.apache.cassandra.db:type=StorageService:stopGossiping", // if we stop gossip this can cause other issues, so avoid - "org.apache.cassandra.db:type=StorageService:resetLocalSchema" // this will fail when there are no other nodes which can serve schema + "org.apache.cassandra.db:type=StorageService:resetLocalSchema", // this will fail when there are no other nodes which can serve schema + "org.apache.cassandra.db:type=StorageService:joinRing", // Causes bootstrapping errors + "org.apache.cassandra.db:type=StorageService:clearConnectionHistory", // Throws a NullPointerException + "org.apache.cassandra.db:type=StorageService:startGossiping", // causes multiple loops to fail + "org.apache.cassandra.db:type=StorageService:startNativeTransport", // causes multiple loops to fail + "org.apache.cassandra.db:type=Tables,keyspace=system,table=local:loadNewSSTables" // Shouldn't attempt to load SSTables as sometimes the temp directories don't work ); - public static final String JMX_SERVICE_URL_FMT = "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi"; - @Test public void testGetters() throws Exception { - try (Cluster cluster = Cluster.build(1).withConfig(c -> c.with(Feature.values())).start()) + for (int i=0; i < 2; i++) { - IInvokableInstance instance = cluster.get(1); + try (Cluster cluster = Cluster.build(1).withConfig(c -> c.with(Feature.values())).start()) + { + testAllValidGetters(cluster); + } + } + } - String jmxHost = instance.config().broadcastAddress().getAddress().getHostAddress(); - String url = String.format(JMX_SERVICE_URL_FMT, jmxHost, instance.config().jmxPort()); + /** + * Tests JMX getters and operations. + * Useful for more than just testing getters, it also is used in JMXFeatureTest + * to make sure we've touched the complete JMX code path. + * @param cluster the cluster to test + * @throws Exception several kinds of exceptions can be thrown, mostly from JMX infrastructure issues. + */ + public static void testAllValidGetters(Cluster cluster) throws Exception + { + for (IInvokableInstance instance: cluster) + { + if (instance.isShutdown()) + { + continue; + } + IInstanceConfig config = instance.config(); List errors = new ArrayList<>(); - try (JMXConnector jmxc = JMXConnectorFactory.connect(new JMXServiceURL(url), null)) + try (JMXConnector jmxc = JMXUtil.getJmxConnector(config)) { MBeanServerConnection mbsc = jmxc.getMBeanServerConnection(); Set metricNames = new TreeSet<>(mbsc.queryNames(null, null));