From 07163cfb5172b8b848e8ecf43581311280154e70 Mon Sep 17 00:00:00 2001 From: Paulo Motta Date: Thu, 28 Nov 2024 17:24:51 -0500 Subject: [PATCH] Fix StorageService JMX mbean not available during bootstrap Patch by Paulo Motta; reviewed by Stefan Miklosovic for CASSANDRA-19902 --- CHANGES.txt | 1 + .../cassandra/service/StorageService.java | 3 +- .../distributed/test/ring/BootstrapTest.java | 96 +++++++++++++++++++ .../org/apache/cassandra/db/RowCacheTest.java | 5 +- .../cassandra/triggers/TriggersTest.java | 3 +- 5 files changed, 101 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 016da31427..d8ac0caa5e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0.5 + * Fix StorageService JMX mbean not available during bootstrap (CASSANDRA-19902) * SSTableIndexWriter#abort() should log more quietly in cases where an exception is not provided (CASSANDRA-20695) * Avoid availability gap between UP and queryability marking for already built SAI indexes on bounce (CASSANDRA-20732) * Make Commitlog flush data safely in Direct IO mode (CASSANDRA-20692) diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index ee38ab2a78..e558635b3c 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1003,6 +1003,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } }, "StorageServiceShutdownHook"); Runtime.getRuntime().addShutdownHook(drainOnShutdown); + registerMBeans(); replacing = isReplacing(); @@ -1053,8 +1054,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE @VisibleForTesting public void completeInitialization() { - if (!initialized) - registerMBeans(); initialized = true; } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java index 5a042251bb..46a179718e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/BootstrapTest.java @@ -18,15 +18,30 @@ package org.apache.cassandra.distributed.test.ring; +import java.io.Closeable; +import java.io.IOException; import java.lang.management.ManagementFactory; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.management.JMX; +import javax.management.MBeanServerConnection; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; + +import com.google.common.util.concurrent.Uninterruptibles; import org.junit.After; import org.junit.Assert; import org.junit.AssumptionViolatedException; @@ -44,19 +59,23 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.ICluster; import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.TokenSupplier; +import org.apache.cassandra.distributed.shared.JMXUtil; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.shared.WithProperties; import org.apache.cassandra.distributed.test.DecommissionTest; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.StorageServiceMBean; import static java.util.Arrays.asList; import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.config.CassandraRelevantProperties.JOIN_RING; import static org.apache.cassandra.config.CassandraRelevantProperties.MIGRATION_DELAY; import static org.apache.cassandra.config.CassandraRelevantProperties.RESET_BOOTSTRAP_PROGRESS; @@ -365,4 +384,81 @@ public class BootstrapTest extends TestBaseImpl } } + /** + * This regression test for CASSANDRA-19902 ensures {@link StorageServiceMBean} JMX + * interface is published before the node finishes bootstrapping + */ + @Test + public void testStorageServiceMBeanIsPublishedOnJMXDuringBootstrap() throws Throwable + { + ExecutorService es = Executors.newFixedThreadPool(1); + try (Cluster cluster = builder().withNodes(2) + .withConfig(config -> config.with(GOSSIP) + .with(NETWORK) + .with(Feature.JMX) + .set("auto_bootstrap", true)) + .withInstanceInitializer(BBBootstrapInterceptor::install) + .createWithoutStarting(); + Closeable ignored = es::shutdown) + { + Runnable test = () -> + { + // Wait for bootstrap to start via countdown latch + IInvokableInstance joiningInstance = cluster.get(2); + joiningInstance.runOnInstance(() -> Uninterruptibles.awaitUninterruptibly(BBBootstrapInterceptor.bootstrapStart)); + // At this point, it should be possible to check bootstrap status via JMX + IInstanceConfig config = joiningInstance.config(); + try (JMXConnector jmxc = JMXUtil.getJmxConnector(config)) + { + MBeanServerConnection mbsc = jmxc.getMBeanServerConnection(); + StorageServiceMBean sp = JMX.newMBeanProxy(mbsc, new ObjectName("org.apache.cassandra.db:type=StorageService"), StorageServiceMBean.class); + assertEquals(sp.getOperationMode(), StorageService.Mode.JOINING.toString()); + } + catch (IOException | MalformedObjectNameException e) + { + throw new RuntimeException(e); + } + finally + { + // Complete bootstrap via countdown latch so test will finish properly + joiningInstance.runOnInstance(() -> BBBootstrapInterceptor.bootstrapReady.countDown()); + } + }; + + Future testResult = es.submit(test); + try + { + cluster.startup(); + } + catch (Exception ex) { + // ignore exceptions from startup process. More interested in the test result. + } + testResult.get(); + } + es.awaitTermination(5, TimeUnit.SECONDS); + } + + public static class BBBootstrapInterceptor + { + final static CountDownLatch bootstrapReady = new CountDownLatch(1); + final static CountDownLatch bootstrapStart = new CountDownLatch(1); + static void install(ClassLoader cl, int nodeNumber) + { + if (nodeNumber != 2) + return; + new ByteBuddy().rebase(StorageService.class) + .method(named("bootstrap").and(takesArguments(2))) + .intercept(MethodDelegation.to(BBBootstrapInterceptor.class)) + .make() + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + + public static boolean bootstrap(Collection tokens, long bootstrapTimeoutMillis) + { + bootstrapStart.countDown(); + Uninterruptibles.awaitUninterruptibly(bootstrapReady); + return false; // bootstrap fails + } + } + } diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java index 4b37e69414..71ca945ae6 100644 --- a/test/unit/org/apache/cassandra/db/RowCacheTest.java +++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java @@ -65,7 +65,7 @@ public class RowCacheTest private static final String CF_CACHEDNOCLUSTER = "CachedNoClustering"; @BeforeClass - public static void defineSchema() throws ConfigurationException + public static void beforeTest() throws ConfigurationException { TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT.setInt(16); SchemaLoader.prepareServer(); @@ -76,6 +76,7 @@ public class RowCacheTest SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHED).caching(CachingParams.CACHE_EVERYTHING), SchemaLoader.standardCFMD(KEYSPACE_CACHED, CF_CACHEDINT, 1, IntegerType.instance) .caching(new CachingParams(true, 100))); + StorageService.instance.initServer(0); } @AfterClass @@ -292,7 +293,6 @@ public class RowCacheTest @Test public void testRowCacheCleanup() throws Exception { - StorageService.instance.initServer(0); CacheService.instance.setRowCacheCapacityInMB(1); rowCacheLoad(100, Integer.MAX_VALUE, 1000); @@ -314,7 +314,6 @@ public class RowCacheTest @Test public void testInvalidateRowCache() throws Exception { - StorageService.instance.initServer(0); CacheService.instance.setRowCacheCapacityInMB(1); rowCacheLoad(100, Integer.MAX_VALUE, 1000); diff --git a/test/unit/org/apache/cassandra/triggers/TriggersTest.java b/test/unit/org/apache/cassandra/triggers/TriggersTest.java index 892a0220f9..77a92b5560 100644 --- a/test/unit/org/apache/cassandra/triggers/TriggersTest.java +++ b/test/unit/org/apache/cassandra/triggers/TriggersTest.java @@ -52,13 +52,12 @@ public class TriggersTest public static void beforeTest() throws ConfigurationException { SchemaLoader.loadSchema(); + StorageService.instance.initServer(0); } @Before public void setup() throws Exception { - StorageService.instance.initServer(0); - String cql = String.format("CREATE KEYSPACE IF NOT EXISTS %s " + "WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}", ksName);