natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.0' into cassandra-3.X

This commit is contained in:
Stefania Alborghetti 2016-10-04 09:53:37 +08:00
parent c089b26976 5115c106db
commit 2e18adf257
8 changed files with 183 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -70,6 +70,7 @@
* Remove compaction Severity from DynamicEndpointSnitch (CASSANDRA-11738)
* Restore resumable hints delivery (CASSANDRA-11960)
Merged from 3.0:
* Unify drain and shutdown processes (CASSANDRA-12509)
* Fix NPE in ComponentOfSlice.isEQ() (CASSANDRA-12706)
* Fix failure in LogTransactionTest (CASSANDRA-12632)
* Fix potentially incomplete non-frozen UDT values when querying with the

3
conf/logback.xml
View File

@ -24,7 +24,8 @@ appender reference in the root level section below.
<configuration scan="true">
<jmxConfigurator />
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
<!-- No shutdown hook; we run it ourselves in StorageService after shutdown -->
<!-- SYSTEMLOG rolling file appender to system.log (INFO level) -->

2
src/java/org/apache/cassandra/concurrent/DebuggableScheduledThreadPoolExecutor.java
View File

@ -47,7 +47,7 @@ public class DebuggableScheduledThreadPoolExecutor extends ScheduledThreadPoolEx
{
if (executor.isShutdown())
{
if (!StorageService.instance.isInShutdownHook())
if (!StorageService.instance.isShutdown())
throw new RejectedExecutionException("ScheduledThreadPoolExecutor has shut down.");
//Give some notification to the caller the task isn't going to run

8
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -2411,7 +2411,13 @@ public class StorageProxy implements StorageProxyMBean
public void setHintedHandoffEnabled(boolean b)
{
DatabaseDescriptor.setHintedHandoffEnabled(b);
synchronized (StorageService.instance)
{
if (b)
StorageService.instance.checkServiceAllowedToStart("hinted handoff");
DatabaseDescriptor.setHintedHandoffEnabled(b);
}
}
public void enableHintsForDC(String dc)

295
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -50,6 +50,7 @@ import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.jmx.JMXConfiguratorMBean;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.hook.DelayingShutdownHook;
import org.apache.cassandra.auth.AuthKeyspace;
import org.apache.cassandra.auth.AuthMigrationListener;
import org.apache.cassandra.batchlog.BatchRemoveVerbHandler;
@ -140,13 +141,19 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public volatile VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(tokenMetadata.partitioner);
private Thread drainOnShutdown = null;
private volatile boolean inShutdownHook = false;
private volatile boolean isShutdown = false;
public static final StorageService instance = new StorageService();
@Deprecated
public boolean isInShutdownHook()
{
return inShutdownHook;
return isShutdown();
}
public boolean isShutdown()
{
return isShutdown;
}
public Collection<Range<Token>> getLocalRanges(String keyspaceName)
@ -187,7 +194,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
private double traceProbability = 0.0;
private static enum Mode { STARTING, NORMAL, JOINING, LEAVING, DECOMMISSIONED, MOVING, DRAINING, DRAINED }
private Mode operationMode = Mode.STARTING;
private volatile Mode operationMode = Mode.STARTING;
/* Used for tracking drain progress */
private volatile int totalCFs, remainingCFs;
@ -309,10 +316,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
// should only be called via JMX
public void startGossiping()
public synchronized void startGossiping()
{
if (!gossipActive)
{
checkServiceAllowedToStart("gossip");
logger.warn("Starting gossip by operator request");
Collection<Token> tokens = SystemKeyspace.getSavedTokens();
@ -338,8 +347,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
// should only be called via JMX
public void startRPCServer()
public synchronized void startRPCServer()
{
checkServiceAllowedToStart("thrift");
if (daemon == null)
{
throw new IllegalStateException("No configured daemon");
@ -366,8 +377,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return daemon.thriftServer.isRunning();
}
public void startNativeTransport()
public synchronized void startNativeTransport()
{
checkServiceAllowedToStart("native transport");
if (daemon == null)
{
throw new IllegalStateException("No configured daemon");
@ -605,65 +618,17 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
drainOnShutdown = new Thread(new WrappedRunnable()
{
@Override
public void runMayThrow() throws InterruptedException, ExecutionException
public void runMayThrow() throws InterruptedException, ExecutionException, IOException
{
inShutdownHook = true;
ExecutorService viewMutationStage = StageManager.getStage(Stage.VIEW_MUTATION);
ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
ExecutorService mutationStage = StageManager.getStage(Stage.MUTATION);
if (mutationStage.isShutdown()
&& counterMutationStage.isShutdown()
&& viewMutationStage.isShutdown())
return; // drained already
if (daemon != null)
shutdownClientServers();
ScheduledExecutors.optionalTasks.shutdown();
Gossiper.instance.stop();
// In-progress writes originating here could generate hints to be written, so shut down MessagingService
// before mutation stage, so we can get all the hints saved before shutting down
MessagingService.instance().shutdown();
viewMutationStage.shutdown();
BatchlogManager.instance.shutdown();
HintsService.instance.pauseDispatch();
counterMutationStage.shutdown();
mutationStage.shutdown();
viewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
counterMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
mutationStage.awaitTermination(3600, TimeUnit.SECONDS);
StorageProxy.instance.verifyNoHintsInProgress();
List<Future<?>> flushes = new ArrayList<>();
for (Keyspace keyspace : Keyspace.all())
{
KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace.getName());
if (!ksm.params.durableWrites)
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
flushes.add(cfs.forceFlush());
}
try
{
FBUtilities.waitOnFutures(flushes);
}
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
// don't let this stop us from shutting down the commitlog and other thread pools
logger.warn("Caught exception while waiting for memtable flushes during shutdown hook", t);
}
CommitLog.instance.shutdownBlocking();
drain(true);
if (FBUtilities.isWindows)
WindowsTimer.endTimerPeriod(DatabaseDescriptor.getWindowsTimerInterval());
HintsService.instance.shutdownBlocking();
// wait for miscellaneous tasks like sstable and commitlog segment deletion
ScheduledExecutors.nonPeriodicTasks.shutdown();
if (!ScheduledExecutors.nonPeriodicTasks.awaitTermination(1, TimeUnit.MINUTES))
logger.warn("Miscellaneous task executor still busy after one minute; proceeding with shutdown");
// Cleanup logback
DelayingShutdownHook logbackHook = new DelayingShutdownHook();
logbackHook.setContext((LoggerContext)LoggerFactory.getILoggerFactory());
logbackHook.run();
}
}, "StorageServiceShutdownHook");
Runtime.getRuntime().addShutdownHook(drainOnShutdown);
@ -1168,7 +1133,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public void rebuild(String sourceDc, String keyspace, String tokens, String specificSources)
{
// check on going rebuild
// check ongoing rebuild
if (!isRebuilding.compareAndSet(false, true))
{
throw new IllegalStateException("Node is still rebuilding. Check nodetool netstats.");
@ -4373,6 +4338,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return operationMode == Mode.JOINING;
}
public boolean isDrained()
{
return operationMode == Mode.DRAINED;
}
public boolean isDraining()
{
return operationMode == Mode.DRAINING;
}
public String getDrainProgress()
{
return String.format("Drained %s/%s ColumnFamilies", remainingCFs, totalCFs);
@ -4380,102 +4355,146 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
/**
* Shuts node off to writes, empties memtables and the commit log.
* There are two differences between drain and the normal shutdown hook:
* - Drain waits for in-progress streaming to complete
* - Drain flushes *all* columnfamilies (shutdown hook only flushes non-durable CFs)
*/
public synchronized void drain() throws IOException, InterruptedException, ExecutionException
{
inShutdownHook = true;
BatchlogManager.instance.shutdown();
HintsService.instance.pauseDispatch();
drain(false);
}
protected synchronized void drain(boolean isFinalShutdown) throws IOException, InterruptedException, ExecutionException
{
ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
ExecutorService viewMutationStage = StageManager.getStage(Stage.VIEW_MUTATION);
ExecutorService mutationStage = StageManager.getStage(Stage.MUTATION);
if (mutationStage.isTerminated()
&& counterMutationStage.isTerminated()
&& viewMutationStage.isTerminated())
{
logger.warn("Cannot drain node (did it already happen?)");
if (!isFinalShutdown)
logger.warn("Cannot drain node (did it already happen?)");
return;
}
setMode(Mode.DRAINING, "starting drain process", true);
shutdownClientServers();
ScheduledExecutors.optionalTasks.shutdown();
Gossiper.instance.stop();
setMode(Mode.DRAINING, "shutting down MessageService", false);
MessagingService.instance().shutdown();
assert !isShutdown;
isShutdown = true;
setMode(Mode.DRAINING, "clearing mutation stage", false);
viewMutationStage.shutdown();
counterMutationStage.shutdown();
mutationStage.shutdown();
viewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
counterMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
mutationStage.awaitTermination(3600, TimeUnit.SECONDS);
StorageProxy.instance.verifyNoHintsInProgress();
setMode(Mode.DRAINING, "flushing column families", false);
// disable autocompaction - we don't want to start any new compactions while we are draining
for (Keyspace keyspace : Keyspace.all())
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
cfs.disableAutoCompaction();
// count CFs first, since forceFlush could block for the flushWriter to get a queue slot empty
totalCFs = 0;
for (Keyspace keyspace : Keyspace.nonSystem())
totalCFs += keyspace.getColumnFamilyStores().size();
remainingCFs = totalCFs;
// flush
List<Future<?>> flushes = new ArrayList<>();
for (Keyspace keyspace : Keyspace.nonSystem())
try
{
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
flushes.add(cfs.forceFlush());
setMode(Mode.DRAINING, "starting drain process", !isFinalShutdown);
BatchlogManager.instance.shutdown();
HintsService.instance.pauseDispatch();
if (daemon != null)
shutdownClientServers();
ScheduledExecutors.optionalTasks.shutdown();
Gossiper.instance.stop();
if (!isFinalShutdown)
setMode(Mode.DRAINING, "shutting down MessageService", false);
// In-progress writes originating here could generate hints to be written, so shut down MessagingService
// before mutation stage, so we can get all the hints saved before shutting down
MessagingService.instance().shutdown();
if (!isFinalShutdown)
setMode(Mode.DRAINING, "clearing mutation stage", false);
viewMutationStage.shutdown();
counterMutationStage.shutdown();
mutationStage.shutdown();
viewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
counterMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
mutationStage.awaitTermination(3600, TimeUnit.SECONDS);
StorageProxy.instance.verifyNoHintsInProgress();
if (!isFinalShutdown)
setMode(Mode.DRAINING, "flushing column families", false);
// disable autocompaction - we don't want to start any new compactions while we are draining
for (Keyspace keyspace : Keyspace.all())
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
cfs.disableAutoCompaction();
// count CFs first, since forceFlush could block for the flushWriter to get a queue slot empty
totalCFs = 0;
for (Keyspace keyspace : Keyspace.nonSystem())
totalCFs += keyspace.getColumnFamilyStores().size();
remainingCFs = totalCFs;
// flush
List<Future<?>> flushes = new ArrayList<>();
for (Keyspace keyspace : Keyspace.nonSystem())
{
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
flushes.add(cfs.forceFlush());
}
// wait for the flushes.
// TODO this is a godawful way to track progress, since they flush in parallel. a long one could
// thus make several short ones "instant" if we wait for them later.
for (Future f : flushes)
{
try
{
FBUtilities.waitOnFuture(f);
}
catch (Throwable t)
{
JVMStabilityInspector.inspectThrowable(t);
// don't let this stop us from shutting down the commitlog and other thread pools
logger.warn("Caught exception while waiting for memtable flushes during shutdown hook", t);
}
remainingCFs--;
}
// flush the system ones after all the rest are done, just in case flushing modifies any system state
// like CASSANDRA-5151. don't bother with progress tracking since system data is tiny.
flushes.clear();
for (Keyspace keyspace : Keyspace.system())
{
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
flushes.add(cfs.forceFlush());
}
FBUtilities.waitOnFutures(flushes);
HintsService.instance.shutdownBlocking();
// Interrupt ongoing compactions and shutdown CM to prevent further compactions.
CompactionManager.instance.forceShutdown();
// whilst we've flushed all the CFs, which will have recycled all completed segments, we want to ensure
// there are no segments to replay, so we force the recycling of any remaining (should be at most one)
CommitLog.instance.forceRecycleAllSegments();
CommitLog.instance.shutdownBlocking();
// wait for miscellaneous tasks like sstable and commitlog segment deletion
ScheduledExecutors.nonPeriodicTasks.shutdown();
if (!ScheduledExecutors.nonPeriodicTasks.awaitTermination(1, TimeUnit.MINUTES))
logger.warn("Miscellaneous task executor still busy after one minute; proceeding with shutdown");
ColumnFamilyStore.shutdownPostFlushExecutor();
setMode(Mode.DRAINED, !isFinalShutdown);
}
// wait for the flushes.
// TODO this is a godawful way to track progress, since they flush in parallel. a long one could
// thus make several short ones "instant" if we wait for them later.
for (Future f : flushes)
catch (Throwable t)
{
FBUtilities.waitOnFuture(f);
remainingCFs--;
logger.error("Caught an exception while draining ", t);
}
// flush the system ones after all the rest are done, just in case flushing modifies any system state
// like CASSANDRA-5151. don't bother with progress tracking since system data is tiny.
flushes.clear();
for (Keyspace keyspace : Keyspace.system())
{
for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
flushes.add(cfs.forceFlush());
}
FBUtilities.waitOnFutures(flushes);
}
HintsService.instance.shutdownBlocking();
/**
* Some services are shutdown during draining and we should not attempt to start them again.
*
* @param service - the name of the service we are trying to start.
* @throws IllegalStateException - an exception that nodetool is able to convert into a message to display to the user
*/
synchronized void checkServiceAllowedToStart(String service)
{
if (isDraining()) // when draining isShutdown is also true, so we check first to return a more accurate message
throw new IllegalStateException(String.format("Unable to start %s because the node is draining.", service));
// Interrupt on going compaction and shutdown to prevent further compaction
CompactionManager.instance.forceShutdown();
// whilst we've flushed all the CFs, which will have recycled all completed segments, we want to ensure
// there are no segments to replay, so we force the recycling of any remaining (should be at most one)
CommitLog.instance.forceRecycleAllSegments();
CommitLog.instance.shutdownBlocking();
// wait for miscellaneous tasks like sstable and commitlog segment deletion
ScheduledExecutors.nonPeriodicTasks.shutdown();
if (!ScheduledExecutors.nonPeriodicTasks.awaitTermination(1, TimeUnit.MINUTES))
logger.warn("Miscellaneous task executor still busy after one minute; proceeding with shutdown");
ColumnFamilyStore.shutdownPostFlushExecutor();
setMode(Mode.DRAINED, true);
if (isShutdown()) // do not rely on operationMode in case it gets changed to decomissioned or other
throw new IllegalStateException(String.format("Unable to start %s because the node was drained.", service));
}
// Never ever do this at home. Used by tests.
@ -4966,8 +4985,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
}
}
public void enableAutoCompaction(String ks, String... tables) throws IOException
public synchronized void enableAutoCompaction(String ks, String... tables) throws IOException
{
checkServiceAllowedToStart("auto compaction");
for (ColumnFamilyStore cfs : getValidColumnFamilies(true, true, ks, tables))
{
cfs.enableAutoCompaction();

2
src/java/org/apache/cassandra/service/StorageServiceMBean.java
View File

@ -526,6 +526,8 @@ public interface StorageServiceMBean extends NotificationEmitter
// allows a node that have been started without joining the ring to join it
public void joinRing() throws IOException;
public boolean isJoined();
public boolean isDrained();
public boolean isDraining();
public void setRpcTimeout(long value);
public long getRpcTimeout();

10
src/java/org/apache/cassandra/tools/NodeProbe.java
View File

@ -616,6 +616,16 @@ public class NodeProbe implements AutoCloseable
return ssProxy.isJoined();
}
public boolean isDrained()
{
return ssProxy.isDrained();
}
public boolean isDraining()
{
return ssProxy.isDraining();
}
public void joinRing() throws IOException
{
ssProxy.joinRing();

4
test/unit/org/apache/cassandra/cql3/validation/entities/UFTest.java
View File

@ -2331,7 +2331,7 @@ public class UFTest extends CQLTester
'}'},
{"org.apache.cassandra.service.StorageService",
"try {" +
" org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isInShutdownHook(); return 0d;" +
" org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isShutdown(); return 0d;" +
"} catch (Exception t) {" +
" throw new RuntimeException(t);" +
'}'},
@ -2371,7 +2371,7 @@ public class UFTest extends CQLTester
"RETURNS NULL ON NULL INPUT " +
"RETURNS double " +
"LANGUAGE javascript\n" +
"AS 'org.apache.cassandra.service.StorageService.instance.isInShutdownHook(); 0;';");
"AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';");
execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
Assert.fail("Javascript security check failed");
}