From b851792c4e3ae32b8d863d9079cca6d135f1cf23 Mon Sep 17 00:00:00 2001 From: Yuki Morishita Date: Wed, 18 May 2016 17:03:39 -0500 Subject: [PATCH] Backport CASSANDRA-11578 patch by yukim; reviewed by Paulo Motta for CASSANDRA-11750 --- CHANGES.txt | 1 + .../apache/cassandra/io/FSErrorHandler.java | 30 ++++++ .../apache/cassandra/io/util/FileUtils.java | 75 +++---------- .../cassandra/service/CassandraDaemon.java | 2 + .../service/DefaultFSErrorHandler.java | 101 ++++++++++++++++++ .../apache/cassandra/db/DirectoriesTest.java | 2 + 6 files changed, 152 insertions(+), 59 deletions(-) create mode 100644 src/java/org/apache/cassandra/io/FSErrorHandler.java create mode 100644 src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java diff --git a/CHANGES.txt b/CHANGES.txt index f73db6ea00..ad9d00c7db 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.15 + * Backport CASSANDRA-11578 (CASSANDRA-11750) * Clear out parent repair session if repair coordinator dies (CASSANDRA-11824) * Set default streaming_socket_timeout_in_ms to 24 hours (CASSANDRA-11840) * Do not consider local node a valid source during replace (CASSANDRA-11848) diff --git a/src/java/org/apache/cassandra/io/FSErrorHandler.java b/src/java/org/apache/cassandra/io/FSErrorHandler.java new file mode 100644 index 0000000000..081ec0bb8b --- /dev/null +++ b/src/java/org/apache/cassandra/io/FSErrorHandler.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io; + +import org.apache.cassandra.io.sstable.CorruptSSTableException; + +/** + * Interface for handling file system errors. + */ +public interface FSErrorHandler +{ + void handleCorruptSSTable(CorruptSSTableException e); + void handleFSError(FSError e); +} diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index 3be7c99f63..f69ed01deb 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.text.DecimalFormat; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; import sun.nio.ch.DirectBuffer; @@ -39,17 +40,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.BlacklistedDirectories; -import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.FSErrorHandler; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.sstable.CorruptSSTableException; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.JVMStabilityInspector; -public class FileUtils +public final class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); private static final double KB = 1024d; @@ -59,6 +57,7 @@ public class FileUtils private static final DecimalFormat df = new DecimalFormat("#.##"); private static final boolean canCleanDirectBuffers; + private static final AtomicReference fsErrorHandler = new AtomicReference<>(); static { @@ -395,63 +394,16 @@ public class FileUtils public static void handleCorruptSSTable(CorruptSSTableException e) { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - StorageService.instance.stopTransports(); - break; - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleCorruptSSTable(e); } - + public static void handleFSError(FSError e) { - if (!StorageService.instance.isSetupCompleted()) - handleStartupFSError(e); - - JVMStabilityInspector.inspectThrowable(e); - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - StorageService.instance.stopTransports(); - break; - case best_effort: - // for both read and write errors mark the path as unwritable. - BlacklistedDirectories.maybeMarkUnwritable(e.path); - if (e instanceof FSReadError) - { - File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path); - if (directory != null) - Keyspace.removeUnreadableSSTables(directory); - } - break; - case ignore: - // already logged, so left nothing to do - break; - default: - throw new IllegalStateException(); - } - } - - private static void handleStartupFSError(Throwable t) - { - switch (DatabaseDescriptor.getDiskFailurePolicy()) - { - case stop_paranoid: - case stop: - case die: - logger.error("Exiting forcefully due to file system exception on startup, disk failure policy \"{}\"", - DatabaseDescriptor.getDiskFailurePolicy(), - t); - JVMStabilityInspector.killCurrentJVM(t, true); - break; - default: - break; - } + FSErrorHandler handler = fsErrorHandler.get(); + if (handler != null) + handler.handleFSError(e); } /** * Get the size of a directory in bytes @@ -470,4 +422,9 @@ public class FileUtils } return length; } + + public static void setFSErrorHandler(FSErrorHandler handler) + { + fsErrorHandler.getAndSet(handler); + } } diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index a27fa207f0..5a1fb14257 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -148,6 +148,8 @@ public class CassandraDaemon */ protected void setup() { + FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); + try { logger.info("Hostname: {}", InetAddress.getLocalHost().getHostName()); diff --git a/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java new file mode 100644 index 0000000000..88a1fce53b --- /dev/null +++ b/src/java/org/apache/cassandra/service/DefaultFSErrorHandler.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service; + +import java.io.File; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.BlacklistedDirectories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.FSError; +import org.apache.cassandra.io.FSErrorHandler; +import org.apache.cassandra.io.FSReadError; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.utils.JVMStabilityInspector; + +public class DefaultFSErrorHandler implements FSErrorHandler +{ + private static final Logger logger = LoggerFactory.getLogger(DefaultFSErrorHandler.class); + + @Override + public void handleCorruptSSTable(CorruptSSTableException e) + { + if (!StorageService.instance.isSetupCompleted()) + handleStartupFSError(e); + + JVMStabilityInspector.inspectThrowable(e); + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + StorageService.instance.stopTransports(); + break; + } + } + + @Override + public void handleFSError(FSError e) + { + if (!StorageService.instance.isSetupCompleted()) + handleStartupFSError(e); + + JVMStabilityInspector.inspectThrowable(e); + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + case stop: + StorageService.instance.stopTransports(); + break; + case best_effort: + // for both read and write errors mark the path as unwritable. + BlacklistedDirectories.maybeMarkUnwritable(e.path); + if (e instanceof FSReadError) + { + File directory = BlacklistedDirectories.maybeMarkUnreadable(e.path); + if (directory != null) + Keyspace.removeUnreadableSSTables(directory); + } + break; + case ignore: + // already logged, so left nothing to do + break; + default: + throw new IllegalStateException(); + } + } + + private static void handleStartupFSError(Throwable t) + { + switch (DatabaseDescriptor.getDiskFailurePolicy()) + { + case stop_paranoid: + case stop: + case die: + logger.error("Exiting forcefully due to file system exception on startup, disk failure policy \"{}\"", + DatabaseDescriptor.getDiskFailurePolicy(), + t); + JVMStabilityInspector.killCurrentJVM(t, true); + break; + default: + break; + } + } +} diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java index b1c51eeea8..4267c1f176 100644 --- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java +++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java @@ -44,6 +44,7 @@ import org.apache.cassandra.db.Directories.DataDirectory; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.DefaultFSErrorHandler; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.io.FSWriteError; @@ -65,6 +66,7 @@ public class DirectoriesTest @BeforeClass public static void beforeClass() throws IOException { + FileUtils.setFSErrorHandler(new DefaultFSErrorHandler()); for (String cf : CFS) { CFM.add(new CFMetaData(KS, cf, ColumnFamilyType.Standard, null));