diff --git a/CHANGES.txt b/CHANGES.txt
index 0658936306..4e8cf2d0f7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0.8
+ * Improve unit tests performance (CASSANDRA-17427)
* Connect to listen address when own broadcast address is requested (CASSANDRA-18200)
* Add safeguard so cleanup fails when node has pending ranges (CASSANDRA-16418)
* Fix legacy clustering serialization for paging with compact storage (CASSANDRA-17507)
diff --git a/build.xml b/build.xml
index a8b21b61f0..dff8ce96b3 100644
--- a/build.xml
+++ b/build.xml
@@ -111,12 +111,18 @@
-
+
+
+
+
+
+
+
@@ -152,6 +158,10 @@
+
+
+
+
+
@@ -1416,6 +1427,9 @@
+
+
+
@@ -2009,6 +2023,7 @@
+ Apache Cassandra ${eclipse.project.name}
diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml
index 6581dcecd6..ba826e39cf 100644
--- a/ide/idea/workspace.xml
+++ b/ide/idea/workspace.xml
@@ -161,13 +161,19 @@
+
+
+
+
+
+
-
+
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index b725d7f9b8..67de29b999 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -204,7 +204,24 @@ public enum CassandraRelevantProperties
* immediately as it is high chance they will fail anyway. It is better to wait a bit instead of flooding logs
* and wasting resources.
*/
- SCHEMA_PULL_BACKOFF_DELAY_MS("cassandra.schema_pull_backoff_delay_ms", "3000");
+ SCHEMA_PULL_BACKOFF_DELAY_MS("cassandra.schema_pull_backoff_delay_ms", "3000"),
+
+ /** When enabled, recursive directory deletion will be executed using a unix command `rm -rf` instead of traversing
+ * and removing individual files. This is now used only tests, but eventually we will make it true by default.*/
+ USE_NIX_RECURSIVE_DELETE("cassandra.use_nix_recursive_delete"),
+
+ /** If set, {@link org.apache.cassandra.net.MessagingService} is shutdown abrtuptly without waiting for anything.
+ * This is an optimization used in unit tests becuase we never restart a node there. The only node is stopoped
+ * when the JVM terminates. Therefore, we can use such optimization and not wait unnecessarily. */
+ NON_GRACEFUL_SHUTDOWN("cassandra.test.messagingService.nonGracefulShutdown"),
+
+ /** Flush changes of {@link org.apache.cassandra.schema.SchemaKeyspace} after each schema modification. In production,
+ * we always do that. However, tests which do not restart nodes may disable this functionality in order to run
+ * faster. Note that this is disabled for unit tests but if an individual test requires schema to be flushed, it
+ * can be also done manually for that particular case: {@code flush(SchemaConstants.SCHEMA_KEYSPACE_NAME);}. */
+ FLUSH_LOCAL_SCHEMA_CHANGES("cassandra.test.flush_local_schema_changes", "true"),
+
+ ;
CassandraRelevantProperties(String key, String defaultVal)
{
@@ -237,6 +254,17 @@ public enum CassandraRelevantProperties
return value == null ? defaultVal : STRING_CONVERTER.convert(value);
}
+ /**
+ * Sets the property to its default value if a default value was specified. Remove the property otherwise.
+ */
+ public void reset()
+ {
+ if (defaultVal != null)
+ System.setProperty(key, defaultVal);
+ else
+ System.getProperties().remove(key);
+ }
+
/**
* Gets the value of a system property as a String.
* @return system property String value if it exists, overrideDefaultValue otherwise.
@@ -355,4 +383,3 @@ public enum CassandraRelevantProperties
return System.getProperties().containsKey(key);
}
}
-
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index 7798bd785d..2aba2f77b9 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -57,12 +57,15 @@ import org.apache.cassandra.io.FSReadError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.SyncUtil;
import static com.google.common.base.Throwables.propagate;
import static org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
+import static org.apache.cassandra.config.CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE;
import static org.apache.cassandra.utils.Throwables.maybeFail;
import static org.apache.cassandra.utils.Throwables.merge;
@@ -671,9 +674,15 @@ public final class FileUtils
*/
public static void deleteRecursive(File dir)
{
+ if (USE_NIX_RECURSIVE_DELETE.getBoolean() && dir.toPath().getFileSystem() == FileSystems.getDefault())
+ {
+ deleteRecursiveUsingNixCommand(dir.toPath(), false);
+ return;
+ }
+
deleteChildrenRecursive(dir);
- // The directory is now empty so now it can be smoked
+ // The directory is now empty, so now it can be smoked
deleteWithConfirm(dir);
}
@@ -688,8 +697,64 @@ public final class FileUtils
if (dir.isDirectory())
{
String[] children = dir.list();
- for (String child : children)
- deleteRecursive(new File(dir, child));
+ if (children.length == 0)
+ return;
+
+ if (USE_NIX_RECURSIVE_DELETE.getBoolean() && dir.toPath().getFileSystem() == FileSystems.getDefault())
+ {
+ for (String child : children)
+ deleteRecursiveUsingNixCommand(dir.toPath().resolve(child), false);
+ }
+ else
+ {
+ for (String child : children)
+ deleteRecursive(new File(dir, child));
+ }
+ }
+ }
+
+ /**
+ * Uses unix `rm -r` to delete a directory recursively.
+ * This method can be much faster than deleting files and directories recursively by traversing them with Java.
+ * Though, we use it only for tests because it provides less information about the problem when something goes wrong.
+ *
+ * @param path path to be deleted
+ * @param quietly if quietly, additional `-f` flag is added to the `rm` command so that it will not complain in case
+ * the provided path is missing
+ */
+ private static void deleteRecursiveUsingNixCommand(Path path, boolean quietly)
+ {
+ String[] cmd = new String[]{ "rm",
+ quietly ? "-drf" : "-dr",
+ path.toAbsolutePath().toString() };
+
+ try
+ {
+ Process p = Runtime.getRuntime().exec(cmd);
+ int result = p.waitFor();
+
+ String out, err;
+ try (BufferedReader outReader = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ BufferedReader errReader = new BufferedReader(new InputStreamReader(p.getErrorStream())))
+ {
+ out = outReader.lines().collect(Collectors.joining("\n"));
+ err = errReader.lines().collect(Collectors.joining("\n"));
+ }
+
+ if (result != 0 && Files.exists(path))
+ {
+ logger.error("{} returned:\nstdout:\n{}\n\nstderr:\n{}", Arrays.toString(cmd), out, err);
+ throw new IOException(String.format("%s returned non-zero exit code: %d%nstdout:%n%s%n%nstderr:%n%s", Arrays.toString(cmd), result, out, err));
+ }
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, path.toString());
+ }
+ catch (InterruptedException e)
+ {
+ Thread.currentThread().interrupt();
+ throw new FSWriteError(e, path.toString());
}
}
diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java
index 4d712e86d5..71f2231eb2 100644
--- a/src/java/org/apache/cassandra/net/MessagingService.java
+++ b/src/java/org/apache/cassandra/net/MessagingService.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.utils.FBUtilities;
import static java.util.Collections.synchronizedList;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.apache.cassandra.concurrent.Stage.MUTATION;
+import static org.apache.cassandra.config.CassandraRelevantProperties.NON_GRACEFUL_SHUTDOWN;
import static org.apache.cassandra.utils.Throwables.maybeFail;
/**
@@ -422,15 +423,20 @@ public final class MessagingService extends MessagingServiceMBeanImpl
}
/**
- * Wait for callbacks and don't allow any more to be created (since they could require writing hints)
+ * Wait for callbacks and don't allow anymore to be created (since they could require writing hints)
*/
public void shutdown()
{
- shutdown(1L, MINUTES, true, true);
+ if (NON_GRACEFUL_SHUTDOWN.getBoolean())
+ // this branch is used in unit-tests when we really never restart a node and shutting down means the end of test
+ shutdownAbrubtly();
+ else
+ shutdown(1L, MINUTES, true, true);
}
public void shutdown(long timeout, TimeUnit units, boolean shutdownGracefully, boolean shutdownExecutors)
{
+ logger.debug("Shutting down: timeout={}s, gracefully={}, shutdownExecutors={}", units.toSeconds(timeout), shutdownGracefully, shutdownExecutors);
if (isShuttingDown)
{
logger.info("Shutdown was already called");
@@ -454,7 +460,7 @@ public final class MessagingService extends MessagingServiceMBeanImpl
() -> {
List inboundExecutors = new ArrayList<>();
inboundSockets.close(synchronizedList(inboundExecutors)::add).get();
- ExecutorUtils.awaitTermination(1L, TimeUnit.MINUTES, inboundExecutors);
+ ExecutorUtils.awaitTermination(timeout, units, inboundExecutors);
},
() -> {
if (shutdownExecutors)
@@ -486,6 +492,30 @@ public final class MessagingService extends MessagingServiceMBeanImpl
}
}
+ public void shutdownAbrubtly()
+ {
+ logger.debug("Shutting down abruptly");
+ if (isShuttingDown)
+ {
+ logger.info("Shutdown was already called");
+ return;
+ }
+
+ isShuttingDown = true;
+ logger.info("Waiting for messaging service to quiesce");
+ // We may need to schedule hints on the mutation stage, so it's erroneous to shut down the mutation stage first
+ assert !MUTATION.executor().isShutdown();
+
+ callbacks.shutdownNow(false);
+ inboundSockets.close();
+ for (OutboundConnections pool : channelManagers.values())
+ pool.close(false);
+
+ maybeFail(socketFactory::shutdownNow,
+ inboundSink::clear,
+ outboundSink::clear);
+ }
+
private void shutdownExecutors(long deadlineNanos) throws TimeoutException, InterruptedException
{
socketFactory.shutdownNow();
diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
index b4a322f8a8..a6fd028753 100644
--- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
+++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java
@@ -73,7 +73,7 @@ final class SchemaKeyspace
private static final Logger logger = LoggerFactory.getLogger(SchemaKeyspace.class);
- private static final boolean FLUSH_SCHEMA_TABLES = Boolean.parseBoolean(System.getProperty("cassandra.test.flush_local_schema_changes", "true"));
+ private static final boolean FLUSH_SCHEMA_TABLES = CassandraRelevantProperties.FLUSH_LOCAL_SCHEMA_CHANGES.getBoolean();
private static final boolean IGNORE_CORRUPTED_SCHEMA_TABLES = Boolean.parseBoolean(System.getProperty("cassandra.ignore_corrupted_schema_tables", "false"));
/**
diff --git a/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java b/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
index 3ff6820714..8f86831cc3 100644
--- a/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
+++ b/src/java/org/apache/cassandra/security/ThreadAwareSecurityManager.java
@@ -30,10 +30,12 @@ import java.security.ProtectionDomain;
import java.util.Collections;
import java.util.Enumeration;
-import io.netty.util.concurrent.FastThreadLocal;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.cassandra.utils.logging.LoggingSupportFactory;
+import io.netty.util.concurrent.FastThreadLocal;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.utils.logging.LoggingSupportFactory;
/**
* Custom {@link SecurityManager} and {@link Policy} implementation that only performs access checks
@@ -46,6 +48,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
*/
public final class ThreadAwareSecurityManager extends SecurityManager
{
+ private static final Logger logger = LoggerFactory.getLogger(ThreadAwareSecurityManager.class);
+
public static final PermissionCollection noPermissions = new PermissionCollection()
{
public void add(Permission permission)
@@ -81,6 +85,14 @@ public final class ThreadAwareSecurityManager extends SecurityManager
{
if (installed)
return;
+
+ // this line is needed - we need to make sure AccessControlException is loaded before we install this SM
+ // otherwise we may get into stackoverflow when javax.security is not allowed package, and ACE is tried to be
+ // loaded when it is going to be thrown from SM (class loader triggers SM to verify javax.security,
+ // it recognizes it as not allowed and attempts to throw it...)
+ //noinspection PlaceholderCountMatchesArgumentCount
+ logger.trace("Initialized thread aware security manager", AccessControlException.class.getName());
+
System.setSecurityManager(new ThreadAwareSecurityManager());
LoggingSupportFactory.getLoggingSupport().onStartup();
installed = true;
diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
index d02a4f76e4..95a2ac5a62 100644
--- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
+++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java
@@ -48,6 +48,7 @@ import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
@@ -163,6 +164,12 @@ public abstract class AbstractCluster implements ICluster factory)
{
super(factory);
@@ -941,4 +948,3 @@ public abstract class AbstractCluster implements ICluster