Merge branch 'cassandra-2.2' into cassandra-3.0

This commit is contained in:
Alex Petrov 2019-02-25 14:14:19 +01:00
commit b27cc37abd
11 changed files with 213 additions and 49 deletions

View File

@ -171,8 +171,6 @@ jobs:
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# append distributed tests
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/distributed/**/test/*.java" > /tmp/all_java_distributed_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
@ -180,11 +178,6 @@ jobs:
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_distributed_tests.txt > /tmp/java_dtests_${CIRCLE_NODE_INDEX}.txt
set +eo pipefail && cat /tmp/java_dtests_${CIRCLE_NODE_INDEX}.txt | cut -c 44-1000000 | grep "Test\.java$" > /tmp/java_dtests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_dtests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_dtests_${CIRCLE_NODE_INDEX}_final.txt
- run:
name: Run Unit Tests
command: |
@ -198,8 +191,8 @@ jobs:
cd /tmp/cassandra
ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
if [ -s "/tmp/java_dtests_${CIRCLE_NODE_INDEX}_final.txt" ]; then
ant testclasslist -Dtest.classlistfile=/tmp/java_dtests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed
if [ "${CIRCLE_NODE_INDEX}" -eq "1" ]; then
ant test-jvm-dtest-forking
fi
no_output_timeout: 15m
- store_test_results:

View File

@ -65,6 +65,7 @@
<property name="test.burn.src" value="${test.dir}/burn"/>
<property name="test.microbench.src" value="${test.dir}/microbench"/>
<property name="test.distributed.src" value="${test.dir}/distributed"/>
<property name="test.distributed.listfile" value="ant-jvm-dtest-list"/>
<property name="dist.dir" value="${build.dir}/dist"/>
<property name="tmp.dir" value="${java.io.tmpdir}"/>
@ -98,7 +99,7 @@
<property name="test.timeout" value="240000" />
<property name="test.long.timeout" value="600000" />
<property name="test.burn.timeout" value="60000000" />
<property name="test.distributed.timeout" value="600000" />
<property name="test.distributed.timeout" value="360000" />
<!-- default for cql tests. Can be override by -Dcassandra.test.use_prepared=false -->
<property name="cassandra.test.use_prepared" value="true" />
@ -1169,15 +1170,6 @@
</copy>
</target>
<target name="dtest-jar" depends="build-test, build" description="Create dtest-compatible jar, including all dependencies">
<jar jarfile="${build.dir}/dtest-${base.version}.jar">
<zipgroupfileset dir="${build.lib}" includes="*.jar" excludes="META-INF/*.SF"/>
<fileset dir="${build.classes.main}"/>
<fileset dir="${test.classes}"/>
<fileset dir="${test.conf}" />
</jar>
</target>
<!-- Defines how to run a set of tests. If you change the defaults for attributes
you should also update them in testmacro.,
The two are split because the helper doesn't generate
@ -1194,6 +1186,7 @@
<attribute name="poffset" default="0"/>
<attribute name="testtag" default=""/>
<attribute name="usejacoco" default="no"/>
<attribute name="showoutput" default="false"/>
<sequential>
<condition property="additionalagent"
@ -1206,7 +1199,7 @@
<mkdir dir="${build.test.dir}/cassandra"/>
<mkdir dir="${build.test.dir}/output"/>
<mkdir dir="${build.test.dir}/output/@{testtag}"/>
<junit-timeout fork="on" forkmode="@{forkmode}" failureproperty="testfailed" maxmemory="1024m" timeout="@{timeout}">
<junit-timeout fork="on" forkmode="@{forkmode}" failureproperty="testfailed" maxmemory="1024m" timeout="@{timeout}" showoutput="@{showoutput}">
<formatter classname="org.apache.cassandra.CassandraXMLJUnitResultFormatter" extension=".xml" usefile="true"/>
<formatter classname="org.apache.cassandra.CassandraBriefJUnitResultFormatter" usefile="false"/>
<jvmarg value="-Dstorage-config=${test.conf}"/>
@ -1365,7 +1358,7 @@
-->
<target name="testsome" depends="build-test" description="Execute specific unit tests" >
<testmacro inputdir="${test.unit.src}" timeout="${test.timeout}">
<test name="${test.name}" methods="${test.methods}"/>
<test name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/>
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
<jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
@ -1607,6 +1600,7 @@
<attribute name="inputdir" />
<attribute name="timeout" default="${test.timeout}" />
<attribute name="forkmode" default="perTest"/>
<attribute name="showoutput" default="true"/>
<element name="optjvmargs" implicit="true" optional="true" />
<attribute name="filter" default="**/${test.name}.java"/>
<attribute name="exclude" default="" />
@ -1618,7 +1612,7 @@
<testmacrohelper inputdir="@{inputdir}" timeout="@{timeout}"
forkmode="@{forkmode}" filter="@{filter}"
exclude="@{exclude}" filelist="@{filelist}" poffset="@{poffset}"
testtag="@{testtag}" >
testtag="@{testtag}" showoutput="false" >
<optjvmargs/>
</testmacrohelper>
<junitreport todir="${build.test.dir}">
@ -1764,6 +1758,46 @@
<testparallel testdelegate="testlist"/>
</target>
<!-- In-JVM dtest targets -->
<target name="list-jvm-dtests" depends="build-test">
<java classname="org.apache.cassandra.distributed.test.TestLocator" fork="no">
<classpath>
<path refid="cassandra.classpath" />
<pathelement location="${test.classes}"/>
<pathelement location="${test.conf}"/>
<fileset dir="${test.lib}">
<include name="**/*.jar" />
</fileset>
</classpath>
<arg value="${test.distributed.listfile}"/>
</java>
</target>
<target name="test-jvm-dtest-forking" depends="list-jvm-dtests" description="Execute In-JVM 'distributed' tests" >
<chmod file="${test.distributed.listfile}" perm="+x"/>
<exec executable="./${test.distributed.listfile}" failonerror="true"/>
<delete file="${test.distributed.listfile}"/>
</target>
<target name="dtest-jar" depends="build-test, build" description="Create dtest-compatible jar, including all dependencies">
<jar jarfile="${build.dir}/dtest-${base.version}.jar">
<zipgroupfileset dir="${build.lib}" includes="*.jar" excludes="META-INF/*.SF"/>
<fileset dir="${build.classes.main}"/>
<fileset dir="${test.classes}"/>
<fileset dir="${test.conf}" />
</jar>
</target>
<target name="test-jvm-dtest" depends="build-test" description="Execute unit tests">
<testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/test/*Test.java">
<jvmarg value="-Dlogback.configurationFile=test/conf/logback-dtest.xml"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
</testmacro>
</target>
<!-- run microbenchmarks suite -->
<target name="microbench" depends="build-test">
<java classname="org.openjdk.jmh.Main"

View File

@ -1127,7 +1127,7 @@ public abstract class LegacyLayout
throw new IllegalStateException(String.format("Got cell for unknown column %s in sstable of %s.%s: " +
"This suggest a problem with the schema which doesn't list " +
"this column. Even if that column was dropped, it should have " +
"been listed as such", UTF8Type.instance.compose(e.columnName), metadata.ksName, metadata.cfName), e);
"been listed as such", metadata.ksName, metadata.cfName, UTF8Type.instance.compose(e.columnName)), e);
throw e;
}

View File

@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.api;
import org.apache.cassandra.locator.InetAddressAndPort;
import java.util.UUID;
import java.util.concurrent.Future;
// The cross-version API requires that an Instance has a constructor signature of (IInstanceConfig, ClassLoader)
public interface IInstance extends IIsolatedExecutor
@ -36,7 +37,7 @@ public interface IInstance extends IIsolatedExecutor
UUID schemaVersion();
void startup();
void shutdown();
Future<Void> shutdown();
// these methods are not for external use, but for simplicity we leave them public and on the normal IInstance interface
void startup(ICluster cluster);

View File

@ -20,7 +20,6 @@ package org.apache.cassandra.distributed.api;
import java.io.Serializable;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
@ -52,7 +51,7 @@ public interface IIsolatedExecutor
}
public interface SerializableTriFunction<I1, I2, I3, O> extends Serializable, TriFunction<I1, I2, I3, O> { }
void shutdown();
Future<Void> shutdown();
/**
* Convert the execution to one performed asynchronously on the IsolatedExecutor, returning a Future of the execution result

View File

@ -30,6 +30,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@ -143,13 +146,14 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster,
}
@Override
public synchronized void shutdown()
public synchronized Future<Void> shutdown()
{
if (isShutdown)
throw new IllegalStateException();
isShutdown = true;
delegate.shutdown();
Future<Void> future = delegate.shutdown();
delegate = null;
return future;
}
@Override
@ -238,12 +242,14 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster,
public void schemaChange(String query)
{
try (SchemaChangeMonitor monitor = new SchemaChangeMonitor())
{
// execute the schema change
coordinator(1).execute(query, ConsistencyLevel.ALL);
monitor.waitForAgreement();
}
get(1).sync(() -> {
try (SchemaChangeMonitor monitor = new SchemaChangeMonitor())
{
// execute the schema change
coordinator(1).execute(query, ConsistencyLevel.ALL);
monitor.waitForAgreement();
}
}).run();
}
/**
@ -375,13 +381,17 @@ public abstract class AbstractCluster<I extends IInstance> implements ICluster,
@Override
public void close()
{
parallelForEach(IInstance::shutdown, 1L, TimeUnit.MINUTES);
FBUtilities.waitOnFutures(instances.stream()
.map(IInstance::shutdown)
.collect(Collectors.toList()),
1L, TimeUnit.MINUTES);
instances.clear();
instanceMap.clear();
// Make sure to only delete directory when threads are stopped
FileUtils.deleteRecursive(root);
//withThreadLeakCheck(futures);
System.gc();
}
// We do not want this check to run every time until we fix problems with tread stops

View File

@ -92,9 +92,9 @@ public abstract class DelegatingInvokableInstance implements IInvokableInstance
}
@Override
public void shutdown()
public Future<Void> shutdown()
{
delegate().shutdown();
return delegate().shutdown();
}
@Override

View File

@ -27,6 +27,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.function.BiConsumer;
@ -35,6 +37,7 @@ import java.util.function.Function;
import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.LoggerContext;
import com.codahale.metrics.MetricFilter;
import org.apache.cassandra.batchlog.BatchlogManager;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.concurrent.SharedExecutorPool;
@ -68,6 +71,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.net.IMessageSink;
import org.apache.cassandra.net.MessageDeliveryTask;
import org.apache.cassandra.net.MessageIn;
@ -356,9 +360,9 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
}
}
public void shutdown()
public Future<Void> shutdown()
{
sync((ExecutorService executor) -> {
Future<?> future = async((ExecutorService executor) -> {
Throwable error = null;
error = parallelRun(error, executor,
Gossiper.instance::stop,
@ -385,11 +389,14 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
StageManager::shutdownAndWait,
SharedExecutorPool.SHARED::shutdown
);
LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
loggerContext.stop();
super.shutdown();
Throwables.maybeFail(error);
}).accept(isolatedExecutor);
}).apply(isolatedExecutor);
return CompletableFuture.runAsync(ThrowingRunnable.toRunnable(future::get), isolatedExecutor)
.thenRun(super::shutdown);
}
private static Throwable parallelRun(Throwable accumulate, ExecutorService runOn, ThrowingRunnable ... runnables)
@ -424,9 +431,4 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
}
return accumulate;
}
public static interface ThrowingRunnable
{
public void run() throws Throwable;
}
}

View File

@ -26,10 +26,13 @@ import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URLClassLoader;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Consumer;
@ -37,7 +40,6 @@ import java.util.function.Function;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.utils.Throwables;
public class IsolatedExecutor implements IIsolatedExecutor
{
@ -52,9 +54,12 @@ public class IsolatedExecutor implements IIsolatedExecutor
this.deserializeOnInstance = lookupDeserializeOneObject(classLoader);
}
public void shutdown()
public Future<Void> shutdown()
{
isolatedExecutor.shutdown();
ThrowingRunnable.toRunnable(((URLClassLoader) classLoader)::close).run();
return CompletableFuture.runAsync(ThrowingRunnable.toRunnable(() -> isolatedExecutor.awaitTermination(60, TimeUnit.SECONDS)),
Executors.newSingleThreadExecutor());
}
public <O> CallableNoExcept<Future<O>> async(CallableNoExcept<O> call) { return () -> isolatedExecutor.submit(call); }
@ -162,4 +167,22 @@ public class IsolatedExecutor implements IIsolatedExecutor
}
}
public interface ThrowingRunnable
{
public void run() throws Throwable;
public static Runnable toRunnable(ThrowingRunnable runnable)
{
return () -> {
try
{
runnable.run();
}
catch (Throwable throwable)
{
throw new RuntimeException(throwable);
}
};
}
}
}

View File

@ -20,6 +20,8 @@ package org.apache.cassandra.distributed.test;
import java.util.Arrays;
import com.google.common.collect.Iterators;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -27,6 +29,13 @@ import org.apache.cassandra.distributed.impl.AbstractCluster;
public class DistributedTestBase
{
@After
public void afterEach()
{
System.runFinalization();
System.gc();
}
public static String KEYSPACE = "distributed_test_keyspace";
@BeforeClass

View File

@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import org.junit.Test;
public class TestLocator
{
private static final String defaultOutputFileName = "run-jvm-dtests";
private static final String testPackage = "org.apache.cassandra.distributed.test";
private static final String testCommandFormat = "ant testsome -Dtest.name=%s -Dtest.methods=%s";
public static void main(String[] args) throws Throwable
{
String outputFileName = defaultOutputFileName;
if (args.length == 1)
{
outputFileName = args[0];
}
try (FileWriter fileWriter = new FileWriter(outputFileName);
PrintWriter printWriter = new PrintWriter(fileWriter))
{
printWriter.println("#!/bin/bash");
for (Class testClass : locateClasses(testPackage))
{
for (Method method : testClass.getMethods())
{
if (method.getAnnotation(Test.class) == null)
continue;
printWriter.println(String.format(testCommandFormat,
testClass.getName(),
method.getName()));
}
}
}
}
private static List<Class> locateClasses(String packageName) throws ClassNotFoundException, IOException
{
ClassLoader classLoader = TestLocator.class.getClassLoader();
Enumeration<URL> resources = classLoader.getResources(packageName.replace('.', '/'));
List<Class> classes = new ArrayList<>();
while (resources.hasMoreElements())
{
URL resource = resources.nextElement();
loadClassesRecursively(new File(resource.getFile()), packageName, classes);
}
return classes;
}
private static void loadClassesRecursively(File directory, String packageName, List<Class> classes) throws ClassNotFoundException
{
for (File file : directory.listFiles())
{
if (file.isDirectory())
loadClassesRecursively(file, packageName + "." + file.getName(), classes);
else if (file.getName().endsWith(".class"))
{
classes.add(Class.forName(packageName + '.' + file.getName().substring(0, file.getName().length() - 6)));
}
}
}
}