natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-4.1' into trunk

This commit is contained in:
Benedict Elliott Smith 2022-07-12 12:29:12 +01:00
parent 6fcaed8fd6 067121da63
commit 8468d33a31
20 changed files with 607 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
.circleci/config-2_1.yml
View File

@ -200,6 +200,10 @@ j8_with_dtests_jobs: &j8_with_dtests_jobs
requires:
- start_j8_jvm_dtests
- j8_build
- j8_simulator_dtests:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
@ -381,6 +385,9 @@ j8_pre-commit_jobs: &j8_pre-commit_jobs
- j8_unit_tests:
requires:
- j8_build
- j8_simulator_dtests:
requires:
- j8_build
- j8_jvm_dtests:
requires:
- j8_build
@ -738,6 +745,15 @@ jobs:
- log_environment
- run_parallel_junit_tests
j8_simulator_dtests:
<<: *j8_small_executor
steps:
- attach_workspace:
at: /home/cassandra
- create_junit_containers
- log_environment
- run_simulator_tests
j8_jvm_dtests:
<<: *j8_small_par_executor
steps:
@ -1312,6 +1328,33 @@ commands:
no_output_timeout: 15m
run_simulator_tests:
parameters:
no_output_timeout:
type: string
default: 30m
steps:
- run:
name: Run Simulator Tests
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant test-simulator-dtest
no_output_timeout: <<parameters.no_output_timeout>>
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
run_junit_tests:
parameters:
target:

113
.circleci/config.yml
View File

@ -1152,6 +1152,112 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_simulator_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Simulator Tests
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant test-simulator-dtest
no_output_timeout: 30m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_VNODES: false
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -3772,6 +3878,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_simulator_dtests:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
@ -3938,6 +4048,9 @@ workflows:
- j8_unit_tests:
requires:
- j8_build
- j8_simulator_dtests:
requires:
- j8_build
- j8_jvm_dtests:
requires:
- j8_build

113
.circleci/config.yml.HIGHRES
View File

@ -1152,6 +1152,112 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_simulator_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Simulator Tests
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant test-simulator-dtest
no_output_timeout: 30m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_VNODES: false
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -3772,6 +3878,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_simulator_dtests:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
@ -3938,6 +4048,9 @@ workflows:
- j8_unit_tests:
requires:
- j8_build
- j8_simulator_dtests:
requires:
- j8_build
- j8_jvm_dtests:
requires:
- j8_build

113
.circleci/config.yml.LOWRES
View File

@ -1152,6 +1152,112 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_simulator_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Simulator Tests
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant test-simulator-dtest
no_output_timeout: 30m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_VNODES: false
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -3772,6 +3878,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_simulator_dtests:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
@ -3938,6 +4048,9 @@ workflows:
- j8_unit_tests:
requires:
- j8_build
- j8_simulator_dtests:
requires:
- j8_build
- j8_jvm_dtests:
requires:
- j8_build

113
.circleci/config.yml.MIDRES
View File

@ -1152,6 +1152,112 @@ jobs:
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- CASSANDRA_USE_JDK11: true
j8_simulator_dtests:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
resource_class: medium
working_directory: ~/
shell: /bin/bash -eo pipefail -l
parallelism: 1
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine unit Tests to Run
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | sed "s;^/home/cassandra/cassandra/test/unit/;;g" | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
no_output_timeout: 15m
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
echo '*** java ***'
which java
java -version
- run:
name: Run Simulator Tests
command: |
set -x
export PATH=$JAVA_HOME/bin:$PATH
time mv ~/cassandra /tmp
cd /tmp/cassandra
if [ -d ~/dtest_jars ]; then
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
fi
ant test-simulator-dtest
no_output_timeout: 30m
- store_test_results:
path: /tmp/cassandra/build/test/output/
- store_artifacts:
path: /tmp/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: /tmp/cassandra/build/test/logs
destination: logs
environment:
- ANT_HOME: /usr/share/ant
- JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- LANG: en_US.UTF-8
- KEEP_TEST_DIR: true
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
- PYTHONIOENCODING: utf-8
- PYTHONUNBUFFERED: true
- CASS_DRIVER_NO_EXTENSIONS: true
- CASS_DRIVER_NO_CYTHON: true
- CASSANDRA_SKIP_SYNC: true
- DTEST_REPO: https://github.com/apache/cassandra-dtest.git
- DTEST_BRANCH: trunk
- CCM_MAX_HEAP_SIZE: 1024M
- CCM_HEAP_NEWSIZE: 256M
- REPEATED_UTEST_TARGET: testsome
- REPEATED_UTEST_CLASS: null
- REPEATED_UTEST_METHODS: null
- REPEATED_UTEST_VNODES: false
- REPEATED_UTEST_COUNT: 100
- REPEATED_UTEST_STOP_ON_FAILURE: false
- REPEATED_DTEST_NAME: null
- REPEATED_DTEST_VNODES: false
- REPEATED_DTEST_COUNT: 100
- REPEATED_DTEST_STOP_ON_FAILURE: false
- REPEATED_UPGRADE_DTEST_NAME: null
- REPEATED_UPGRADE_DTEST_COUNT: 100
- REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE: false
- REPEATED_JVM_UPGRADE_DTEST_CLASS: null
- REPEATED_JVM_UPGRADE_DTEST_METHODS: null
- REPEATED_JVM_UPGRADE_DTEST_COUNT: 100
- REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE: false
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_cqlsh-dtests-py3-with-vnodes:
docker:
- image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest
@ -3772,6 +3878,10 @@ workflows:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_simulator_dtests:
requires:
- start_j8_jvm_dtests
- j8_build
- j8_jvm_dtests_vnode:
requires:
- start_j8_jvm_dtests
@ -3938,6 +4048,9 @@ workflows:
- j8_unit_tests:
requires:
- j8_build
- j8_simulator_dtests:
requires:
- j8_build
- j8_jvm_dtests:
requires:
- j8_build

9
build.xml
View File

@ -115,6 +115,7 @@
<property name="test.long.timeout" value="600000" />
<property name="test.burn.timeout" value="60000000" />
<property name="test.distributed.timeout" value="900000" />
<property name="test.simulation.timeout" value="1800000" />
<!-- default for cql tests. Can be override by -Dcassandra.test.use_prepared=false -->
<property name="cassandra.test.use_prepared" value="true" />
@ -2022,11 +2023,16 @@
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
</testmacro>
<testmacro inputdir="${test.simulator-test.src}" timeout="${test.distributed.timeout}" forkmode="perTest" showoutput="true" filter="**/test/${test.name}.java">
</target>
<target name="test-simulator-dtest" depends="build-test" description="Execute simulator dtests">
<testmacro inputdir="${test.simulator-test.src}" timeout="${test.simulation.timeout}" forkmode="perTest" showoutput="true" filter="**/test/${test.name}.java">
<jvmarg value="-Dlogback.configurationFile=test/conf/logback-simulator.xml"/>
<jvmarg value="-Dcassandra.ring_delay_ms=10000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.skip_sync=true" />
<jvmarg value="-Dcassandra.debugrefcount=false"/>
<jvmarg value="-Dcassandra.test.simulator.determinismcheck=strict"/>
<!-- Support Simulator Tests -->
<jvmarg line="-javaagent:${test.lib}/jars/simulator-asm.jar"/>
<jvmarg line="-Xbootclasspath/a:${test.lib}/jars/simulator-bootstrap.jar"/>
@ -2034,6 +2040,7 @@
<jvmarg line="-XX:-TieredCompilation"/>
<jvmarg line="-XX:Tier4CompileThreshold=1000"/>
<jvmarg line="-XX:ReservedCodeCacheSize=256M"/>
<jvmarg line="-Xmx8G"/>
</testmacro>
</target>

13
src/java/org/apache/cassandra/concurrent/ExecutorFactory.java
View File

@ -183,14 +183,25 @@ public interface ExecutorFactory extends ExecutorBuilderFactory.Jmxable<Executor
// deliberately not volatile to ensure zero overhead outside of testing;
// depend on other memory visibility primitives to ensure visibility
private static ExecutorFactory FACTORY = new ExecutorFactory.Default(Global.class.getClassLoader(), null, JVMStabilityInspector::uncaughtException);
private static boolean modified;
public static ExecutorFactory executorFactory()
{
return FACTORY;
}
public static void unsafeSet(ExecutorFactory executorFactory)
public static synchronized void unsafeSet(ExecutorFactory executorFactory)
{
FACTORY = executorFactory;
modified = true;
}
public static synchronized boolean tryUnsafeSet(ExecutorFactory executorFactory)
{
if (modified)
return false;
unsafeSet(executorFactory);
return true;
}
}

2
src/java/org/apache/cassandra/db/memtable/AbstractAllocatorMemtable.java
View File

@ -202,7 +202,7 @@ public abstract class AbstractAllocatorMemtable extends AbstractMemtableWithComm
((AbstractAllocatorMemtable) current).flushIfPeriodExpired();
}
};
ScheduledExecutors.scheduledTasks.schedule(runnable, period, TimeUnit.MILLISECONDS);
ScheduledExecutors.scheduledTasks.scheduleSelfRecurring(runnable, period, TimeUnit.MILLISECONDS);
}
private void flushIfPeriodExpired()

15
src/java/org/apache/cassandra/io/util/ChecksumWriter.java
View File

@ -18,7 +18,9 @@
package org.apache.cassandra.io.util;
import java.io.*;
import java.io.DataOutput;
import java.io.IOError;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.zip.CRC32;
@ -27,8 +29,6 @@ import javax.annotation.Nonnull;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.File;
public class ChecksumWriter
{
private final CRC32 incrementalChecksum = new CRC32();
@ -90,12 +90,11 @@ public class ChecksumWriter
public void writeFullChecksum(@Nonnull File digestFile)
{
try (FileOutputStream fos = new FileOutputStream(digestFile.toJavaIOFile());
DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos)))
try (FileOutputStreamPlus fos = new FileOutputStreamPlus(digestFile))
{
out.write(String.valueOf(fullChecksum.getValue()).getBytes(StandardCharsets.UTF_8));
out.flush();
fos.getFD().sync();
fos.write(String.valueOf(fullChecksum.getValue()).getBytes(StandardCharsets.UTF_8));
fos.flush();
fos.getChannel().force(true);
}
catch (IOException e)
{

2
src/java/org/apache/cassandra/service/paxos/PaxosPrepare.java
View File

@ -596,7 +596,7 @@ public class PaxosPrepare extends PaxosRequestCallback<PaxosPrepare.Response> im
// presence of an incomplete proposal can be ignored, as either the proposal is a re-proposal of the same
// commit or the commit has already reached a quorum
else if (hasInProgressProposal())
signalDone(FOUND_INCOMPLETE_ACCEPTED);
signalDone(hasOnlyPromises ? FOUND_INCOMPLETE_ACCEPTED : SUPERSEDED);
else if (withLatest() >= participants.sizeOfConsensusQuorum)
signalDone(hasOnlyPromises ? PROMISED : READ_PERMITTED);

2
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -574,7 +574,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
{
// org.apache.cassandra.distributed.impl.AbstractCluster.startup sets the exception handler for the thread
// so extract it to populate ExecutorFactory.Global
ExecutorFactory.Global.unsafeSet(new ExecutorFactory.Default(Thread.currentThread().getContextClassLoader(), null, Thread.getDefaultUncaughtExceptionHandler()));
ExecutorFactory.Global.tryUnsafeSet(new ExecutorFactory.Default(Thread.currentThread().getContextClassLoader(), null, Thread.getDefaultUncaughtExceptionHandler()));
if (config.has(GOSSIP))
{
// TODO: hacky

38
test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActionListener.java
View File

@ -34,4 +34,42 @@ public interface ClusterActionListener
TopologyChangeValidator newTopologyChangeValidator(Object id);
RepairValidator newRepairValidator(Object id);
class NoOpListener implements ClusterActionListener
{
@Override
public TopologyChangeValidator newTopologyChangeValidator(Object id)
{
return new TopologyChangeValidator()
{
@Override
public void before(Topology before, int[] participatingKeys)
{
}
@Override
public void after(Topology after)
{
}
};
}
@Override
public RepairValidator newRepairValidator(Object id)
{
return new RepairValidator()
{
@Override
public void before(Topology topology, boolean repairPaxos, boolean repairOnlyPaxos)
{
}
@Override
public void after()
{
}
};
}
}
}

12
test/simulator/main/org/apache/cassandra/simulator/cluster/ClusterActions.java
View File

@ -28,6 +28,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,6 +62,7 @@ import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChan
import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange.LEAVE;
import static org.apache.cassandra.simulator.cluster.ClusterActions.TopologyChange.REPLACE;
import static org.apache.cassandra.simulator.systems.NonInterceptible.Permit.REQUIRED;
import static org.apache.cassandra.simulator.utils.KindOfSequence.UNIFORM;
// TODO (feature): add Gossip failures (up to some acceptable number)
@ -123,6 +125,12 @@ public class ClusterActions extends SimulatedSystems
this.changePaxosVariantTo = changePaxosVariantTo;
}
public static Options noActions(int clusterSize)
{
int[] rf = new int[]{clusterSize};
return new Options(0, UNIFORM.period(null, null), Choices.uniform(), rf, rf, rf, null);
}
public Options changePaxosVariantTo(PaxosVariant newVariant)
{
return new Options(this, newVariant);
@ -141,6 +149,10 @@ public class ClusterActions extends SimulatedSystems
Debug debug)
{
super(simulated);
Preconditions.checkNotNull(cluster);
Preconditions.checkNotNull(options);
Preconditions.checkNotNull(listener);
Preconditions.checkNotNull(debug);
this.cluster = cluster;
this.options = options;
this.listener = listener;

9
test/simulator/main/org/apache/cassandra/simulator/paxos/Ballots.java
View File

@ -72,14 +72,16 @@ public class Ballots
public static class LatestBallots
{
public final long promise;
public final long accept;
public final long accept; // the ballot actually accepted
public final long acceptOf; // the original ballot (i.e. if a reproposal accept != acceptOf)
public final long commit;
public final long persisted;
public LatestBallots(long promise, long accept, long commit, long persisted)
public LatestBallots(long promise, long accept, long acceptOf, long commit, long persisted)
{
this.promise = promise;
this.accept = accept;
this.acceptOf = acceptOf;
this.commit = commit;
this.persisted = persisted;
}
@ -111,7 +113,8 @@ public class Ballots
long baseTable = latestBallotFromBaseTable(key, metadata);
return new LatestBallots(
promised.unixMicros(),
accepted == null || accepted.update.isEmpty() ? 0L : latestBallot(accepted.update.iterator()),
accepted == null || accepted.update.isEmpty() ? 0L : accepted.ballot.unixMicros(),
accepted == null || accepted.update.isEmpty() ? 0L : accepted.update.stats().minTimestamp,
latestBallot(committed.update.iterator()),
baseTable
);

3
test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosRepairValidator.java
View File

@ -81,8 +81,9 @@ public class PaxosRepairValidator implements RepairValidator
long committedBefore = stream(before).mapToLong(Ballots.LatestBallots::permanent).max().orElse(0L);
// anything accepted by a quorum should be persisted
long acceptedBefore = stream(before).mapToLong(n -> n.accept).max().orElse(0L);
long acceptedOfBefore = stream(before).filter(n -> n.accept == acceptedBefore).mapToLong(n -> n.acceptOf).findAny().orElse(0L);
int countAccepted = (int) stream(before).filter(n -> n.accept == acceptedBefore).count();
expectPersisted = countAccepted >= quorum ? acceptedBefore : committedBefore;
expectPersisted = countAccepted >= quorum ? acceptedOfBefore : committedBefore;
kind = countAccepted >= quorum ? "agreed" : "committed";
}
else

3
test/simulator/main/org/apache/cassandra/simulator/paxos/PaxosTopologyChangeVerifier.java
View File

@ -82,10 +82,11 @@ public class PaxosTopologyChangeVerifier implements TopologyChangeValidator
// note that we will not always witness something newer than the latest accepted proposal,
// because if we don't witness it during repair, we will simply invalidate it with the low bound
long acceptedBefore = stream(before).mapToLong(n -> n.accept).max().orElse(0L);
long acceptedOfBefore = stream(before).filter(n -> n.accept == acceptedBefore).mapToLong(n -> n.acceptOf).findAny().orElse(0L);
int countBefore = (int) stream(before).filter(n -> n.accept == acceptedBefore).count();
int countAfter = countBefore < quorumAfter
? (int) stream(after).filter(n -> n.any() >= acceptedBefore).count()
: (int) stream(after).filter(n -> n.permanent() >= acceptedBefore).count();
: (int) stream(after).filter(n -> n.permanent() >= acceptedOfBefore).count();
if (countBefore >= quorumBefore && countAfter < quorumAfter)
{

4
test/simulator/main/org/apache/cassandra/simulator/utils/KindOfSequence.java
View File

@ -344,8 +344,6 @@ public enum KindOfSequence
}
}
public LinkLatency linkLatency(int nodes, LongRange nanos, RandomSource random)
{
switch (this)
@ -361,7 +359,7 @@ public enum KindOfSequence
{
switch (this)
{
default:throw new AssertionError();
default: throw new AssertionError();
case UNIFORM: return new UniformPeriod(nanos);
case UNIFORM_STEP: return new UniformStepPeriod(nanos, random);
case RANDOMWALK: return new RandomWalkPeriod(nanos, random);

4
test/simulator/test/org/apache/cassandra/simulator/test/ClassWithSynchronizedMethods.java
View File

@ -23,7 +23,11 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.cassandra.utils.Nemesis;
import org.apache.cassandra.utils.Simulate;
import static org.apache.cassandra.utils.Simulate.With.MONITORS;
@Simulate(with = MONITORS)
public class ClassWithSynchronizedMethods
{
@Nemesis

14
test/simulator/test/org/apache/cassandra/simulator/test/ShortPaxosSimulationTest.java
View File

@ -33,10 +33,12 @@ public class ShortPaxosSimulationTest
{
PaxosSimulationRunner.main(new String[] { "run", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30" });
}
@Test
public void selfReconcileTest() throws IOException
{
PaxosSimulationRunner.main(new String[] { "reconcile", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30", "--with-self" });
}
//
// fails due to OOM DirectMemory - unclear why
//
// @Test
// public void selfReconcileTest() throws IOException
// {
// PaxosSimulationRunner.main(new String[] { "reconcile", "-n", "3..6", "-t", "1000", "-c", "2", "--cluster-action-limit", "2", "-s", "30", "--with-self" });
// }
}

10
test/simulator/test/org/apache/cassandra/simulator/test/TrivialSimulationTest.java
View File

@ -19,6 +19,8 @@
package org.apache.cassandra.simulator.test;
import java.io.IOException;
import java.util.Collections;
import java.util.EnumMap;
import java.util.IdentityHashMap;
import org.junit.Test;
@ -27,10 +29,14 @@ import org.apache.cassandra.concurrent.ExecutorFactory;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.simulator.ActionList;
import org.apache.cassandra.simulator.Debug;
import org.apache.cassandra.simulator.cluster.ClusterActionListener.NoOpListener;
import org.apache.cassandra.simulator.cluster.ClusterActions;
import org.apache.cassandra.simulator.cluster.ClusterActions.Options;
import org.apache.cassandra.utils.concurrent.CountDownLatch;
import static org.apache.cassandra.simulator.cluster.ClusterActions.InitialConfiguration.initializeAll;
import static org.apache.cassandra.simulator.cluster.ClusterActions.Options.noActions;
public class TrivialSimulationTest extends SimulationTestBase
{
@ -38,7 +44,9 @@ public class TrivialSimulationTest extends SimulationTestBase
public void trivialTest() throws IOException // for demonstration/experiment purposes
{
simulate((simulation) -> {
ClusterActions clusterActions = new ClusterActions(simulation.simulated, simulation.cluster, null,null, null);
Options options = noActions(simulation.cluster.size());
ClusterActions clusterActions = new ClusterActions(simulation.simulated, simulation.cluster,
options, new NoOpListener(), new Debug(new EnumMap<>(Debug.Info.class), new int[0]));
return ActionList.of(clusterActions.initializeCluster(initializeAll(simulation.cluster.size())),
simulation.schemaChange(1, "CREATE KEYSPACE ks WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor' : 3}"),
simulation.schemaChange(1, "CREATE TABLE IF NOT EXISTS ks.tbl (pk int PRIMARY KEY, v int)"));