mirror of https://github.com/apache/cassandra
Merge branch 'cassandra-3.0' into cassandra-3.11
This commit is contained in:
Marcus Eriksson
commit
3bb388b364
|
|
@ -61,6 +61,16 @@ with_dtests_jobs: &with_dtest_jobs
|
|||
- utests_stress:
|
||||
requires:
|
||||
- start_utests_stress
|
||||
- start_jvm_upgrade_dtest:
|
||||
type: approval
|
||||
requires:
|
||||
- build
|
||||
- dtest_jars_build:
|
||||
requires:
|
||||
- start_jvm_upgrade_dtest
|
||||
- j8_jvm_upgrade_dtests:
|
||||
requires:
|
||||
- dtest_jars_build
|
||||
# Java 8 dtests (on request)
|
||||
- start_j8_dtests:
|
||||
type: approval
|
||||
|
|
@ -127,6 +137,18 @@ jobs:
|
|||
- cassandra
|
||||
- .m2
|
||||
|
||||
dtest_jars_build:
|
||||
executor: java8-executor
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- build_cassandra_dtest_jars
|
||||
- persist_to_workspace:
|
||||
root: /home/cassandra
|
||||
paths:
|
||||
- dtest_jars
|
||||
|
||||
j8_unit_tests:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
|
|
@ -144,6 +166,14 @@ jobs:
|
|||
- run_junit_tests:
|
||||
target: test-jvm-dtest-forking
|
||||
|
||||
j8_jvm_upgrade_dtests:
|
||||
<<: *j8_seq_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run_junit_tests:
|
||||
target: test-jvm-upgrade-dtest-forking
|
||||
|
||||
utests_long:
|
||||
<<: *j8_seq_executor
|
||||
steps:
|
||||
|
|
@ -277,6 +307,47 @@ commands:
|
|||
fi
|
||||
no_output_timeout: 15m
|
||||
|
||||
build_cassandra_dtest_jars:
|
||||
steps:
|
||||
- run:
|
||||
name: Build Cassandra DTest jars
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
cd ~/cassandra
|
||||
git remote add apache git://github.com/apache/cassandra.git
|
||||
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
|
||||
# check out the correct cassandra version:
|
||||
git remote set-branches --add apache '$branch'
|
||||
git fetch --depth 1 apache $branch
|
||||
git checkout $branch
|
||||
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Exit, if we didn't build successfully
|
||||
if [ "${RETURN}" -ne "0" ]; then
|
||||
echo "Build failed with exit code: ${RETURN}"
|
||||
exit ${RETURN}
|
||||
fi
|
||||
done
|
||||
# and build the dtest-jar for the branch under test
|
||||
git checkout origin/$CIRCLE_BRANCH
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
mkdir ~/dtest_jars
|
||||
cp build/dtest*.jar ~/dtest_jars
|
||||
ls -l ~/dtest_jars
|
||||
no_output_timeout: 15m
|
||||
|
||||
run_eclipse_warnings:
|
||||
steps:
|
||||
- run:
|
||||
|
|
@ -325,6 +396,9 @@ commands:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean <<parameters.target>>
|
||||
no_output_timeout: <<parameters.no_output_timeout>>
|
||||
- store_test_results:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,51 @@
|
|||
version: 2
|
||||
jobs:
|
||||
j8_jvm_upgrade_dtests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-upgrade-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -109,37 +155,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
|
||||
\ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
|
||||
\ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
|
||||
\ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
|
||||
\ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
|
||||
\ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
|
||||
\ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
|
||||
INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -280,6 +300,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean stress-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -419,36 +442,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
|
||||
\ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
|
||||
if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
|
||||
else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
|
||||
\ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
|
||||
\ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
|
||||
\ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
|
||||
\ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
|
||||
\ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -490,6 +488,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -533,6 +534,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean long-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -644,6 +648,75 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
dtest_jars_build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Build Cassandra DTest jars
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
cd ~/cassandra
|
||||
git remote add apache git://github.com/apache/cassandra.git
|
||||
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
|
||||
# check out the correct cassandra version:
|
||||
git remote set-branches --add apache '$branch'
|
||||
git fetch --depth 1 apache $branch
|
||||
git checkout $branch
|
||||
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Exit, if we didn't build successfully
|
||||
if [ "${RETURN}" -ne "0" ]; then
|
||||
echo "Build failed with exit code: ${RETURN}"
|
||||
exit ${RETURN}
|
||||
fi
|
||||
done
|
||||
# and build the dtest-jar for the branch under test
|
||||
git checkout origin/$CIRCLE_BRANCH
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
mkdir ~/dtest_jars
|
||||
cp build/dtest*.jar ~/dtest_jars
|
||||
ls -l ~/dtest_jars
|
||||
no_output_timeout: 15m
|
||||
- persist_to_workspace:
|
||||
root: /home/cassandra
|
||||
paths:
|
||||
- dtest_jars
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
workflows:
|
||||
version: 2
|
||||
build_and_run_tests:
|
||||
|
|
@ -676,6 +749,16 @@ workflows:
|
|||
- utests_stress:
|
||||
requires:
|
||||
- start_utests_stress
|
||||
- start_jvm_upgrade_dtest:
|
||||
type: approval
|
||||
requires:
|
||||
- build
|
||||
- dtest_jars_build:
|
||||
requires:
|
||||
- start_jvm_upgrade_dtest
|
||||
- j8_jvm_upgrade_dtests:
|
||||
requires:
|
||||
- dtest_jars_build
|
||||
- start_j8_dtests:
|
||||
type: approval
|
||||
requires:
|
||||
|
|
@ -693,468 +776,3 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
|
||||
# Original config.yml file:
|
||||
# version: 2.1
|
||||
#
|
||||
# default_env_vars: &default_env_vars
|
||||
# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# ANT_HOME: /usr/share/ant
|
||||
# LANG: en_US.UTF-8
|
||||
# KEEP_TEST_DIR: true
|
||||
# DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
# PYTHONIOENCODING: utf-8
|
||||
# PYTHONUNBUFFERED: true
|
||||
# CASS_DRIVER_NO_EXTENSIONS: true
|
||||
# CASS_DRIVER_NO_CYTHON: true
|
||||
# #Skip all syncing to disk to avoid performance issues in flaky CI environments
|
||||
# CASSANDRA_SKIP_SYNC: true
|
||||
# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
# DTEST_BRANCH: master
|
||||
# CCM_MAX_HEAP_SIZE: 1024M
|
||||
# CCM_HEAP_NEWSIZE: 256M
|
||||
#
|
||||
# j8_par_executor: &j8_par_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# #exec_resource_class: xlarge
|
||||
# parallelism: 4
|
||||
#
|
||||
# j8_seq_executor: &j8_seq_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# #exec_resource_class: xlarge
|
||||
# parallelism: 1 # sequential, single container tests: no parallelism benefits
|
||||
#
|
||||
# with_dtests_jobs: &with_dtest_jobs
|
||||
# jobs:
|
||||
# - build
|
||||
# # Java 8 unit tests will be run automatically
|
||||
# - j8_unit_tests:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_jvm_dtests:
|
||||
# requires:
|
||||
# - build
|
||||
# # specialized unit tests (all run on request using Java 8)
|
||||
# - start_utests_long:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_long:
|
||||
# requires:
|
||||
# - start_utests_long
|
||||
# - start_utests_compression:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_compression:
|
||||
# requires:
|
||||
# - start_utests_compression
|
||||
# - start_utests_stress:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_stress:
|
||||
# requires:
|
||||
# - start_utests_stress
|
||||
# # Java 8 dtests (on request)
|
||||
# - start_j8_dtests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# # Java 8 upgrade tests
|
||||
# - start_upgrade_tests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_upgradetests-no-vnodes:
|
||||
# requires:
|
||||
# - start_upgrade_tests
|
||||
#
|
||||
# with_dtest_jobs_only: &with_dtest_jobs_only
|
||||
# jobs:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
#
|
||||
# workflows:
|
||||
# version: 2
|
||||
# build_and_run_tests: *with_dtest_jobs
|
||||
# #build_and_run_tests: *with_dtest_jobs_only
|
||||
#
|
||||
# executors:
|
||||
# java8-executor:
|
||||
# parameters:
|
||||
# exec_resource_class:
|
||||
# type: string
|
||||
# default: medium
|
||||
# docker:
|
||||
# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
# resource_class: << parameters.exec_resource_class >>
|
||||
# working_directory: ~/
|
||||
# shell: /bin/bash -eo pipefail -l
|
||||
# environment:
|
||||
# <<: *default_env_vars
|
||||
# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
#
|
||||
# jobs:
|
||||
# build:
|
||||
# executor: java8-executor
|
||||
# parallelism: 1 # This job doesn't benefit from parallelism
|
||||
# steps:
|
||||
# - log_environment
|
||||
# - clone_cassandra
|
||||
# - build_cassandra
|
||||
# - run_eclipse_warnings
|
||||
# - persist_to_workspace:
|
||||
# root: /home/cassandra
|
||||
# paths:
|
||||
# - cassandra
|
||||
# - .m2
|
||||
#
|
||||
# j8_unit_tests:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests
|
||||
#
|
||||
# j8_jvm_dtests:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: test-jvm-dtest-forking
|
||||
#
|
||||
# utests_long:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: long-test
|
||||
#
|
||||
# utests_compression:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests:
|
||||
# target: testclasslist-compression
|
||||
#
|
||||
# utests_stress:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: stress-test
|
||||
#
|
||||
# j8_dtests-with-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_with_vnodes
|
||||
# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_with_vnodes
|
||||
# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_dtests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_without_vnodes
|
||||
# run_dtests_extra_args: '--skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_without_vnodes
|
||||
# pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_upgradetests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# run_dtests_extra_args: '--execute-upgrade-tests'
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# tests_filter_pattern: '^upgrade_tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# pytest_extra_args: '--execute-upgrade-tests'
|
||||
#
|
||||
# commands:
|
||||
# log_environment:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Log Environment Information
|
||||
# command: |
|
||||
# echo '*** id ***'
|
||||
# id
|
||||
# echo '*** cat /proc/cpuinfo ***'
|
||||
# cat /proc/cpuinfo
|
||||
# echo '*** free -m ***'
|
||||
# free -m
|
||||
# echo '*** df -m ***'
|
||||
# df -m
|
||||
# echo '*** ifconfig -a ***'
|
||||
# ifconfig -a
|
||||
# echo '*** uname -a ***'
|
||||
# uname -a
|
||||
# echo '*** mount ***'
|
||||
# mount
|
||||
# echo '*** env ***'
|
||||
# env
|
||||
# echo '*** java ***'
|
||||
# which java
|
||||
# java -version
|
||||
#
|
||||
# clone_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
|
||||
#
|
||||
# clone_dtest:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra dtest Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
#
|
||||
# build_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Build Cassandra
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
# for x in $(seq 1 3); do
|
||||
# ${ANT_HOME}/bin/ant clean jar
|
||||
# RETURN=\"$?\"
|
||||
# if [ \"${RETURN}\" -eq \"0\" ]; then
|
||||
# break
|
||||
# fi
|
||||
# done
|
||||
# # Exit, if we didn't build successfully
|
||||
# if [ \"${RETURN}\" -ne \"0\" ]; then
|
||||
# echo \"Build failed with exit code: ${RETURN}\"
|
||||
# exit ${RETURN}
|
||||
# fi
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_eclipse_warnings:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run eclipse-warnings
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# ant eclipse-warnings
|
||||
#
|
||||
# create_junit_containers:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Unit Tests to Run
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
# echo \"***java tests***\"
|
||||
#
|
||||
# # get all of our unit test filenames
|
||||
# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
|
||||
#
|
||||
# # split up the unit tests into groups based on the number of containers we have
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
|
||||
# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
#
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant clean <<parameters.target>>
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# run_parallel_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# default: testclasslist
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# create_venv:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Configure virtualenv and python Dependencies
|
||||
# command: |
|
||||
# # note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
# pip3 freeze
|
||||
#
|
||||
# create_dtest_containers:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# run_dtests_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# tests_filter_pattern:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Tests to Run (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 5m
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# cd cassandra-dtest
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
#
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
|
||||
# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
|
||||
# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
|
||||
# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
|
||||
# else
|
||||
# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
|
||||
# fi
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# run_dtests:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# pytest_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run dtests (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 15m
|
||||
# command: |
|
||||
# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# java -version
|
||||
# cd ~/cassandra-dtest
|
||||
# mkdir -p /tmp/dtest
|
||||
#
|
||||
# echo \"env: $(env)\"
|
||||
# echo \"** done env\"
|
||||
# mkdir -p /tmp/results/dtests
|
||||
# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
|
||||
# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
# - store_test_results:
|
||||
# path: /tmp/results
|
||||
# - store_artifacts:
|
||||
# path: /tmp/dtest
|
||||
# destination: dtest_<<parameters.file_tag>>
|
||||
# - store_artifacts:
|
||||
# path: ~/cassandra-dtest/logs
|
||||
# destination: dtest_<<parameters.file_tag>>_logs
|
||||
|
|
@ -1,5 +1,51 @@
|
|||
version: 2
|
||||
jobs:
|
||||
j8_jvm_upgrade_dtests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-upgrade-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 2048M
|
||||
- CCM_HEAP_NEWSIZE: 512M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -109,37 +155,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
|
||||
\ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
|
||||
\ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
|
||||
\ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
|
||||
\ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
|
||||
\ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
|
||||
\ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
|
||||
INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -280,6 +300,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean stress-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -419,36 +442,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
|
||||
\ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
|
||||
if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
|
||||
else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
|
||||
\ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
|
||||
\ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
|
||||
\ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
|
||||
\ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
|
||||
\ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -490,6 +488,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -533,6 +534,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean long-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -644,6 +648,75 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 512M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
dtest_jars_build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Build Cassandra DTest jars
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
cd ~/cassandra
|
||||
git remote add apache git://github.com/apache/cassandra.git
|
||||
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
|
||||
# check out the correct cassandra version:
|
||||
git remote set-branches --add apache '$branch'
|
||||
git fetch --depth 1 apache $branch
|
||||
git checkout $branch
|
||||
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Exit, if we didn't build successfully
|
||||
if [ "${RETURN}" -ne "0" ]; then
|
||||
echo "Build failed with exit code: ${RETURN}"
|
||||
exit ${RETURN}
|
||||
fi
|
||||
done
|
||||
# and build the dtest-jar for the branch under test
|
||||
git checkout origin/$CIRCLE_BRANCH
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
mkdir ~/dtest_jars
|
||||
cp build/dtest*.jar ~/dtest_jars
|
||||
ls -l ~/dtest_jars
|
||||
no_output_timeout: 15m
|
||||
- persist_to_workspace:
|
||||
root: /home/cassandra
|
||||
paths:
|
||||
- dtest_jars
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 2048M
|
||||
- CCM_HEAP_NEWSIZE: 512M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
workflows:
|
||||
version: 2
|
||||
build_and_run_tests:
|
||||
|
|
@ -676,6 +749,16 @@ workflows:
|
|||
- utests_stress:
|
||||
requires:
|
||||
- start_utests_stress
|
||||
- start_jvm_upgrade_dtest:
|
||||
type: approval
|
||||
requires:
|
||||
- build
|
||||
- dtest_jars_build:
|
||||
requires:
|
||||
- start_jvm_upgrade_dtest
|
||||
- j8_jvm_upgrade_dtests:
|
||||
requires:
|
||||
- dtest_jars_build
|
||||
- start_j8_dtests:
|
||||
type: approval
|
||||
requires:
|
||||
|
|
@ -693,468 +776,3 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
|
||||
# Original config.yml file:
|
||||
# version: 2.1
|
||||
#
|
||||
# default_env_vars: &default_env_vars
|
||||
# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# ANT_HOME: /usr/share/ant
|
||||
# LANG: en_US.UTF-8
|
||||
# KEEP_TEST_DIR: true
|
||||
# DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
# PYTHONIOENCODING: utf-8
|
||||
# PYTHONUNBUFFERED: true
|
||||
# CASS_DRIVER_NO_EXTENSIONS: true
|
||||
# CASS_DRIVER_NO_CYTHON: true
|
||||
# #Skip all syncing to disk to avoid performance issues in flaky CI environments
|
||||
# CASSANDRA_SKIP_SYNC: true
|
||||
# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
# DTEST_BRANCH: master
|
||||
# CCM_MAX_HEAP_SIZE: 2048M
|
||||
# CCM_HEAP_NEWSIZE: 512M
|
||||
#
|
||||
# j8_par_executor: &j8_par_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# exec_resource_class: xlarge
|
||||
# parallelism: 100
|
||||
#
|
||||
# j8_seq_executor: &j8_seq_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# exec_resource_class: xlarge
|
||||
# parallelism: 1 # sequential, single container tests: no parallelism benefits
|
||||
#
|
||||
# with_dtests_jobs: &with_dtest_jobs
|
||||
# jobs:
|
||||
# - build
|
||||
# # Java 8 unit tests will be run automatically
|
||||
# - j8_unit_tests:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_jvm_dtests:
|
||||
# requires:
|
||||
# - build
|
||||
# # specialized unit tests (all run on request using Java 8)
|
||||
# - start_utests_long:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_long:
|
||||
# requires:
|
||||
# - start_utests_long
|
||||
# - start_utests_compression:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_compression:
|
||||
# requires:
|
||||
# - start_utests_compression
|
||||
# - start_utests_stress:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_stress:
|
||||
# requires:
|
||||
# - start_utests_stress
|
||||
# # Java 8 dtests (on request)
|
||||
# - start_j8_dtests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# # Java 8 upgrade tests
|
||||
# - start_upgrade_tests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_upgradetests-no-vnodes:
|
||||
# requires:
|
||||
# - start_upgrade_tests
|
||||
#
|
||||
# with_dtest_jobs_only: &with_dtest_jobs_only
|
||||
# jobs:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
#
|
||||
# workflows:
|
||||
# version: 2
|
||||
# build_and_run_tests: *with_dtest_jobs
|
||||
# #build_and_run_tests: *with_dtest_jobs_only
|
||||
#
|
||||
# executors:
|
||||
# java8-executor:
|
||||
# parameters:
|
||||
# exec_resource_class:
|
||||
# type: string
|
||||
# default: medium
|
||||
# docker:
|
||||
# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
# resource_class: << parameters.exec_resource_class >>
|
||||
# working_directory: ~/
|
||||
# shell: /bin/bash -eo pipefail -l
|
||||
# environment:
|
||||
# <<: *default_env_vars
|
||||
# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
#
|
||||
# jobs:
|
||||
# build:
|
||||
# executor: java8-executor
|
||||
# parallelism: 1 # This job doesn't benefit from parallelism
|
||||
# steps:
|
||||
# - log_environment
|
||||
# - clone_cassandra
|
||||
# - build_cassandra
|
||||
# - run_eclipse_warnings
|
||||
# - persist_to_workspace:
|
||||
# root: /home/cassandra
|
||||
# paths:
|
||||
# - cassandra
|
||||
# - .m2
|
||||
#
|
||||
# j8_unit_tests:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests
|
||||
#
|
||||
# j8_jvm_dtests:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: test-jvm-dtest-forking
|
||||
#
|
||||
# utests_long:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: long-test
|
||||
#
|
||||
# utests_compression:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests:
|
||||
# target: testclasslist-compression
|
||||
#
|
||||
# utests_stress:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: stress-test
|
||||
#
|
||||
# j8_dtests-with-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_with_vnodes
|
||||
# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_with_vnodes
|
||||
# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_dtests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_without_vnodes
|
||||
# run_dtests_extra_args: '--skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_without_vnodes
|
||||
# pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_upgradetests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# run_dtests_extra_args: '--execute-upgrade-tests'
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# tests_filter_pattern: '^upgrade_tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# pytest_extra_args: '--execute-upgrade-tests'
|
||||
#
|
||||
# commands:
|
||||
# log_environment:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Log Environment Information
|
||||
# command: |
|
||||
# echo '*** id ***'
|
||||
# id
|
||||
# echo '*** cat /proc/cpuinfo ***'
|
||||
# cat /proc/cpuinfo
|
||||
# echo '*** free -m ***'
|
||||
# free -m
|
||||
# echo '*** df -m ***'
|
||||
# df -m
|
||||
# echo '*** ifconfig -a ***'
|
||||
# ifconfig -a
|
||||
# echo '*** uname -a ***'
|
||||
# uname -a
|
||||
# echo '*** mount ***'
|
||||
# mount
|
||||
# echo '*** env ***'
|
||||
# env
|
||||
# echo '*** java ***'
|
||||
# which java
|
||||
# java -version
|
||||
#
|
||||
# clone_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
|
||||
#
|
||||
# clone_dtest:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra dtest Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
#
|
||||
# build_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Build Cassandra
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
# for x in $(seq 1 3); do
|
||||
# ${ANT_HOME}/bin/ant clean jar
|
||||
# RETURN=\"$?\"
|
||||
# if [ \"${RETURN}\" -eq \"0\" ]; then
|
||||
# break
|
||||
# fi
|
||||
# done
|
||||
# # Exit, if we didn't build successfully
|
||||
# if [ \"${RETURN}\" -ne \"0\" ]; then
|
||||
# echo \"Build failed with exit code: ${RETURN}\"
|
||||
# exit ${RETURN}
|
||||
# fi
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_eclipse_warnings:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run eclipse-warnings
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# ant eclipse-warnings
|
||||
#
|
||||
# create_junit_containers:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Unit Tests to Run
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
# echo \"***java tests***\"
|
||||
#
|
||||
# # get all of our unit test filenames
|
||||
# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
|
||||
#
|
||||
# # split up the unit tests into groups based on the number of containers we have
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
|
||||
# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
#
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant clean <<parameters.target>>
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# run_parallel_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# default: testclasslist
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# create_venv:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Configure virtualenv and python Dependencies
|
||||
# command: |
|
||||
# # note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
# pip3 freeze
|
||||
#
|
||||
# create_dtest_containers:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# run_dtests_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# tests_filter_pattern:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Tests to Run (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 5m
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# cd cassandra-dtest
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
#
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
|
||||
# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
|
||||
# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
|
||||
# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
|
||||
# else
|
||||
# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
|
||||
# fi
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# run_dtests:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# pytest_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run dtests (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 15m
|
||||
# command: |
|
||||
# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# java -version
|
||||
# cd ~/cassandra-dtest
|
||||
# mkdir -p /tmp/dtest
|
||||
#
|
||||
# echo \"env: $(env)\"
|
||||
# echo \"** done env\"
|
||||
# mkdir -p /tmp/results/dtests
|
||||
# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
|
||||
# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
# - store_test_results:
|
||||
# path: /tmp/results
|
||||
# - store_artifacts:
|
||||
# path: /tmp/dtest
|
||||
# destination: dtest_<<parameters.file_tag>>
|
||||
# - store_artifacts:
|
||||
# path: ~/cassandra-dtest/logs
|
||||
# destination: dtest_<<parameters.file_tag>>_logs
|
||||
|
|
@ -1,5 +1,51 @@
|
|||
version: 2
|
||||
jobs:
|
||||
j8_jvm_upgrade_dtests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Run Unit Tests (test-jvm-upgrade-dtest-forking)
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-upgrade-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
path: /tmp/cassandra/build/test/output/
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/output
|
||||
destination: junitxml
|
||||
- store_artifacts:
|
||||
path: /tmp/cassandra/build/test/logs
|
||||
destination: logs
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -109,37 +155,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail &&\
|
||||
\ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw\
|
||||
\ > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match\
|
||||
\ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\
|
||||
\ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes\
|
||||
\ > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt\
|
||||
\ | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\n\
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail &&\
|
||||
\ cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\
|
||||
INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -280,6 +300,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean stress-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -419,36 +442,11 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently\
|
||||
\ executed on every circle container\n# so the goal here is to get the circleci\
|
||||
\ script to return the tests *this* container will run\n# which we do via\
|
||||
\ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
echo \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\
|
||||
\ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\
|
||||
\ --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\n\
|
||||
if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\n\
|
||||
else\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes\
|
||||
\ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\
|
||||
\ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\
|
||||
\ /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\
|
||||
export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\
|
||||
java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\
|
||||
\ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\
|
||||
\ the \"set -o pipefail\" here so that the exit code that circleci will\
|
||||
\ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\
|
||||
\ /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd\
|
||||
\ ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\
|
||||
\ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml\
|
||||
\ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\
|
||||
\ 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
command: "echo \"cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`\nset -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt\n"
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
|
|
@ -490,6 +488,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean test-jvm-dtest-forking
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -533,6 +534,9 @@ jobs:
|
|||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
time mv ~/cassandra /tmp
|
||||
cd /tmp/cassandra
|
||||
if [ -d ~/dtest_jars ]; then
|
||||
cp ~/dtest_jars/dtest* /tmp/cassandra/build/
|
||||
fi
|
||||
ant clean long-test
|
||||
no_output_timeout: 15m
|
||||
- store_test_results:
|
||||
|
|
@ -644,6 +648,75 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
dtest_jars_build:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 1
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Build Cassandra DTest jars
|
||||
command: |
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
cd ~/cassandra
|
||||
git remote add apache git://github.com/apache/cassandra.git
|
||||
for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do
|
||||
# check out the correct cassandra version:
|
||||
git remote set-branches --add apache '$branch'
|
||||
git fetch --depth 1 apache $branch
|
||||
git checkout $branch
|
||||
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Exit, if we didn't build successfully
|
||||
if [ "${RETURN}" -ne "0" ]; then
|
||||
echo "Build failed with exit code: ${RETURN}"
|
||||
exit ${RETURN}
|
||||
fi
|
||||
done
|
||||
# and build the dtest-jar for the branch under test
|
||||
git checkout origin/$CIRCLE_BRANCH
|
||||
for x in $(seq 1 3); do
|
||||
${ANT_HOME}/bin/ant clean jar dtest-jar
|
||||
RETURN="$?"
|
||||
if [ "${RETURN}" -eq "0" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
mkdir ~/dtest_jars
|
||||
cp build/dtest*.jar ~/dtest_jars
|
||||
ls -l ~/dtest_jars
|
||||
no_output_timeout: 15m
|
||||
- persist_to_workspace:
|
||||
root: /home/cassandra
|
||||
paths:
|
||||
- dtest_jars
|
||||
environment:
|
||||
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
workflows:
|
||||
version: 2
|
||||
build_and_run_tests:
|
||||
|
|
@ -676,6 +749,16 @@ workflows:
|
|||
- utests_stress:
|
||||
requires:
|
||||
- start_utests_stress
|
||||
- start_jvm_upgrade_dtest:
|
||||
type: approval
|
||||
requires:
|
||||
- build
|
||||
- dtest_jars_build:
|
||||
requires:
|
||||
- start_jvm_upgrade_dtest
|
||||
- j8_jvm_upgrade_dtests:
|
||||
requires:
|
||||
- dtest_jars_build
|
||||
- start_j8_dtests:
|
||||
type: approval
|
||||
requires:
|
||||
|
|
@ -693,468 +776,3 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
|
||||
# Original config.yml file:
|
||||
# version: 2.1
|
||||
#
|
||||
# default_env_vars: &default_env_vars
|
||||
# JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# ANT_HOME: /usr/share/ant
|
||||
# LANG: en_US.UTF-8
|
||||
# KEEP_TEST_DIR: true
|
||||
# DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
# PYTHONIOENCODING: utf-8
|
||||
# PYTHONUNBUFFERED: true
|
||||
# CASS_DRIVER_NO_EXTENSIONS: true
|
||||
# CASS_DRIVER_NO_CYTHON: true
|
||||
# #Skip all syncing to disk to avoid performance issues in flaky CI environments
|
||||
# CASSANDRA_SKIP_SYNC: true
|
||||
# DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
# DTEST_BRANCH: master
|
||||
# CCM_MAX_HEAP_SIZE: 1024M
|
||||
# CCM_HEAP_NEWSIZE: 256M
|
||||
#
|
||||
# j8_par_executor: &j8_par_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# #exec_resource_class: xlarge
|
||||
# parallelism: 4
|
||||
#
|
||||
# j8_seq_executor: &j8_seq_executor
|
||||
# executor:
|
||||
# name: java8-executor
|
||||
# #exec_resource_class: xlarge
|
||||
# parallelism: 1 # sequential, single container tests: no parallelism benefits
|
||||
#
|
||||
# with_dtests_jobs: &with_dtest_jobs
|
||||
# jobs:
|
||||
# - build
|
||||
# # Java 8 unit tests will be run automatically
|
||||
# - j8_unit_tests:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_jvm_dtests:
|
||||
# requires:
|
||||
# - build
|
||||
# # specialized unit tests (all run on request using Java 8)
|
||||
# - start_utests_long:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_long:
|
||||
# requires:
|
||||
# - start_utests_long
|
||||
# - start_utests_compression:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_compression:
|
||||
# requires:
|
||||
# - start_utests_compression
|
||||
# - start_utests_stress:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - utests_stress:
|
||||
# requires:
|
||||
# - start_utests_stress
|
||||
# # Java 8 dtests (on request)
|
||||
# - start_j8_dtests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - start_j8_dtests
|
||||
# # Java 8 upgrade tests
|
||||
# - start_upgrade_tests:
|
||||
# type: approval
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_upgradetests-no-vnodes:
|
||||
# requires:
|
||||
# - start_upgrade_tests
|
||||
#
|
||||
# with_dtest_jobs_only: &with_dtest_jobs_only
|
||||
# jobs:
|
||||
# - build
|
||||
# - j8_dtests-with-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
# - j8_dtests-no-vnodes:
|
||||
# requires:
|
||||
# - build
|
||||
#
|
||||
# workflows:
|
||||
# version: 2
|
||||
# build_and_run_tests: *with_dtest_jobs
|
||||
# #build_and_run_tests: *with_dtest_jobs_only
|
||||
#
|
||||
# executors:
|
||||
# java8-executor:
|
||||
# parameters:
|
||||
# exec_resource_class:
|
||||
# type: string
|
||||
# default: medium
|
||||
# docker:
|
||||
# - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
# resource_class: << parameters.exec_resource_class >>
|
||||
# working_directory: ~/
|
||||
# shell: /bin/bash -eo pipefail -l
|
||||
# environment:
|
||||
# <<: *default_env_vars
|
||||
# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
# JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
#
|
||||
# jobs:
|
||||
# build:
|
||||
# executor: java8-executor
|
||||
# parallelism: 1 # This job doesn't benefit from parallelism
|
||||
# steps:
|
||||
# - log_environment
|
||||
# - clone_cassandra
|
||||
# - build_cassandra
|
||||
# - run_eclipse_warnings
|
||||
# - persist_to_workspace:
|
||||
# root: /home/cassandra
|
||||
# paths:
|
||||
# - cassandra
|
||||
# - .m2
|
||||
#
|
||||
# j8_unit_tests:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests
|
||||
#
|
||||
# j8_jvm_dtests:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: test-jvm-dtest-forking
|
||||
#
|
||||
# utests_long:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: long-test
|
||||
#
|
||||
# utests_compression:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - create_junit_containers
|
||||
# - log_environment
|
||||
# - run_parallel_junit_tests:
|
||||
# target: testclasslist-compression
|
||||
#
|
||||
# utests_stress:
|
||||
# <<: *j8_seq_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - run_junit_tests:
|
||||
# target: stress-test
|
||||
#
|
||||
# j8_dtests-with-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_with_vnodes
|
||||
# run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_with_vnodes
|
||||
# pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_dtests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_without_vnodes
|
||||
# run_dtests_extra_args: '--skip-resource-intensive-tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_without_vnodes
|
||||
# pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
#
|
||||
# j8_upgradetests-no-vnodes:
|
||||
# <<: *j8_par_executor
|
||||
# steps:
|
||||
# - attach_workspace:
|
||||
# at: /home/cassandra
|
||||
# - clone_dtest
|
||||
# - create_venv
|
||||
# - create_dtest_containers:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# run_dtests_extra_args: '--execute-upgrade-tests'
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# tests_filter_pattern: '^upgrade_tests'
|
||||
# - run_dtests:
|
||||
# file_tag: j8_upgradetests_without_vnodes
|
||||
# extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
# pytest_extra_args: '--execute-upgrade-tests'
|
||||
#
|
||||
# commands:
|
||||
# log_environment:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Log Environment Information
|
||||
# command: |
|
||||
# echo '*** id ***'
|
||||
# id
|
||||
# echo '*** cat /proc/cpuinfo ***'
|
||||
# cat /proc/cpuinfo
|
||||
# echo '*** free -m ***'
|
||||
# free -m
|
||||
# echo '*** df -m ***'
|
||||
# df -m
|
||||
# echo '*** ifconfig -a ***'
|
||||
# ifconfig -a
|
||||
# echo '*** uname -a ***'
|
||||
# uname -a
|
||||
# echo '*** mount ***'
|
||||
# mount
|
||||
# echo '*** env ***'
|
||||
# env
|
||||
# echo '*** java ***'
|
||||
# which java
|
||||
# java -version
|
||||
#
|
||||
# clone_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
|
||||
#
|
||||
# clone_dtest:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Clone Cassandra dtest Repository (via git)
|
||||
# command: |
|
||||
# git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
#
|
||||
# build_cassandra:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Build Cassandra
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
# for x in $(seq 1 3); do
|
||||
# ${ANT_HOME}/bin/ant clean jar
|
||||
# RETURN=\"$?\"
|
||||
# if [ \"${RETURN}\" -eq \"0\" ]; then
|
||||
# break
|
||||
# fi
|
||||
# done
|
||||
# # Exit, if we didn't build successfully
|
||||
# if [ \"${RETURN}\" -ne \"0\" ]; then
|
||||
# echo \"Build failed with exit code: ${RETURN}\"
|
||||
# exit ${RETURN}
|
||||
# fi
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_eclipse_warnings:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run eclipse-warnings
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# cd ~/cassandra
|
||||
# ant eclipse-warnings
|
||||
#
|
||||
# create_junit_containers:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Unit Tests to Run
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# rm -fr ~/cassandra-dtest/upgrade_tests
|
||||
# echo \"***java tests***\"
|
||||
#
|
||||
# # get all of our unit test filenames
|
||||
# set -eo pipefail && circleci tests glob \"$HOME/cassandra/test/unit/**/*.java\" > /tmp/all_java_unit_tests.txt
|
||||
#
|
||||
# # split up the unit tests into groups based on the number of containers we have
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
|
||||
# set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep \"Test\\.java$\" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
# echo \"** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt\"
|
||||
# cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
|
||||
#
|
||||
# no_output_timeout: 15m
|
||||
#
|
||||
# run_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant clean <<parameters.target>>
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# run_parallel_junit_tests:
|
||||
# parameters:
|
||||
# target:
|
||||
# type: string
|
||||
# default: testclasslist
|
||||
# no_output_timeout:
|
||||
# type: string
|
||||
# default: 15m
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run Unit Tests (<<parameters.target>>)
|
||||
# # Please note that we run `clean` and therefore rebuild the project, as we can't run tests on Java 8 in case
|
||||
# # based on Java 11 builds.
|
||||
# command: |
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# time mv ~/cassandra /tmp
|
||||
# cd /tmp/cassandra
|
||||
# ant <<parameters.target>> -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit
|
||||
# no_output_timeout: <<parameters.no_output_timeout>>
|
||||
# - store_test_results:
|
||||
# path: /tmp/cassandra/build/test/output/
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/output
|
||||
# destination: junitxml
|
||||
# - store_artifacts:
|
||||
# path: /tmp/cassandra/build/test/logs
|
||||
# destination: logs
|
||||
#
|
||||
# create_venv:
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Configure virtualenv and python Dependencies
|
||||
# command: |
|
||||
# # note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# # rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
# pip3 freeze
|
||||
#
|
||||
# create_dtest_containers:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# run_dtests_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# tests_filter_pattern:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Determine Tests to Run (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 5m
|
||||
# command: |
|
||||
# # reminder: this code (along with all the steps) is independently executed on every circle container
|
||||
# # so the goal here is to get the circleci script to return the tests *this* container will run
|
||||
# # which we do via the `circleci` cli tool.
|
||||
#
|
||||
# cd cassandra-dtest
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
#
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# echo \"***Collected DTests (<<parameters.file_tag>>)***\"
|
||||
# set -eo pipefail && ./run_dtests.py <<parameters.run_dtests_extra_args>> --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_<<parameters.file_tag>>_raw --cassandra-dir=../cassandra
|
||||
# if [ -z '<<parameters.tests_filter_pattern>>' ]; then
|
||||
# mv /tmp/all_dtest_tests_<<parameters.file_tag>>_raw /tmp/all_dtest_tests_<<parameters.file_tag>>
|
||||
# else
|
||||
# grep -e '<<parameters.tests_filter_pattern>>' /tmp/all_dtest_tests_<<parameters.file_tag>>_raw > /tmp/all_dtest_tests_<<parameters.file_tag>> || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }
|
||||
# fi
|
||||
# set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_<<parameters.file_tag>> > /tmp/split_dtest_tests_<<parameters.file_tag>>.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# run_dtests:
|
||||
# parameters:
|
||||
# file_tag:
|
||||
# type: string
|
||||
# pytest_extra_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# extra_env_args:
|
||||
# type: string
|
||||
# default: ''
|
||||
# steps:
|
||||
# - run:
|
||||
# name: Run dtests (<<parameters.file_tag>>)
|
||||
# no_output_timeout: 15m
|
||||
# command: |
|
||||
# echo \"cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt\"
|
||||
# cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt
|
||||
#
|
||||
# source ~/env/bin/activate
|
||||
# export PATH=$JAVA_HOME/bin:$PATH
|
||||
# if [ -n '<<parameters.extra_env_args>>' ]; then
|
||||
# export <<parameters.extra_env_args>>
|
||||
# fi
|
||||
#
|
||||
# java -version
|
||||
# cd ~/cassandra-dtest
|
||||
# mkdir -p /tmp/dtest
|
||||
#
|
||||
# echo \"env: $(env)\"
|
||||
# echo \"** done env\"
|
||||
# mkdir -p /tmp/results/dtests
|
||||
# # we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
# export SPLIT_TESTS=`cat /tmp/split_dtest_tests_<<parameters.file_tag>>_final.txt`
|
||||
# set -o pipefail && cd ~/cassandra-dtest && pytest <<parameters.pytest_extra_args>> --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_<<parameters.file_tag>>.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
# - store_test_results:
|
||||
# path: /tmp/results
|
||||
# - store_artifacts:
|
||||
# path: /tmp/dtest
|
||||
# destination: dtest_<<parameters.file_tag>>
|
||||
# - store_artifacts:
|
||||
# path: ~/cassandra-dtest/logs
|
||||
# destination: dtest_<<parameters.file_tag>>_logs
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
* Fix LegacyLayout to have same behavior as 2.x when handling unknown column names (CASSANDRA-15081)
|
||||
* Update nodetool help stop output (CASSANDRA-15401)
|
||||
Merged from 3.0:
|
||||
* Run in-jvm upgrade dtests in circleci (CASSANDRA-15506)
|
||||
* Include updates to static column in mutation size calculations (CASSANDRA-15293)
|
||||
* Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292)
|
||||
* GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306)
|
||||
|
|
|
|||
35
build.xml
35
build.xml
|
|
@ -67,6 +67,8 @@
|
|||
<property name="test.microbench.src" value="${test.dir}/microbench"/>
|
||||
<property name="test.distributed.src" value="${test.dir}/distributed"/>
|
||||
<property name="test.distributed.listfile" value="ant-jvm-dtest-list"/>
|
||||
<property name="test.distributed.upgrade.listfile" value="ant-jvm-dtest-upgrade-list"/>
|
||||
<property name="test.distributed.upgrade.package" value="org.apache.cassandra.distributed.upgrade"/>
|
||||
<property name="dist.dir" value="${build.dir}/dist"/>
|
||||
<property name="tmp.dir" value="${java.io.tmpdir}"/>
|
||||
|
||||
|
|
@ -1862,6 +1864,39 @@
|
|||
</testmacro>
|
||||
</target>
|
||||
|
||||
<!-- In-JVM upgrade dtests -->
|
||||
<target name="list-jvm-upgrade-dtests" depends="build-test">
|
||||
<java classname="org.apache.cassandra.distributed.test.TestLocator" fork="no">
|
||||
<classpath>
|
||||
<path refid="cassandra.classpath" />
|
||||
<pathelement location="${test.classes}"/>
|
||||
<pathelement location="${test.conf}"/>
|
||||
<fileset dir="${test.lib}">
|
||||
<include name="**/*.jar" />
|
||||
</fileset>
|
||||
</classpath>
|
||||
<arg value="${test.distributed.upgrade.listfile}"/>
|
||||
<arg value="${test.distributed.upgrade.package}"/>
|
||||
</java>
|
||||
</target>
|
||||
|
||||
<target name="test-jvm-upgrade-dtest-forking" depends="list-jvm-upgrade-dtests" description="Execute In-JVM 'distributed' upgrade tests" >
|
||||
<chmod file="${test.distributed.upgrade.listfile}" perm="+x"/>
|
||||
<exec executable="./${test.distributed.upgrade.listfile}" failonerror="true"/>
|
||||
<delete file="${test.distributed.upgrade.listfile}"/>
|
||||
</target>
|
||||
|
||||
<target name="test-jvm-upgrade-dtest" depends="build-test" description="Execute in-jvm dtests">
|
||||
<testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/upgrade/*Test.java">
|
||||
<jvmarg value="-Dlogback.configurationFile=test/conf/logback-dtest.xml"/>
|
||||
<jvmarg value="-Dcassandra.ring_delay_ms=10000"/>
|
||||
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
|
||||
<jvmarg value="-Djava.io.tmpdir=${tmp.dir}"/>
|
||||
<jvmarg value="-Dcassandra.skip_sync=true" />
|
||||
<jvmarg value="-XX:MaxMetaspaceSize=512M"/>
|
||||
</testmacro>
|
||||
</target>
|
||||
|
||||
<!-- Use this with an FQDN for test class, and a csv list of methods like this:
|
||||
ant test-jvm-dtest-some -Dtest.name=org.apache.cassandra.distributed.test.ResourceLeakTest -Dtest.methods=looperTest
|
||||
-->
|
||||
|
|
|
|||
|
|
@ -39,14 +39,18 @@ public class TestLocator
|
|||
public static void main(String[] args) throws Throwable
|
||||
{
|
||||
String outputFileName = defaultOutputFileName;
|
||||
if (args.length == 1)
|
||||
if (args.length >= 1)
|
||||
{
|
||||
outputFileName = args[0];
|
||||
}
|
||||
String testPackage = TestLocator.testPackage;
|
||||
if (args.length == 2)
|
||||
testPackage = args[1];
|
||||
try (FileWriter fileWriter = new FileWriter(outputFileName);
|
||||
PrintWriter printWriter = new PrintWriter(fileWriter))
|
||||
{
|
||||
printWriter.println("#!/bin/bash");
|
||||
printWriter.println("ret=0");
|
||||
for (Class testClass : locateClasses(testPackage))
|
||||
{
|
||||
for (Method method : testClass.getMethods())
|
||||
|
|
@ -57,8 +61,10 @@ public class TestLocator
|
|||
printWriter.println(String.format(testCommandFormat,
|
||||
testClass.getName(),
|
||||
method.getName()));
|
||||
printWriter.println("if [ $? -ne 0 ]; then ret=1; fi");
|
||||
}
|
||||
}
|
||||
printWriter.println("exit $ret");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue