natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
cassandra/.circleci/config.yml

255 lines
11 KiB
YAML
Raw Blame History

env_vars: &env_vars
JAVA_HOME: /usr/local/openjdk8u154-cassandra-b02
ANT_HOME: /usr/local/apache-ant-1.10.1
LANG: en_US.UTF-8
JDK_HOME: /usr/local/openjdk8u154-cassandra-b02
JAVA8_HOME: /usr/local/openjdk8u154-cassandra-b02
JAVA7_HOME: /usr/local/openjdk7u82-cassandra-b02
DEBUG: true
PRINT_DEBUG: true
KEEP_TEST_DIR: true
CASSANDRA_DIR: /home/cassandra/cassandra
DEFAULT_DIR: /home/cassandra/cassandra-dtest
PYTHONIOENCODING: utf-8
PYTHONUNBUFFERED: true
CASS_DRIVER_NO_EXTENSIONS: true
CASS_DRIVER_NO_CYTHON: true
CCM_MAX_HEAP_SIZE: 3072M
CCM_HEAP_NEWSIZE: 256M
version: 2
jobs:
build:
resource_class: xlarge
working_directory: ~/
shell: /bin/bash -eo pipefail -l
docker:
- image: kjellman/cassandra-test:0.1.2
environment:
<<: *env_vars
steps:
- run:
name: Log Environment Information
command: |
echo '*** id ***'
id
echo '*** cat /proc/cpuinfo ***'
cat /proc/cpuinfo
echo '*** free -m ***'
free -m
echo '*** df -m ***'
df -m
echo '*** ifconfig -a ***'
ifconfig -a
echo '*** uname -a ***'
uname -a
echo '*** mount ***'
mount
echo '*** env ***'
env
- run:
name: Clone Cassandra Repository (via git)
command: |
export LANG=en_US.UTF-8
git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra
- run:
name: Build Cassandra
command: |
export LANG=en_US.UTF-8
export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
cd ~/cassandra
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
for x in $(seq 1 3); do
${ANT_HOME}/bin/ant clean jar
RETURN="$?"
if [ "${RETURN}" -eq "0" ]; then
break
fi
done
# Exit, if we didn't build successfully
if [ "${RETURN}" -ne "0" ]; then
echo "Build failed with exit code: ${RETURN}"
exit ${RETURN}
fi
no_output_timeout: 20m
- persist_to_workspace:
root: /home/cassandra
paths:
- cassandra
- .m2
unit_tests:
resource_class: xlarge
parallelism: 100
working_directory: ~/
shell: /bin/bash -eo pipefail -l
docker:
- image: kjellman/cassandra-test:0.1.2
environment:
<<: *env_vars
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Determine Tests to Run
no_output_timeout: 10m
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
export LANG=en_US.UTF-8
rm -fr ~/cassandra-dtest/upgrade_tests
echo "***java tests***"
# get all of our unit test filenames
set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt
# split up the unit tests into groups based on the number of containers we have
set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
- run:
name: Run Unit Tests
command: |
export LANG=en_US.UTF-8
export JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF8"
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
cd ~/cassandra
ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt || true
no_output_timeout: 15m
- store_test_results:
path: ~/cassandra/build/test/output/
- store_artifacts:
path: ~/cassandra/build/test/output
destination: junitxml
- store_artifacts:
path: ~/cassandra/build/test/logs
destination: logs
dtests:
resource_class: xlarge
parallelism: 100
working_directory: ~/
shell: /bin/bash -eo pipefail -l
docker:
- image: kjellman/cassandra-test:0.1.2
environment:
<<: *env_vars
steps:
- attach_workspace:
at: /home/cassandra
- run:
name: Clone Cassandra dtest Repository (via git)
command: |
export LANG=en_US.UTF-8
git clone --single-branch --depth 1 git://github.com/jasobrown/cassandra-dtest.git ~/cassandra-dtest
- run:
name: Configure virtualenv and python Dependencies
command: |
# note, this should be super quick as all dependencies should be pre-installed in the docker image
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
export LANG=en_US.UTF-8
source ~/env/bin/activate
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
export CASS_DRIVER_NO_EXTENSIONS=true
export CASS_DRIVER_NO_CYTHON=true
pip install -r ~/cassandra-dtest/requirements.txt
pip freeze
- run:
name: Determine Tests to Run
no_output_timeout: 10m
command: |
# reminder: this code (along with all the steps) is independently executed on every circle container
# so the goal here is to get the circleci script to return the tests *this* container will run
# which we do via the `circleci` cli tool.
export LANG=en_US.UTF-8
echo "***python tests***"
cd cassandra-dtest
export CASSANDRA_DIR=/home/cassandra/cassandra
source ~/env/bin/activate
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
echo "***regular python tests***"
set -eo pipefail && ./run_dtests.py --vnodes true --nose-options='--verbosity=3 --dtest-collect-only' 2>&1 | tee /tmp/pythontestlist
set -eo pipefail && cat /tmp/pythontestlist | grep -v 'upgrade_tests' | grep "... ok" | rev | cut -c 8- | rev > /tmp/testlist_trimmed
set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/testlist_trimmed > /tmp/python_tests.txt
cat /tmp/python_tests.txt
cat /tmp/python_tests.txt | grep -v "\-novnode" | grep -v "\-offheap" > /tmp/regular_python_tests.txt
cat /tmp/regular_python_tests.txt | tr '\n' ' ' > /tmp/regular_python_tests_final.txt
cat /tmp/regular_python_tests_final.txt
echo "***novnode python tests***"
rm /tmp/pythontestlist || true
./run_dtests.py --vnodes false --nose-options='--verbosity=3 --dtest-collect-only --attr=!resource-intensive' 2>&1 | tee -a /tmp/pythontestlist
set -eo pipefail && cat /tmp/pythontestlist | grep -v 'upgrade_tests' | grep "... ok" | rev | cut -c 8- | rev > /tmp/testlist_trimmed
set -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/testlist_trimmed > /tmp/novnode_python_tests.txt
cat /tmp/novnode_python_tests.txt | tr '\n' ' ' > /tmp/novnode_python_tests_final.txt
cat /tmp/novnode_python_tests_final.txt
# echo "***offheap python tests***"
# rm /tmp/pythontestlist || true
# TEST_TAG=offheap ./run_dtests.py --vnodes true --nose-options='--verbosity=3 --dtest-collect-only --attr=!resource-intensive' 2>&1 | grep -v "upgrade_tests." | tee -a /tmp/pythontestlist
# cat /tmp/python_tests.txt | grep "\-offheap" | rev | cut -c 9- | rev > /tmp/offheap_python_tests.txt
# cat /tmp/offheap_python_tests.txt | tr '\n' ' ' > /tmp/offheap_python_tests_final.txt
# cat /tmp/offheap_python_tests_final.txt
- run:
name: Run dtests
no_output_timeout: 30m
command: |
echo "cat /tmp/regular_python_tests_final.txt"
cat /tmp/regular_python_tests_final.txt
source ~/env/bin/activate
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
cd ~/cassandra-dtest
mkdir -p /tmp/dtest
export NUM_TOKENS="32"
echo "env: $(env)"
echo "** done env"
./run_dtests.py --vnodes true --nose-options="--verbosity=3 --with-xunit --nocapture" `cat /tmp/regular_python_tests_final.txt` 2>&1 | tee /tmp/dtest/stdout.txt || true
# store the test results somewhere where we can pick them up
mv nosetests.xml nosetests-regular.xml
mkdir -p /tmp/results/ || true
find ~/cassandra-dtest/ -name "*.xml" -exec cp {} /tmp/results/ \;
- run:
name: Run dtests (without vnodes)
no_output_timeout: 30m
command: |
source ~/env/bin/activate
export PATH=$PATH:$ANT_HOME/bin:$JAVA_HOME/bin
cd ~/cassandra-dtest
export TEST_TAG="novnodes"
./run_dtests.py --vnodes false --nose-options="--verbosity=3 --with-dtestxunit --nocapture --attr=!resource-intensive" `cat /tmp/novnode_python_tests_final.txt` 2>&1 | tee /tmp/dtest/stdout-novnodes.txt || true
# store the test results somewhere where we can pick them up
mv nosetests.xml nosetests-novnode.xml
mkdir -p /tmp/results/ || true
find ~/cassandra-dtest/ -name "*.xml" -exec cp {} /tmp/results/ \;
- store_test_results:
path: /tmp/results/
- store_artifacts:
path: /tmp/dtest
destination: dtest
- store_artifacts:
path: ~/cassandra-dtest/logs
destination: dtest_logs
workflows:
version: 2
build_and_run_tests:
jobs:
- build
- unit_tests:
requires:
- build
- dtests:
requires:
- unit_tests
Reference in New Issue View Git Blame Copy Permalink