mirror of https://github.com/apache/cassandra
Make cqlsh and cqlshlib Python 2 & 3 compatible
Patch by Patrick Bannister; reviewed by Dinesh Joshi, Andy Tolbert and David Capwell for CASSANDRA-10190
This commit is contained in:
Patrick Bannister
Dinesh A. Joshi
parent
05c70abc63
commit
bf9a1d487b
|
|
@ -138,6 +138,46 @@ j8_with_dtests_jobs: &j8_with_dtests_jobs
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
- start_j8_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- j8_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- start_j8_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- j8_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
|
||||
j11_with_dtests_jobs: &j11_with_dtests_jobs
|
||||
jobs:
|
||||
|
|
@ -164,6 +204,26 @@ j11_with_dtests_jobs: &j11_with_dtests_jobs
|
|||
- j11_dtests-no-vnodes:
|
||||
requires:
|
||||
- start_j11_dtests
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
|
||||
j8_with_dtest_jobs_only: &j8_with_dtest_jobs_only
|
||||
jobs:
|
||||
|
|
@ -355,7 +415,7 @@ jobs:
|
|||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_with_vnodes
|
||||
run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
|
|
@ -370,7 +430,7 @@ jobs:
|
|||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_with_vnodes
|
||||
run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests'
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
|
|
@ -384,7 +444,7 @@ jobs:
|
|||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_without_vnodes
|
||||
run_dtests_extra_args: '--skip-resource-intensive-tests'
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
|
|
@ -399,7 +459,7 @@ jobs:
|
|||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_without_vnodes
|
||||
run_dtests_extra_args: '--skip-resource-intensive-tests'
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
|
|
@ -421,6 +481,126 @@ jobs:
|
|||
extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true'
|
||||
pytest_extra_args: '--execute-upgrade-tests'
|
||||
|
||||
j8_cqlsh-dtests-py2-with-vnodes:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_with_vnodes
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
|
||||
|
||||
j8_cqlsh-dtests-py3-with-vnodes:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_with_vnodes
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j8_cqlsh-dtests-py2-no-vnodes:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_without_vnodes
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
|
||||
|
||||
j8_cqlsh-dtests-py3-no-vnodes:
|
||||
<<: *j8_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j8_without_vnodes
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j8_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j11_cqlsh-dtests-py2-with-vnodes:
|
||||
<<: *j11_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_with_vnodes
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
|
||||
|
||||
j11_cqlsh-dtests-py3-with-vnodes:
|
||||
<<: *j11_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_with_vnodes
|
||||
run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_with_vnodes
|
||||
pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
j11_cqlsh-dtests-py2-no-vnodes:
|
||||
<<: *j11_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_without_vnodes
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7'
|
||||
|
||||
j11_cqlsh-dtests-py3-no-vnodes:
|
||||
<<: *j11_par_executor
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- clone_dtest
|
||||
- create_venv
|
||||
- create_dtest_containers:
|
||||
file_tag: j11_without_vnodes
|
||||
run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'"
|
||||
- run_dtests:
|
||||
file_tag: j11_without_vnodes
|
||||
pytest_extra_args: '--skip-resource-intensive-tests'
|
||||
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
|
||||
|
||||
commands:
|
||||
log_environment:
|
||||
steps:
|
||||
|
|
@ -635,6 +815,7 @@ commands:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
|
||||
create_dtest_containers:
|
||||
|
|
|
|||
|
|
@ -86,6 +86,82 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_unit_tests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -173,6 +249,389 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -223,7 +682,7 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -281,7 +740,7 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -707,6 +1166,83 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -735,7 +1271,7 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -930,6 +1466,82 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_long:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -1070,7 +1682,7 @@ jobs:
|
|||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -1336,6 +1948,46 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
- start_j8_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- j8_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- start_j8_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- j8_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
java11_build_and_run_tests:
|
||||
jobs:
|
||||
- j11_build
|
||||
|
|
@ -1359,3 +2011,23 @@ workflows:
|
|||
- j11_dtests-no-vnodes:
|
||||
requires:
|
||||
- start_j11_dtests
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
|
|
|
|||
|
|
@ -86,6 +86,83 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_unit_tests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -173,6 +250,394 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -219,11 +684,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -277,11 +743,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -334,6 +801,7 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_upgradetests_without_vnodes)
|
||||
|
|
@ -707,6 +1175,84 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -731,11 +1277,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -930,6 +1477,83 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: xlarge
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 100
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_long:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -1066,11 +1690,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -1336,6 +1961,46 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
- start_j8_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- j8_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- start_j8_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- j8_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
java11_build_and_run_tests:
|
||||
jobs:
|
||||
- j11_build
|
||||
|
|
@ -1359,3 +2024,23 @@ workflows:
|
|||
- j11_dtests-no-vnodes:
|
||||
requires:
|
||||
- start_j11_dtests
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
|
|
|
|||
|
|
@ -86,6 +86,83 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_unit_tests:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -173,6 +250,394 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_cqlsh-dtests-py3-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j11_cqlsh-dtests-py2-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_with_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_with_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
|
|
@ -219,11 +684,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -277,11 +743,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -334,6 +801,7 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_upgradetests_without_vnodes)
|
||||
|
|
@ -707,6 +1175,84 @@ jobs:
|
|||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j11_cqlsh-dtests-py2-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11:20181210
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j11_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j11_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64
|
||||
- CASSANDRA_USE_JDK11: true
|
||||
j8_dtests-with-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -731,11 +1277,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_with_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_with_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -930,6 +1477,83 @@ jobs:
|
|||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
j8_cqlsh-dtests-py3-no-vnodes:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
resource_class: medium
|
||||
working_directory: ~/
|
||||
shell: /bin/bash -eo pipefail -l
|
||||
parallelism: 4
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: /home/cassandra
|
||||
- run:
|
||||
name: Clone Cassandra dtest Repository (via git)
|
||||
command: |
|
||||
git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest
|
||||
- run:
|
||||
name: Configure virtualenv and python Dependencies
|
||||
command: |
|
||||
# note, this should be super quick as all dependencies should be pre-installed in the docker image
|
||||
# if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated
|
||||
# we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and
|
||||
# rebuild the docker image! (it automatically pulls the latest requirements.txt on build)
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j8_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j8_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
command: |
|
||||
echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt"
|
||||
cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt
|
||||
|
||||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then
|
||||
export CQLSH_PYTHON=/usr/bin/python3.6
|
||||
fi
|
||||
|
||||
java -version
|
||||
cd ~/cassandra-dtest
|
||||
mkdir -p /tmp/dtest
|
||||
|
||||
echo "env: $(env)"
|
||||
echo "** done env"
|
||||
mkdir -p /tmp/results/dtests
|
||||
# we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee
|
||||
export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt`
|
||||
set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt
|
||||
- store_test_results:
|
||||
path: /tmp/results
|
||||
- store_artifacts:
|
||||
path: /tmp/dtest
|
||||
destination: dtest_j8_without_vnodes
|
||||
- store_artifacts:
|
||||
path: ~/cassandra-dtest/logs
|
||||
destination: dtest_j8_without_vnodes_logs
|
||||
environment:
|
||||
- ANT_HOME: /usr/share/ant
|
||||
- LANG: en_US.UTF-8
|
||||
- KEEP_TEST_DIR: true
|
||||
- DEFAULT_DIR: /home/cassandra/cassandra-dtest
|
||||
- PYTHONIOENCODING: utf-8
|
||||
- PYTHONUNBUFFERED: true
|
||||
- CASS_DRIVER_NO_EXTENSIONS: true
|
||||
- CASS_DRIVER_NO_CYTHON: true
|
||||
- CASSANDRA_SKIP_SYNC: true
|
||||
- DTEST_REPO: git://github.com/apache/cassandra-dtest.git
|
||||
- DTEST_BRANCH: master
|
||||
- CCM_MAX_HEAP_SIZE: 1024M
|
||||
- CCM_HEAP_NEWSIZE: 256M
|
||||
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
|
||||
utests_long:
|
||||
docker:
|
||||
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
|
||||
|
|
@ -1066,11 +1690,12 @@ jobs:
|
|||
source ~/env/bin/activate
|
||||
export PATH=$JAVA_HOME/bin:$PATH
|
||||
pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
|
||||
pip3 uninstall -y cqlsh
|
||||
pip3 freeze
|
||||
- run:
|
||||
name: Determine Tests to Run (j11_without_vnodes)
|
||||
no_output_timeout: 5m
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n"
|
||||
- run:
|
||||
name: Run dtests (j11_without_vnodes)
|
||||
no_output_timeout: 15m
|
||||
|
|
@ -1336,6 +1961,46 @@ workflows:
|
|||
- j8_upgradetests-no-vnodes:
|
||||
requires:
|
||||
- start_upgrade_tests
|
||||
- start_j8_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- j8_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-with-vnodes
|
||||
- start_j8_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j8_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- j8_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j8_cqlsh_tests-no-vnodes
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j8_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
java11_build_and_run_tests:
|
||||
jobs:
|
||||
- j11_build
|
||||
|
|
@ -1359,3 +2024,23 @@ workflows:
|
|||
- j11_dtests-no-vnodes:
|
||||
requires:
|
||||
- start_j11_dtests
|
||||
- start_j11_cqlsh_tests-with-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- j11_cqlsh-dtests-py3-with-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-with-vnodes
|
||||
- start_j11_cqlsh_tests-no-vnodes:
|
||||
type: approval
|
||||
requires:
|
||||
- j11_build
|
||||
- j11_cqlsh-dtests-py2-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
- j11_cqlsh-dtests-py3-no-vnodes:
|
||||
requires:
|
||||
- start_j11_cqlsh_tests-no-vnodes
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
4.0-alpha4
|
||||
* Make cqlsh and cqlshlib Python 2 & 3 compatible (CASSANDRA-10190)
|
||||
* Added documentation for Full Query Logging (CASSANDRA-15475)
|
||||
* Added documentation for backups (CASSANDRA-15479)
|
||||
* Documentation gives the wrong instruction to activate remote jmx (CASSANDRA-15535)
|
||||
|
|
|
|||
81
bin/cqlsh
81
bin/cqlsh
|
|
@ -15,12 +15,79 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# bash code here; finds a suitable python interpreter and execs this file.
|
||||
# prefer unqualified "python" if suitable:
|
||||
python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \
|
||||
&& exec python "`python -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
|
||||
for pyver in 2.7; do
|
||||
which python$pyver > /dev/null 2>&1 && exec python$pyver "`python$pyver -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
|
||||
# shell script to find a suitable Python interpreter and run cqlsh.py
|
||||
|
||||
# parse arguments
|
||||
PARAMS=""
|
||||
|
||||
# Use the Python that is specified in the env
|
||||
if [ -n "$CQLSH_PYTHON" ]; then
|
||||
USER_SPECIFIED_PYTHON="$CQLSH_PYTHON"
|
||||
fi
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--python)
|
||||
if [ $# -lt 2 ]; then
|
||||
echo "You must specify a python interpreter path with the --python option"
|
||||
exit 1
|
||||
fi
|
||||
USER_SPECIFIED_PYTHON="$2"
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
PARAMS="$PARAMS $1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "No appropriate python interpreter found." >&2
|
||||
|
||||
# get a version string for a Python interpreter
|
||||
get_python_version() {
|
||||
interpreter=$1
|
||||
version=$(command -v "$interpreter" > /dev/null 2>&1 && $interpreter -c "import os; print('{}.{}'.format(os.sys.version_info.major, os.sys.version_info.minor))")
|
||||
echo "$version"
|
||||
}
|
||||
|
||||
# test whether a version string matches one of the supported versions for cqlsh
|
||||
is_supported_version() {
|
||||
version=$1
|
||||
if [ "$version" = "3.6" ] || [ "$version" = "2.7" ]; then
|
||||
echo "supported"
|
||||
else
|
||||
echo "unsupported"
|
||||
fi
|
||||
}
|
||||
|
||||
run_if_supported_version() {
|
||||
interpreter="$1"
|
||||
params="$2"
|
||||
version=$(get_python_version "$interpreter")
|
||||
if [ -n "$version" ]; then
|
||||
if [ "$(is_supported_version "$version")" = "supported" ]; then
|
||||
# We need the params to be unquoted, otherwise the shell will just interpret it as one giant string
|
||||
# shellcheck disable=SC2086
|
||||
exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" $params
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [ "$USER_SPECIFIED_PYTHON" != "" ]; then
|
||||
# run a user specified Python interpreter
|
||||
run_if_supported_version "$USER_SPECIFIED_PYTHON" "$PARAMS"
|
||||
else
|
||||
# try unqualified python first, then python3, then python2.7
|
||||
for interpreter in python python3 python2.7; do
|
||||
run_if_supported_version "$interpreter" "$PARAMS"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "No appropriate Python interpreter found." >&2
|
||||
exit 1
|
||||
|
|
|
|||
413
bin/cqlsh.py
413
bin/cqlsh.py
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,2 @@
|
|||
FROM ubuntu:bionic
|
||||
RUN apt-get update && apt-get install -y python-minimal
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
FROM ubuntu:bionic
|
||||
RUN apt-get update && apt-get install -y python3-minimal
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
== Overview
|
||||
|
||||
This directory contains code primarily for cqlsh. cqlsh uses cqlshlib in this directory.
|
||||
Currently, cqlshlib supports Python 2 as well as Python 3. Support for Python 3 is relatively
|
||||
new.
|
||||
|
||||
== Requirements
|
||||
. Python 3 and 2.7 (for cqlsh)
|
||||
. virtualenv
|
||||
. Docker (optional)
|
||||
|
||||
== Running tests
|
||||
|
||||
In order to run tests for cqlshlib, run cassandra-cqlsh-tests.sh in this directory. It will
|
||||
automatically setup a virtualenv with the appropriate version of Python and run tests inside it.
|
||||
|
||||
There are Dockerfiles that can be used to test whether cqlsh works with a default, barebones
|
||||
Python installation. Assuming Cassandra's source is checked out at `$CASSANDRA_DIR`. To test, first
|
||||
build the Docker image containing the barebones Python installation -
|
||||
|
||||
$ docker build . --file Dockerfile.ubuntu.py3 -t ubuntu-lts-py3
|
||||
|
||||
Next, run cqlsh inside the newly built image -
|
||||
|
||||
$ docker run -v $CASSANDRA_DIR:/code -it ubuntu-lts-py3:latest /code/cassandra/bin/cqlsh host.docker.internal
|
||||
|
||||
This will try to spawn a cqlsh instance inside the Docker container running Ubuntu LTS (18.04)
|
||||
with minimal Python installation. It will try to connect to the Cassandra instance running on the
|
||||
Docker host at port 9042. If you have Cassandra running elsewhere, replace host.docker.internal
|
||||
with the IP / hostname as usual. Please ensure that the IP / host is accessible from _within_ the
|
||||
Docker container.
|
||||
|
|
@ -7,12 +7,22 @@
|
|||
################################
|
||||
|
||||
WORKSPACE=$1
|
||||
PYTHON_VERSION=$2
|
||||
|
||||
if [ "${WORKSPACE}" = "" ]; then
|
||||
echo "Specify Cassandra source directory"
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ "${PYTHON_VERSION}" = "" ]; then
|
||||
PYTHON_VERSION=python3
|
||||
fi
|
||||
|
||||
if [ "${PYTHON_VERSION}" != "python3" -a "${PYTHON_VERSION}" != "python2" ]; then
|
||||
echo "Specify Python version python3 or python2"
|
||||
exit
|
||||
fi
|
||||
|
||||
export PYTHONIOENCODING="utf-8"
|
||||
export PYTHONUNBUFFERED=true
|
||||
export CASS_DRIVER_NO_EXTENSIONS=true
|
||||
|
|
@ -23,6 +33,10 @@ export CCM_CONFIG_DIR=${WORKSPACE}/.ccm
|
|||
export NUM_TOKENS="32"
|
||||
export CASSANDRA_DIR=${WORKSPACE}
|
||||
|
||||
if [ -z "$CASSANDRA_USE_JDK11" ]; then
|
||||
export CASSANDRA_USE_JDK11=true
|
||||
fi
|
||||
|
||||
# Loop to prevent failure due to maven-ant-tasks not downloading a jar..
|
||||
for x in $(seq 1 3); do
|
||||
ant -buildfile ${CASSANDRA_DIR}/build.xml realclean jar
|
||||
|
|
@ -39,8 +53,9 @@ fi
|
|||
|
||||
# Set up venv with dtest dependencies
|
||||
set -e # enable immediate exit if venv setup fails
|
||||
virtualenv --python=python2 --no-site-packages venv
|
||||
virtualenv --python=$PYTHON_VERSION --no-site-packages venv
|
||||
source venv/bin/activate
|
||||
|
||||
pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt
|
||||
pip freeze
|
||||
|
||||
|
|
@ -59,10 +74,11 @@ fi
|
|||
ccm remove test || true # in case an old ccm cluster is left behind
|
||||
ccm create test -n 1 --install-dir=${CASSANDRA_DIR}
|
||||
ccm updateconf "enable_user_defined_functions: true"
|
||||
ccm updateconf "enable_scripted_user_defined_functions: true"
|
||||
|
||||
version_from_build=$(ccm node1 versionfrombuild)
|
||||
export pre_or_post_cdc=$(python -c """from distutils.version import LooseVersion
|
||||
print \"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\"
|
||||
print (\"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\")
|
||||
""")
|
||||
case "${pre_or_post_cdc}" in
|
||||
postcdc)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import ConfigParser
|
||||
from __future__ import unicode_literals
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
|
|
@ -26,6 +26,8 @@ import os
|
|||
import platform
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import six
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
|
|
@ -36,25 +38,30 @@ from bisect import bisect_right
|
|||
from calendar import timegm
|
||||
from collections import defaultdict, namedtuple
|
||||
from decimal import Decimal
|
||||
from Queue import Queue
|
||||
from random import randint
|
||||
from StringIO import StringIO
|
||||
from io import BytesIO, StringIO
|
||||
from select import select
|
||||
from uuid import UUID
|
||||
from util import profile_on, profile_off
|
||||
from .util import profile_on, profile_off
|
||||
|
||||
from six import ensure_str, ensure_text
|
||||
from six.moves import configparser
|
||||
from six.moves import range
|
||||
from six.moves.queue import Queue
|
||||
|
||||
from cassandra import OperationTimedOut
|
||||
from cassandra.cluster import Cluster, DefaultConnection
|
||||
from cassandra.cqltypes import ReversedType, UserType
|
||||
from cassandra.cqltypes import ReversedType, UserType, BytesType
|
||||
from cassandra.metadata import protect_name, protect_names, protect_value
|
||||
from cassandra.policies import RetryPolicy, WhiteListRoundRobinPolicy, DCAwareRoundRobinPolicy, FallthroughRetryPolicy
|
||||
from cassandra.query import BatchStatement, BatchType, SimpleStatement, tuple_factory
|
||||
from cassandra.util import Date, Time
|
||||
from cqlshlib.util import profile_on, profile_off
|
||||
|
||||
from cql3handling import CqlRuleSet
|
||||
from displaying import NO_COLOR_MAP
|
||||
from formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter
|
||||
from sslhandling import ssl_settings
|
||||
from cqlshlib.cql3handling import CqlRuleSet
|
||||
from cqlshlib.displaying import NO_COLOR_MAP
|
||||
from cqlshlib.formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter
|
||||
from cqlshlib.sslhandling import ssl_settings
|
||||
|
||||
PROFILE_ON = False
|
||||
STRACE_ON = False
|
||||
|
|
@ -79,7 +86,7 @@ def printdebugmsg(msg):
|
|||
|
||||
|
||||
def printmsg(msg, eol='\n', encoding='utf8'):
|
||||
sys.stdout.write(msg.encode(encoding))
|
||||
sys.stdout.write(msg)
|
||||
sys.stdout.write(eol)
|
||||
sys.stdout.flush()
|
||||
|
||||
|
|
@ -138,7 +145,7 @@ class SendingChannel(object):
|
|||
try:
|
||||
msg = self.pending_messages.get()
|
||||
self.pipe.send(msg)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
printmsg('%s: %s' % (e.__class__.__name__, e.message))
|
||||
|
||||
feeding_thread = threading.Thread(target=feed)
|
||||
|
|
@ -160,7 +167,7 @@ class SendingChannels(object):
|
|||
A group of one way channels for sending messages.
|
||||
"""
|
||||
def __init__(self, num_channels):
|
||||
self.pipes = [OneWayPipe() for _ in xrange(num_channels)]
|
||||
self.pipes = [OneWayPipe() for _ in range(num_channels)]
|
||||
self.channels = [SendingChannel(p) for p in self.pipes]
|
||||
self.num_channels = num_channels
|
||||
|
||||
|
|
@ -177,7 +184,7 @@ class ReceivingChannels(object):
|
|||
A group of one way channels for receiving messages.
|
||||
"""
|
||||
def __init__(self, num_channels):
|
||||
self.pipes = [OneWayPipe() for _ in xrange(num_channels)]
|
||||
self.pipes = [OneWayPipe() for _ in range(num_channels)]
|
||||
self.channels = [ReceivingChannel(p) for p in self.pipes]
|
||||
self._readers = [p.reader for p in self.pipes]
|
||||
self._rlocks = [p.rlock for p in self.pipes]
|
||||
|
|
@ -276,7 +283,7 @@ class CopyTask(object):
|
|||
if not os.path.isfile(config_file):
|
||||
return opts
|
||||
|
||||
configs = ConfigParser.RawConfigParser()
|
||||
configs = configparser.RawConfigParser()
|
||||
configs.readfp(open(config_file))
|
||||
|
||||
ret = dict()
|
||||
|
|
@ -305,8 +312,8 @@ class CopyTask(object):
|
|||
"""
|
||||
Convert all option values to valid string literals unless they are path names
|
||||
"""
|
||||
return dict([(k, v.decode('string_escape') if k not in ['errfile', 'ratefile'] else v)
|
||||
for k, v, in opts.iteritems()])
|
||||
return dict([(k, v if k not in ['errfile', 'ratefile'] else v)
|
||||
for k, v, in opts.items()])
|
||||
|
||||
def parse_options(self, opts, direction):
|
||||
"""
|
||||
|
|
@ -319,9 +326,9 @@ class CopyTask(object):
|
|||
opts = self.clean_options(self.maybe_read_config_file(opts, direction))
|
||||
|
||||
dialect_options = dict()
|
||||
dialect_options['quotechar'] = opts.pop('quote', '"')
|
||||
dialect_options['escapechar'] = opts.pop('escape', '\\')
|
||||
dialect_options['delimiter'] = opts.pop('delimiter', ',')
|
||||
dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"'))
|
||||
dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\'))
|
||||
dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ','))
|
||||
if dialect_options['quotechar'] == dialect_options['escapechar']:
|
||||
dialect_options['doublequote'] = True
|
||||
del dialect_options['escapechar']
|
||||
|
|
@ -329,7 +336,7 @@ class CopyTask(object):
|
|||
dialect_options['doublequote'] = False
|
||||
|
||||
copy_options = dict()
|
||||
copy_options['nullval'] = opts.pop('null', '')
|
||||
copy_options['nullval'] = ensure_str(opts.pop('null', ''))
|
||||
copy_options['header'] = bool(opts.pop('header', '').lower() == 'true')
|
||||
copy_options['encoding'] = opts.pop('encoding', 'utf8')
|
||||
copy_options['maxrequests'] = int(opts.pop('maxrequests', 6))
|
||||
|
|
@ -351,7 +358,7 @@ class CopyTask(object):
|
|||
copy_options['consistencylevel'] = shell.consistency_level
|
||||
copy_options['decimalsep'] = opts.pop('decimalsep', '.')
|
||||
copy_options['thousandssep'] = opts.pop('thousandssep', '')
|
||||
copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')]
|
||||
copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')]
|
||||
copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16)))
|
||||
copy_options['begintoken'] = opts.pop('begintoken', '')
|
||||
copy_options['endtoken'] = opts.pop('endtoken', '')
|
||||
|
|
@ -494,7 +501,9 @@ class CopyTask(object):
|
|||
cql_version=shell.conn.cql_version,
|
||||
config_file=self.config_file,
|
||||
protocol_version=self.protocol_version,
|
||||
debug=shell.debug
|
||||
debug=shell.debug,
|
||||
coverage=shell.coverage,
|
||||
coveragerc_path=shell.coveragerc_path
|
||||
)
|
||||
|
||||
def validate_columns(self):
|
||||
|
|
@ -534,7 +543,7 @@ class ExportWriter(object):
|
|||
self.columns = columns
|
||||
self.options = options
|
||||
self.header = options.copy['header']
|
||||
self.max_output_size = long(options.copy['maxoutputsize'])
|
||||
self.max_output_size = int(options.copy['maxoutputsize'])
|
||||
self.current_dest = None
|
||||
self.num_files = 0
|
||||
|
||||
|
|
@ -577,10 +586,10 @@ class ExportWriter(object):
|
|||
return CsvDest(output=sys.stdout, close=False)
|
||||
else:
|
||||
try:
|
||||
ret = CsvDest(output=open(source_name, 'wb'), close=True)
|
||||
ret = CsvDest(output=open(source_name, 'w'), close=True)
|
||||
self.num_files += 1
|
||||
return ret
|
||||
except IOError, e:
|
||||
except IOError as e:
|
||||
self.shell.printerr("Can't open %r for writing: %s" % (source_name, e))
|
||||
return None
|
||||
|
||||
|
|
@ -604,7 +613,7 @@ class ExportWriter(object):
|
|||
if (self.num_written + num) > self.max_output_size:
|
||||
num_remaining = self.max_output_size - self.num_written
|
||||
last_switch = 0
|
||||
for i, row in enumerate(filter(None, data.split(os.linesep))):
|
||||
for i, row in enumerate([_f for _f in data.split(os.linesep) if _f]):
|
||||
if i == num_remaining:
|
||||
self._next_dest()
|
||||
last_switch = i
|
||||
|
|
@ -625,8 +634,8 @@ class ExportTask(CopyTask):
|
|||
CopyTask.__init__(self, shell, ks, table, columns, fname, opts, protocol_version, config_file, 'to')
|
||||
|
||||
options = self.options
|
||||
self.begin_token = long(options.copy['begintoken']) if options.copy['begintoken'] else None
|
||||
self.end_token = long(options.copy['endtoken']) if options.copy['endtoken'] else None
|
||||
self.begin_token = int(options.copy['begintoken']) if options.copy['begintoken'] else None
|
||||
self.end_token = int(options.copy['endtoken']) if options.copy['endtoken'] else None
|
||||
self.writer = ExportWriter(fname, shell, columns, options)
|
||||
|
||||
def run(self):
|
||||
|
|
@ -637,7 +646,7 @@ class ExportTask(CopyTask):
|
|||
shell = self.shell
|
||||
|
||||
if self.options.unrecognized:
|
||||
shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(self.options.unrecognized.keys()))
|
||||
shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(list(self.options.unrecognized.keys())))
|
||||
return
|
||||
|
||||
if not self.validate_columns():
|
||||
|
|
@ -650,11 +659,11 @@ class ExportTask(CopyTask):
|
|||
if not self.writer.open():
|
||||
return 0
|
||||
|
||||
columns = u"[" + u", ".join(self.columns) + u"]"
|
||||
self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
|
||||
columns = "[" + ", ".join(self.columns) + "]"
|
||||
self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
|
||||
|
||||
params = self.make_params()
|
||||
for i in xrange(self.num_processes):
|
||||
for i in range(self.num_processes):
|
||||
self.processes.append(ExportProcess(self.update_params(params, i)))
|
||||
|
||||
self.start_processes()
|
||||
|
|
@ -695,15 +704,15 @@ class ExportTask(CopyTask):
|
|||
"""
|
||||
ret = (prev, curr)
|
||||
if begin_token:
|
||||
if ret[1] < begin_token:
|
||||
if curr < begin_token:
|
||||
return None
|
||||
elif ret[0] < begin_token:
|
||||
ret = (begin_token, ret[1])
|
||||
elif (prev is None) or (prev < begin_token):
|
||||
ret = (begin_token, curr)
|
||||
|
||||
if end_token:
|
||||
if ret[0] > end_token:
|
||||
if (ret[0] is not None) and (ret[0] > end_token):
|
||||
return None
|
||||
elif ret[1] > end_token:
|
||||
elif (curr is not None) and (curr > end_token):
|
||||
ret = (ret[0], end_token)
|
||||
|
||||
return ret
|
||||
|
|
@ -730,7 +739,7 @@ class ExportTask(CopyTask):
|
|||
ranges[(begin_token, end_token)] = make_range_data()
|
||||
return ranges
|
||||
|
||||
ring = shell.get_ring(self.ks).items()
|
||||
ring = list(shell.get_ring(self.ks).items())
|
||||
ring.sort()
|
||||
|
||||
if not ring:
|
||||
|
|
@ -760,6 +769,9 @@ class ExportTask(CopyTask):
|
|||
# For the last ring interval we query the same replicas that hold the first token in the ring
|
||||
if previous is not None and (not end_token or previous < end_token):
|
||||
ranges[(previous, end_token)] = first_range_data
|
||||
elif previous is None and (not end_token or previous < end_token):
|
||||
previous = begin_token if begin_token else min_token
|
||||
ranges[(previous, end_token)] = first_range_data
|
||||
|
||||
if not ranges:
|
||||
shell.printerr('Found no ranges to query, check begin and end tokens: %s - %s' % (begin_token, end_token))
|
||||
|
|
@ -809,7 +821,7 @@ class ExportTask(CopyTask):
|
|||
total_requests = len(ranges)
|
||||
max_attempts = self.options.copy['maxattempts']
|
||||
|
||||
self.send_work(ranges, ranges.keys())
|
||||
self.send_work(ranges, list(ranges.keys()))
|
||||
|
||||
num_processes = len(processes)
|
||||
succeeded = 0
|
||||
|
|
@ -881,8 +893,8 @@ class FilesReader(object):
|
|||
"""
|
||||
def make_source(fname):
|
||||
try:
|
||||
return open(fname, 'rb')
|
||||
except IOError, e:
|
||||
return open(fname, 'r')
|
||||
except IOError as e:
|
||||
raise IOError("Can't open %r for reading: %s" % (fname, e))
|
||||
|
||||
for path in paths.split(','):
|
||||
|
|
@ -913,14 +925,14 @@ class FilesReader(object):
|
|||
self.close_current_source()
|
||||
while self.current_source is None:
|
||||
try:
|
||||
self.current_source = self.sources.next()
|
||||
self.current_source = next(self.sources)
|
||||
if self.current_source:
|
||||
self.num_sources += 1
|
||||
except StopIteration:
|
||||
return False
|
||||
|
||||
if self.header:
|
||||
self.current_source.next()
|
||||
next(self.current_source)
|
||||
|
||||
return True
|
||||
|
||||
|
|
@ -939,9 +951,9 @@ class FilesReader(object):
|
|||
return []
|
||||
|
||||
rows = []
|
||||
for i in xrange(min(max_rows, self.chunk_size)):
|
||||
for i in range(min(max_rows, self.chunk_size)):
|
||||
try:
|
||||
row = self.current_source.next()
|
||||
row = next(self.current_source)
|
||||
self.num_read += 1
|
||||
|
||||
if 0 <= self.max_rows < self.num_read:
|
||||
|
|
@ -955,7 +967,7 @@ class FilesReader(object):
|
|||
self.next_source()
|
||||
break
|
||||
|
||||
return filter(None, rows)
|
||||
return [_f for _f in rows if _f]
|
||||
|
||||
|
||||
class PipeReader(object):
|
||||
|
|
@ -977,7 +989,7 @@ class PipeReader(object):
|
|||
|
||||
def read_rows(self, max_rows):
|
||||
rows = []
|
||||
for i in xrange(min(max_rows, self.chunk_size)):
|
||||
for i in range(min(max_rows, self.chunk_size)):
|
||||
row = self.inpipe.recv()
|
||||
if row is None:
|
||||
self.exhausted = True
|
||||
|
|
@ -1143,14 +1155,14 @@ class ImportTask(CopyTask):
|
|||
shell = self.shell
|
||||
|
||||
if self.options.unrecognized:
|
||||
shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(self.options.unrecognized.keys()))
|
||||
shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(list(self.options.unrecognized.keys())))
|
||||
return
|
||||
|
||||
if not self.validate_columns():
|
||||
return 0
|
||||
|
||||
columns = u"[" + u", ".join(self.valid_columns) + u"]"
|
||||
self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
|
||||
columns = "[" + ", ".join(self.valid_columns) + "]"
|
||||
self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding)
|
||||
|
||||
try:
|
||||
params = self.make_params()
|
||||
|
|
@ -1172,8 +1184,8 @@ class ImportTask(CopyTask):
|
|||
if pr:
|
||||
profile_off(pr, file_name='parent_profile_%d.txt' % (os.getpid(),))
|
||||
|
||||
except Exception, exc:
|
||||
shell.printerr(unicode(exc))
|
||||
except Exception as exc:
|
||||
shell.printerr(str(exc))
|
||||
if shell.debug:
|
||||
traceback.print_exc()
|
||||
return 0
|
||||
|
|
@ -1195,7 +1207,7 @@ class ImportTask(CopyTask):
|
|||
|
||||
self.outmsg.channels[-1].send(None)
|
||||
if shell.tty:
|
||||
print
|
||||
print()
|
||||
|
||||
def import_records(self):
|
||||
"""
|
||||
|
|
@ -1280,7 +1292,7 @@ class FeedingProcess(mp.Process):
|
|||
A process that reads from import sources and sends chunks to worker processes.
|
||||
"""
|
||||
def __init__(self, inpipe, outpipe, worker_pipes, fname, options, parent_cluster):
|
||||
mp.Process.__init__(self, target=self.run)
|
||||
super(FeedingProcess, self).__init__(target=self.run)
|
||||
self.inpipe = inpipe
|
||||
self.outpipe = outpipe
|
||||
self.worker_pipes = worker_pipes
|
||||
|
|
@ -1329,7 +1341,7 @@ class FeedingProcess(mp.Process):
|
|||
reader = self.reader
|
||||
try:
|
||||
reader.start()
|
||||
except IOError, exc:
|
||||
except IOError as exc:
|
||||
self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message))
|
||||
|
||||
channels = self.worker_channels
|
||||
|
|
@ -1338,7 +1350,7 @@ class FeedingProcess(mp.Process):
|
|||
failed_attempts = 0
|
||||
|
||||
while not reader.exhausted:
|
||||
channels_eligible = filter(lambda c: c.num_pending() < max_pending_chunks, channels)
|
||||
channels_eligible = [c for c in channels if c.num_pending() < max_pending_chunks]
|
||||
if not channels_eligible:
|
||||
failed_attempts += 1
|
||||
delay = randint(1, pow(2, failed_attempts))
|
||||
|
|
@ -1358,7 +1370,7 @@ class FeedingProcess(mp.Process):
|
|||
rows = reader.read_rows(max_rows)
|
||||
if rows:
|
||||
sent += self.send_chunk(ch, rows)
|
||||
except Exception, exc:
|
||||
except Exception as exc:
|
||||
self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message))
|
||||
|
||||
if reader.exhausted:
|
||||
|
|
@ -1392,7 +1404,7 @@ class ChildProcess(mp.Process):
|
|||
"""
|
||||
|
||||
def __init__(self, params, target):
|
||||
mp.Process.__init__(self, target=target)
|
||||
super(ChildProcess, self).__init__(target=target)
|
||||
self.inpipe = params['inpipe']
|
||||
self.outpipe = params['outpipe']
|
||||
self.inmsg = None # must be initialized after fork on Windows
|
||||
|
|
@ -1425,6 +1437,12 @@ class ChildProcess(mp.Process):
|
|||
self.test_failures = json.loads(os.environ.get('CQLSH_COPY_TEST_FAILURES', ''))
|
||||
else:
|
||||
self.test_failures = None
|
||||
# attributes for coverage
|
||||
self.coverage = params['coverage']
|
||||
self.coveragerc_path = params['coveragerc_path']
|
||||
self.coverage_collection = None
|
||||
self.sigterm_handler = None
|
||||
self.sighup_handler = None
|
||||
|
||||
def on_fork(self):
|
||||
"""
|
||||
|
|
@ -1442,6 +1460,31 @@ class ChildProcess(mp.Process):
|
|||
self.inmsg.close()
|
||||
self.outmsg.close()
|
||||
|
||||
def start_coverage(self):
|
||||
import coverage
|
||||
self.coverage_collection = coverage.Coverage(config_file=self.coveragerc_path)
|
||||
self.coverage_collection.start()
|
||||
|
||||
# save current handlers for SIGTERM and SIGHUP
|
||||
self.sigterm_handler = signal.getsignal(signal.SIGTERM)
|
||||
self.sighup_handler = signal.getsignal(signal.SIGTERM)
|
||||
|
||||
def handle_sigterm():
|
||||
self.stop_coverage()
|
||||
self.close()
|
||||
self.terminate()
|
||||
|
||||
# set custom handler for SIGHUP and SIGTERM
|
||||
# needed to make sure coverage data is saved
|
||||
signal.signal(signal.SIGTERM, handle_sigterm)
|
||||
signal.signal(signal.SIGHUP, handle_sigterm)
|
||||
|
||||
def stop_coverage(self):
|
||||
self.coverage_collection.stop()
|
||||
self.coverage_collection.save()
|
||||
signal.signal(signal.SIGTERM, self.sigterm_handler)
|
||||
signal.signal(signal.SIGHUP, self.sighup_handler)
|
||||
|
||||
|
||||
class ExpBackoffRetryPolicy(RetryPolicy):
|
||||
"""
|
||||
|
|
@ -1547,9 +1590,13 @@ class ExportProcess(ChildProcess):
|
|||
self.options = options
|
||||
|
||||
def run(self):
|
||||
if self.coverage:
|
||||
self.start_coverage()
|
||||
try:
|
||||
self.inner_run()
|
||||
finally:
|
||||
if self.coverage:
|
||||
self.stop_coverage()
|
||||
self.close()
|
||||
|
||||
def inner_run(self):
|
||||
|
|
@ -1582,7 +1629,7 @@ class ExportProcess(ChildProcess):
|
|||
if print_traceback and sys.exc_info()[1] == err:
|
||||
traceback.print_exc()
|
||||
else:
|
||||
msg = unicode(err)
|
||||
msg = str(err)
|
||||
return msg
|
||||
|
||||
def report_error(self, err, token_range):
|
||||
|
|
@ -1606,7 +1653,7 @@ class ExportProcess(ChildProcess):
|
|||
self.attach_callbacks(token_range, future, session)
|
||||
|
||||
def num_requests(self):
|
||||
return sum(session.num_requests() for session in self.hosts_to_sessions.values())
|
||||
return sum(session.num_requests() for session in list(self.hosts_to_sessions.values()))
|
||||
|
||||
def get_session(self, hosts, token_range):
|
||||
"""
|
||||
|
|
@ -1626,7 +1673,7 @@ class ExportProcess(ChildProcess):
|
|||
for host in hosts:
|
||||
try:
|
||||
ret = self.connect(host)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
errors.append(self.get_error_message(e))
|
||||
|
||||
if ret:
|
||||
|
|
@ -1639,7 +1686,7 @@ class ExportProcess(ChildProcess):
|
|||
return None
|
||||
|
||||
def connect(self, host):
|
||||
if host in self.hosts_to_sessions.keys():
|
||||
if host in list(self.hosts_to_sessions.keys()):
|
||||
session = self.hosts_to_sessions[host]
|
||||
session.add_request()
|
||||
return session
|
||||
|
|
@ -1687,17 +1734,18 @@ class ExportProcess(ChildProcess):
|
|||
return # no rows in this range
|
||||
|
||||
try:
|
||||
output = StringIO()
|
||||
output = StringIO() if six.PY3 else BytesIO()
|
||||
writer = csv.writer(output, **self.options.dialect)
|
||||
|
||||
for row in rows:
|
||||
writer.writerow(map(self.format_value, row, cql_types))
|
||||
print("cqlshlib.copyutil.ExportProcess.write_rows_to_csv(): writing row")
|
||||
writer.writerow(list(map(self.format_value, row, cql_types)))
|
||||
|
||||
data = (output.getvalue(), len(rows))
|
||||
self.send((token_range, data))
|
||||
output.close()
|
||||
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
self.report_error(e, token_range)
|
||||
|
||||
def format_value(self, val, cqltype):
|
||||
|
|
@ -1712,15 +1760,16 @@ class ExportProcess(ChildProcess):
|
|||
if not hasattr(cqltype, 'precision'):
|
||||
cqltype.precision = self.double_precision if cqltype.type_name == 'double' else self.float_precision
|
||||
|
||||
return formatter(val, cqltype=cqltype,
|
||||
encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format,
|
||||
float_precision=cqltype.precision, nullval=self.nullval, quote=False,
|
||||
decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
|
||||
boolean_styles=self.boolean_styles)
|
||||
formatted = formatter(val, cqltype=cqltype,
|
||||
encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format,
|
||||
float_precision=cqltype.precision, nullval=self.nullval, quote=False,
|
||||
decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
|
||||
boolean_styles=self.boolean_styles)
|
||||
return formatted if six.PY3 else formatted.encode('utf8')
|
||||
|
||||
def close(self):
|
||||
ChildProcess.close(self)
|
||||
for session in self.hosts_to_sessions.values():
|
||||
for session in list(self.hosts_to_sessions.values()):
|
||||
session.shutdown()
|
||||
|
||||
def prepare_query(self, partition_key, token_range, attempts):
|
||||
|
|
@ -1782,6 +1831,22 @@ class ParseError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class ImmutableDict(frozenset):
|
||||
"""
|
||||
Immutable dictionary implementation to represent map types.
|
||||
We need to pass BoundStatement.bind() a dict() because it calls iteritems(),
|
||||
except we can't create a dict with another dict as the key, hence we use a class
|
||||
that adds iteritems to a frozen set of tuples (which is how dict are normally made
|
||||
immutable in python).
|
||||
Must be declared in the top level of the module to be available for pickling.
|
||||
"""
|
||||
iteritems = frozenset.__iter__
|
||||
|
||||
def items(self):
|
||||
for k, v in self.iteritems():
|
||||
yield k, v
|
||||
|
||||
|
||||
class ImportConversion(object):
|
||||
"""
|
||||
A class for converting strings to values when importing from csv, used by ImportProcess,
|
||||
|
|
@ -1840,7 +1905,7 @@ class ImportConversion(object):
|
|||
select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks),
|
||||
protect_name(parent.table),
|
||||
where_clause)
|
||||
return parent.session.prepare(select_query)
|
||||
return parent.session.prepare(ensure_str(select_query))
|
||||
|
||||
@staticmethod
|
||||
def unprotect(v):
|
||||
|
|
@ -1871,20 +1936,20 @@ class ImportConversion(object):
|
|||
return bytearray.fromhex(v[2:])
|
||||
|
||||
def convert_text(v, **_):
|
||||
return v
|
||||
return ensure_str(v)
|
||||
|
||||
def convert_uuid(v, **_):
|
||||
return UUID(v)
|
||||
|
||||
def convert_bool(v, **_):
|
||||
return True if v.lower() == self.boolean_styles[0].lower() else False
|
||||
return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False
|
||||
|
||||
def get_convert_integer_fcn(adapter=int):
|
||||
"""
|
||||
Return a slow and a fast integer conversion function depending on self.thousands_sep
|
||||
"""
|
||||
if self.thousands_sep:
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str('')))
|
||||
else:
|
||||
return lambda v, ct=cql_type: adapter(v)
|
||||
|
||||
|
|
@ -1892,12 +1957,14 @@ class ImportConversion(object):
|
|||
"""
|
||||
Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep
|
||||
"""
|
||||
empty_str = ensure_str('')
|
||||
dot_str = ensure_str('.')
|
||||
if self.thousands_sep and self.decimal_sep:
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '').replace(self.decimal_sep, '.'))
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str))
|
||||
elif self.thousands_sep:
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str))
|
||||
elif self.decimal_sep:
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, '.'))
|
||||
return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, dot_str))
|
||||
else:
|
||||
return lambda v, ct=cql_type: adapter(v)
|
||||
|
||||
|
|
@ -1949,14 +2016,20 @@ class ImportConversion(object):
|
|||
return ret
|
||||
|
||||
# this should match all possible CQL and CQLSH datetime formats
|
||||
p = re.compile("(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" + # YYYY-MM-DD[( |'T')]
|
||||
"(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" + # [HH:MM[:SS[.NNNNNN]]]
|
||||
"(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]]
|
||||
p = re.compile(r"(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" # YYYY-MM-DD[( |'T')]
|
||||
+ r"(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" # [HH:MM[:SS[.NNNNNN]]]
|
||||
+ r"(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]]
|
||||
|
||||
def convert_datetime(val, **_):
|
||||
try:
|
||||
tval = time.strptime(val, self.date_time_format)
|
||||
return timegm(tval) * 1e3 # scale seconds to millis for the raw value
|
||||
if six.PY2:
|
||||
# Python 2 implementation
|
||||
tval = time.strptime(val, self.date_time_format)
|
||||
return timegm(tval) * 1e3 # scale seconds to millis for the raw value
|
||||
else:
|
||||
# Python 3 implementation
|
||||
dtval = datetime.datetime.strptime(val, self.date_time_format)
|
||||
return dtval.timestamp() * 1000
|
||||
except ValueError:
|
||||
pass # if it's not in the default format we try CQL formats
|
||||
|
||||
|
|
@ -1986,7 +2059,7 @@ class ImportConversion(object):
|
|||
offset = -time.timezone
|
||||
|
||||
# scale seconds to millis for the raw value
|
||||
return ((timegm(tval) + offset) * 1e3) + milliseconds
|
||||
return ((timegm(tval) + offset) * 1000) + milliseconds
|
||||
|
||||
def convert_date(v, **_):
|
||||
return Date(v)
|
||||
|
|
@ -2005,16 +2078,12 @@ class ImportConversion(object):
|
|||
|
||||
def convert_map(val, ct=cql_type):
|
||||
"""
|
||||
We need to pass to BoundStatement.bind() a dict() because it calls iteritems(),
|
||||
except we can't create a dict with another dict as the key, hence we use a class
|
||||
that adds iteritems to a frozen set of tuples (which is how dict are normally made
|
||||
immutable in python).
|
||||
See ImmutableDict above for a discussion of why a special object is needed here.
|
||||
"""
|
||||
class ImmutableDict(frozenset):
|
||||
iteritems = frozenset.__iter__
|
||||
|
||||
split_format_str = ensure_str('{%s}')
|
||||
sep = ensure_str(':')
|
||||
return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1]))
|
||||
for v in [split('{%s}' % vv, sep=':') for vv in split(val)]))
|
||||
for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]))
|
||||
|
||||
def convert_user_type(val, ct=cql_type):
|
||||
"""
|
||||
|
|
@ -2025,7 +2094,9 @@ class ImportConversion(object):
|
|||
Also note that it is possible that the subfield names in the csv are in the
|
||||
wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959.
|
||||
"""
|
||||
vals = [v for v in [split('{%s}' % vv, sep=':') for vv in split(val)]]
|
||||
split_format_str = ensure_str('{%s}')
|
||||
sep = ensure_str(':')
|
||||
vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]]
|
||||
dict_vals = dict((unprotect(v[0]), v[1]) for v in vals)
|
||||
sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val())
|
||||
for n, t in zip(ct.fieldnames, ct.subtypes)]
|
||||
|
|
@ -2053,11 +2124,11 @@ class ImportConversion(object):
|
|||
'ascii': convert_text,
|
||||
'float': get_convert_decimal_fcn(),
|
||||
'double': get_convert_decimal_fcn(),
|
||||
'bigint': get_convert_integer_fcn(adapter=long),
|
||||
'bigint': get_convert_integer_fcn(adapter=int),
|
||||
'int': get_convert_integer_fcn(),
|
||||
'varint': get_convert_integer_fcn(),
|
||||
'inet': convert_text,
|
||||
'counter': get_convert_integer_fcn(adapter=long),
|
||||
'counter': get_convert_integer_fcn(adapter=int),
|
||||
'timestamp': convert_datetime,
|
||||
'timeuuid': convert_uuid,
|
||||
'date': convert_date,
|
||||
|
|
@ -2082,7 +2153,7 @@ class ImportConversion(object):
|
|||
or "NULL" otherwise. Note that for counters we never use prepared statements, so we
|
||||
only check is_counter when use_prepared_statements is false.
|
||||
"""
|
||||
return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL")
|
||||
return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL"))
|
||||
|
||||
def convert_row(self, row):
|
||||
"""
|
||||
|
|
@ -2102,7 +2173,7 @@ class ImportConversion(object):
|
|||
def convert(c, v):
|
||||
try:
|
||||
return c(v) if v != self.nullval else self.get_null_val()
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
# if we could not convert an empty string, then self.nullval has been set to a marker
|
||||
# because the user needs to import empty strings, except that the converters for some types
|
||||
# will fail to convert an empty string, in this case the null value should be inserted
|
||||
|
|
@ -2112,7 +2183,7 @@ class ImportConversion(object):
|
|||
|
||||
if self.debug:
|
||||
traceback.print_exc()
|
||||
raise ParseError("Failed to parse %s : %s" % (val, e.message))
|
||||
raise ParseError("Failed to parse %s : %s" % (v, e.message if hasattr(e, 'message') else str(e)))
|
||||
|
||||
return [convert(conv, val) for conv, val in zip(converters, row)]
|
||||
|
||||
|
|
@ -2149,6 +2220,7 @@ class ImportConversion(object):
|
|||
val = serialize(i, row[i])
|
||||
length = len(val)
|
||||
pk_values.append(struct.pack(">H%dsB" % length, length, val, 0))
|
||||
|
||||
return b"".join(pk_values)
|
||||
|
||||
if len(partition_key_indexes) == 1:
|
||||
|
|
@ -2204,7 +2276,7 @@ class TokenMap(object):
|
|||
|
||||
def filter_replicas(self, hosts):
|
||||
shuffled = tuple(sorted(hosts, key=lambda k: random.random()))
|
||||
return filter(lambda r: r.is_up is not False and r.datacenter == self.local_dc, shuffled) if hosts else ()
|
||||
return [r for r in shuffled if r.is_up is not False and r.datacenter == self.local_dc] if hosts else ()
|
||||
|
||||
|
||||
class FastTokenAwarePolicy(DCAwareRoundRobinPolicy):
|
||||
|
|
@ -2237,7 +2309,7 @@ class FastTokenAwarePolicy(DCAwareRoundRobinPolicy):
|
|||
return conn.in_flight < min(conn.max_request_id, self.max_inflight_messages)
|
||||
return True
|
||||
|
||||
for i in xrange(self.max_backoff_attempts):
|
||||
for i in range(self.max_backoff_attempts):
|
||||
for r in filter(replica_is_not_overloaded, replicas):
|
||||
yield r
|
||||
|
||||
|
|
@ -2269,7 +2341,7 @@ class ImportProcess(ChildProcess):
|
|||
ChildProcess.__init__(self, params=params, target=self.run)
|
||||
|
||||
self.skip_columns = params['skip_columns']
|
||||
self.valid_columns = [c.encode(self.encoding) for c in params['valid_columns']]
|
||||
self.valid_columns = [c for c in params['valid_columns']]
|
||||
self.skip_column_indexes = [i for i, c in enumerate(self.columns) if c in self.skip_columns]
|
||||
|
||||
options = params['options']
|
||||
|
|
@ -2312,6 +2384,9 @@ class ImportProcess(ChildProcess):
|
|||
return self._session
|
||||
|
||||
def run(self):
|
||||
if self.coverage:
|
||||
self.start_coverage()
|
||||
|
||||
try:
|
||||
pr = profile_on() if PROFILE_ON else None
|
||||
|
||||
|
|
@ -2321,10 +2396,12 @@ class ImportProcess(ChildProcess):
|
|||
if pr:
|
||||
profile_off(pr, file_name='worker_profile_%d.txt' % (os.getpid(),))
|
||||
|
||||
except Exception, exc:
|
||||
except Exception as exc:
|
||||
self.report_error(exc)
|
||||
|
||||
finally:
|
||||
if self.coverage:
|
||||
self.stop_coverage()
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
|
|
@ -2361,6 +2438,7 @@ class ImportProcess(ChildProcess):
|
|||
if self.ttl >= 0:
|
||||
query += 'USING TTL %s' % (self.ttl,)
|
||||
make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement)
|
||||
query = ensure_str(query)
|
||||
|
||||
conv = ImportConversion(self, table_meta, prepared_statement)
|
||||
tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session)
|
||||
|
|
@ -2399,15 +2477,15 @@ class ImportProcess(ChildProcess):
|
|||
# causes the statement to be None, then we should not report the error so that we can test
|
||||
# the parent process handling missing batches from child processes
|
||||
|
||||
except Exception, exc:
|
||||
except Exception as exc:
|
||||
self.report_error(exc, chunk, chunk['rows'])
|
||||
|
||||
def wrap_make_statement(self, inner_make_statement):
|
||||
def make_statement(query, conv, chunk, batch, replicas):
|
||||
try:
|
||||
return inner_make_statement(query, conv, batch, replicas)
|
||||
except Exception, exc:
|
||||
print "Failed to make batch statement: {}".format(exc)
|
||||
except Exception as exc:
|
||||
print("Failed to make batch statement: {}".format(exc))
|
||||
self.report_error(exc, chunk, batch['rows'])
|
||||
return None
|
||||
|
||||
|
|
@ -2428,12 +2506,12 @@ class ImportProcess(ChildProcess):
|
|||
set_clause = []
|
||||
for i, value in enumerate(row):
|
||||
if i in conv.primary_key_indexes:
|
||||
where_clause.append("%s=%s" % (self.valid_columns[i], value))
|
||||
where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value)))
|
||||
else:
|
||||
set_clause.append("%s=%s+%s" % (self.valid_columns[i], self.valid_columns[i], value))
|
||||
set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value)))
|
||||
|
||||
full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause))
|
||||
statement.add(full_query_text)
|
||||
full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause))
|
||||
statement.add(ensure_str(full_query_text))
|
||||
return statement
|
||||
|
||||
def make_prepared_batch_statement(self, query, _, batch, replicas):
|
||||
|
|
@ -2457,7 +2535,8 @@ class ImportProcess(ChildProcess):
|
|||
statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level)
|
||||
statement.replicas = replicas
|
||||
statement.keyspace = self.ks
|
||||
statement._statements_and_parameters = [(False, query % (','.join(r),), ()) for r in batch['rows']]
|
||||
field_sep = b',' if six.PY2 else ','
|
||||
statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']]
|
||||
return statement
|
||||
|
||||
def convert_rows(self, conv, chunk):
|
||||
|
|
@ -2477,14 +2556,14 @@ class ImportProcess(ChildProcess):
|
|||
def convert_row(r):
|
||||
try:
|
||||
return conv.convert_row(r)
|
||||
except Exception, err:
|
||||
errors[err.message].append(r)
|
||||
except Exception as err:
|
||||
errors[err.message if hasattr(err, 'message') else str(err)].append(r)
|
||||
return None
|
||||
|
||||
converted_rows = filter(None, [convert_row(r) for r in rows])
|
||||
converted_rows = [_f for _f in [convert_row(r) for r in rows] if _f]
|
||||
|
||||
if errors:
|
||||
for msg, rows in errors.iteritems():
|
||||
for msg, rows in errors.items():
|
||||
self.report_error(ParseError(msg), chunk, rows)
|
||||
return converted_rows
|
||||
|
||||
|
|
@ -2549,27 +2628,27 @@ class ImportProcess(ChildProcess):
|
|||
try:
|
||||
pk = get_row_partition_key_values(row)
|
||||
rows_by_ring_pos[get_ring_pos(ring, pk_to_token_value(pk))].append(row)
|
||||
except Exception, e:
|
||||
except Exception as e:
|
||||
errors[e.message].append(row)
|
||||
|
||||
if errors:
|
||||
for msg, rows in errors.iteritems():
|
||||
for msg, rows in errors.items():
|
||||
self.report_error(ParseError(msg), chunk, rows)
|
||||
|
||||
replicas = tm.replicas
|
||||
filter_replicas = tm.filter_replicas
|
||||
rows_by_replica = defaultdict(list)
|
||||
for ring_pos, rows in rows_by_ring_pos.iteritems():
|
||||
for ring_pos, rows in rows_by_ring_pos.items():
|
||||
if len(rows) > min_batch_size:
|
||||
for i in xrange(0, len(rows), max_batch_size):
|
||||
for i in range(0, len(rows), max_batch_size):
|
||||
yield filter_replicas(replicas[ring_pos]), make_batch(chunk['id'], rows[i:i + max_batch_size])
|
||||
else:
|
||||
# select only the first valid replica to guarantee more overlap or none at all
|
||||
rows_by_replica[filter_replicas(replicas[ring_pos])[:1]].extend(rows)
|
||||
rows_by_replica[tuple(filter_replicas(replicas[ring_pos])[:1])].extend(rows) # TODO: revisit tuple wrapper
|
||||
|
||||
# Now send the batches by replica
|
||||
for replicas, rows in rows_by_replica.iteritems():
|
||||
for i in xrange(0, len(rows), max_batch_size):
|
||||
for replicas, rows in rows_by_replica.items():
|
||||
for i in range(0, len(rows), max_batch_size):
|
||||
yield replicas, make_batch(chunk['id'], rows[i:i + max_batch_size])
|
||||
|
||||
def result_callback(self, _, batch, chunk):
|
||||
|
|
@ -2590,7 +2669,8 @@ class ImportProcess(ChildProcess):
|
|||
def report_error(self, err, chunk=None, rows=None, attempts=1, final=True):
|
||||
if self.debug and sys.exc_info()[1] == err:
|
||||
traceback.print_exc()
|
||||
self.outmsg.send(ImportTaskError(err.__class__.__name__, err.message, rows, attempts, final))
|
||||
err_msg = err.message if hasattr(err, 'message') else str(err)
|
||||
self.outmsg.send(ImportTaskError(err.__class__.__name__, err_msg, rows, attempts, final))
|
||||
if final and chunk is not None:
|
||||
self.update_chunk(rows, chunk)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,15 +14,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from .cqlhandling import CqlParsingRuleSet, Hint
|
||||
from cassandra.metadata import maybe_escape_name
|
||||
|
||||
from cqlshlib import helptopics
|
||||
from cqlshlib.cqlhandling import CqlParsingRuleSet, Hint
|
||||
|
||||
simple_cql_types = set(('ascii', 'bigint', 'blob', 'boolean', 'counter', 'date', 'decimal', 'double', 'duration', 'float',
|
||||
'inet', 'int', 'smallint', 'text', 'time', 'timestamp', 'timeuuid', 'tinyint', 'uuid', 'varchar', 'varint'))
|
||||
simple_cql_types.difference_update(('set', 'map', 'list'))
|
||||
|
||||
from . import helptopics
|
||||
cqldocs = helptopics.CQL3HelpTopics()
|
||||
|
||||
|
||||
|
|
@ -442,8 +441,8 @@ def ks_prop_val_mapkey_completer(ctxt, cass):
|
|||
optname = ctxt.get_binding('propname')[-1]
|
||||
if optname != 'replication':
|
||||
return ()
|
||||
keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
|
||||
valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
|
||||
keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
|
||||
valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
|
||||
for k, v in zip(keysseen, valsseen):
|
||||
if k == 'class':
|
||||
repclass = v
|
||||
|
|
@ -454,7 +453,7 @@ def ks_prop_val_mapkey_completer(ctxt, cass):
|
|||
opts = set(('replication_factor',))
|
||||
elif repclass == 'NetworkTopologyStrategy':
|
||||
return [Hint('<dc_name>')]
|
||||
return map(escape_value, opts.difference(keysseen))
|
||||
return list(map(escape_value, opts.difference(keysseen)))
|
||||
|
||||
|
||||
def ks_prop_val_mapval_completer(ctxt, cass):
|
||||
|
|
@ -463,7 +462,7 @@ def ks_prop_val_mapval_completer(ctxt, cass):
|
|||
return ()
|
||||
currentkey = dequote_value(ctxt.get_binding('propmapkey')[-1])
|
||||
if currentkey == 'class':
|
||||
return map(escape_value, CqlRuleSet.replication_strategies)
|
||||
return list(map(escape_value, CqlRuleSet.replication_strategies))
|
||||
return [Hint('<term>')]
|
||||
|
||||
|
||||
|
|
@ -471,8 +470,8 @@ def ks_prop_val_mapender_completer(ctxt, cass):
|
|||
optname = ctxt.get_binding('propname')[-1]
|
||||
if optname != 'replication':
|
||||
return [',']
|
||||
keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
|
||||
valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
|
||||
keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
|
||||
valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
|
||||
for k, v in zip(keysseen, valsseen):
|
||||
if k == 'class':
|
||||
repclass = v
|
||||
|
|
@ -488,8 +487,8 @@ def ks_prop_val_mapender_completer(ctxt, cass):
|
|||
|
||||
|
||||
def cf_prop_name_completer(ctxt, cass):
|
||||
return [c[0] for c in (CqlRuleSet.columnfamily_layout_options +
|
||||
CqlRuleSet.columnfamily_layout_map_options)]
|
||||
return [c[0] for c in (CqlRuleSet.columnfamily_layout_options
|
||||
+ CqlRuleSet.columnfamily_layout_map_options)]
|
||||
|
||||
|
||||
def cf_prop_val_completer(ctxt, cass):
|
||||
|
|
@ -522,13 +521,13 @@ def cf_prop_val_mapkey_completer(ctxt, cass):
|
|||
break
|
||||
else:
|
||||
return ()
|
||||
keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ()))
|
||||
valsseen = map(dequote_value, ctxt.get_binding('propmapval', ()))
|
||||
pairsseen = dict(zip(keysseen, valsseen))
|
||||
keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ())))
|
||||
valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ())))
|
||||
pairsseen = dict(list(zip(keysseen, valsseen)))
|
||||
if optname == 'compression':
|
||||
return map(escape_value, set(subopts).difference(keysseen))
|
||||
return list(map(escape_value, set(subopts).difference(keysseen)))
|
||||
if optname == 'caching':
|
||||
return map(escape_value, set(subopts).difference(keysseen))
|
||||
return list(map(escape_value, set(subopts).difference(keysseen)))
|
||||
if optname == 'compaction':
|
||||
opts = set(subopts)
|
||||
try:
|
||||
|
|
@ -545,7 +544,7 @@ def cf_prop_val_mapkey_completer(ctxt, cass):
|
|||
elif csc == 'TimeWindowCompactionStrategy':
|
||||
opts = opts.union(set(CqlRuleSet.time_window_compaction_strategy_options))
|
||||
|
||||
return map(escape_value, opts)
|
||||
return list(map(escape_value, opts))
|
||||
return ()
|
||||
|
||||
|
||||
|
|
@ -554,11 +553,11 @@ def cf_prop_val_mapval_completer(ctxt, cass):
|
|||
key = dequote_value(ctxt.get_binding('propmapkey')[-1])
|
||||
if opt == 'compaction':
|
||||
if key == 'class':
|
||||
return map(escape_value, CqlRuleSet.available_compaction_classes)
|
||||
return list(map(escape_value, CqlRuleSet.available_compaction_classes))
|
||||
return [Hint('<option_value>')]
|
||||
elif opt == 'compression':
|
||||
if key == 'sstable_compression':
|
||||
return map(escape_value, CqlRuleSet.available_compression_classes)
|
||||
return list(map(escape_value, CqlRuleSet.available_compression_classes))
|
||||
return [Hint('<option_value>')]
|
||||
elif opt == 'caching':
|
||||
if key == 'rows_per_partition':
|
||||
|
|
@ -584,19 +583,19 @@ def storagetype_completer(ctxt, cass):
|
|||
|
||||
@completer_for('keyspaceName', 'ksname')
|
||||
def ks_name_completer(ctxt, cass):
|
||||
return map(maybe_escape_name, cass.get_keyspace_names())
|
||||
return list(map(maybe_escape_name, cass.get_keyspace_names()))
|
||||
|
||||
|
||||
@completer_for('nonSystemKeyspaceName', 'ksname')
|
||||
def non_system_ks_name_completer(ctxt, cass):
|
||||
ksnames = [n for n in cass.get_keyspace_names() if n not in SYSTEM_KEYSPACES]
|
||||
return map(maybe_escape_name, ksnames)
|
||||
return list(map(maybe_escape_name, ksnames))
|
||||
|
||||
|
||||
@completer_for('alterableKeyspaceName', 'ksname')
|
||||
def alterable_ks_name_completer(ctxt, cass):
|
||||
ksnames = [n for n in cass.get_keyspace_names() if n not in NONALTERBALE_KEYSPACES]
|
||||
return map(maybe_escape_name, ksnames)
|
||||
return list(map(maybe_escape_name, ksnames))
|
||||
|
||||
|
||||
def cf_ks_name_completer(ctxt, cass):
|
||||
|
|
@ -629,7 +628,7 @@ def cf_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, cfnames)
|
||||
return list(map(maybe_escape_name, cfnames))
|
||||
|
||||
|
||||
@completer_for('materializedViewName', 'mvname')
|
||||
|
|
@ -643,7 +642,7 @@ def mv_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, mvnames)
|
||||
return list(map(maybe_escape_name, mvnames))
|
||||
|
||||
|
||||
completer_for('userTypeName', 'ksname')(cf_ks_name_completer)
|
||||
|
|
@ -661,7 +660,7 @@ def ut_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, utnames)
|
||||
return list(map(maybe_escape_name, utnames))
|
||||
|
||||
|
||||
completer_for('userTypeName', 'utname')(ut_name_completer)
|
||||
|
|
@ -752,7 +751,7 @@ def udf_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, udfnames)
|
||||
return list(map(maybe_escape_name, udfnames))
|
||||
|
||||
|
||||
def uda_name_completer(ctxt, cass):
|
||||
|
|
@ -765,7 +764,7 @@ def uda_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, udanames)
|
||||
return list(map(maybe_escape_name, udanames))
|
||||
|
||||
|
||||
def udf_uda_name_completer(ctxt, cass):
|
||||
|
|
@ -778,7 +777,7 @@ def udf_uda_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, functionnames)
|
||||
return list(map(maybe_escape_name, functionnames))
|
||||
|
||||
|
||||
def ref_udf_name_completer(ctxt, cass):
|
||||
|
|
@ -786,7 +785,7 @@ def ref_udf_name_completer(ctxt, cass):
|
|||
udanames = cass.get_userfunction_names(None)
|
||||
except Exception:
|
||||
return ()
|
||||
return map(maybe_escape_name, udanames)
|
||||
return list(map(maybe_escape_name, udanames))
|
||||
|
||||
|
||||
completer_for('functionAggregateName', 'ksname')(cf_ks_name_completer)
|
||||
|
|
@ -844,7 +843,7 @@ def relation_token_subject_completer(ctxt, cass):
|
|||
def select_relation_lhs_completer(ctxt, cass):
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
filterable = set()
|
||||
already_filtered_on = map(dequote_name, ctxt.get_binding('rel_lhs', ()))
|
||||
already_filtered_on = list(map(dequote_name, ctxt.get_binding('rel_lhs', ())))
|
||||
for num in range(0, len(layout.partition_key)):
|
||||
if num == 0 or layout.partition_key[num - 1].name in already_filtered_on:
|
||||
filterable.add(layout.partition_key[num].name)
|
||||
|
|
@ -855,9 +854,9 @@ def select_relation_lhs_completer(ctxt, cass):
|
|||
filterable.add(layout.clustering_key[num].name)
|
||||
else:
|
||||
break
|
||||
for idx in layout.indexes.itervalues():
|
||||
for idx in layout.indexes.values():
|
||||
filterable.add(idx.index_options["target"])
|
||||
return map(maybe_escape_name, filterable)
|
||||
return list(map(maybe_escape_name, filterable))
|
||||
|
||||
|
||||
explain_completion('selector', 'colname')
|
||||
|
|
@ -880,9 +879,9 @@ syntax_rules += r'''
|
|||
def regular_column_names(table_meta):
|
||||
if not table_meta or not table_meta.columns:
|
||||
return []
|
||||
regular_columns = list(set(table_meta.columns.keys()) -
|
||||
set([key.name for key in table_meta.partition_key]) -
|
||||
set([key.name for key in table_meta.clustering_key]))
|
||||
regular_columns = list(set(table_meta.columns.keys())
|
||||
- set([key.name for key in table_meta.partition_key])
|
||||
- set([key.name for key in table_meta.clustering_key]))
|
||||
return regular_columns
|
||||
|
||||
|
||||
|
|
@ -895,13 +894,13 @@ def insert_colname_completer(ctxt, cass):
|
|||
if k.name not in colnames:
|
||||
return [maybe_escape_name(k.name)]
|
||||
normalcols = set(regular_column_names(layout)) - colnames
|
||||
return map(maybe_escape_name, normalcols)
|
||||
return list(map(maybe_escape_name, normalcols))
|
||||
|
||||
|
||||
@completer_for('insertStatement', 'newval')
|
||||
def insert_newval_completer(ctxt, cass):
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
insertcols = map(dequote_name, ctxt.get_binding('colname'))
|
||||
insertcols = list(map(dequote_name, ctxt.get_binding('colname')))
|
||||
valuesdone = ctxt.get_binding('newval', ())
|
||||
if len(valuesdone) >= len(insertcols):
|
||||
return []
|
||||
|
|
@ -974,7 +973,7 @@ def update_option_completer(ctxt, cass):
|
|||
@completer_for('assignment', 'updatecol')
|
||||
def update_col_completer(ctxt, cass):
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
return map(maybe_escape_name, regular_column_names(layout))
|
||||
return list(map(maybe_escape_name, regular_column_names(layout)))
|
||||
|
||||
|
||||
@completer_for('assignment', 'update_rhs')
|
||||
|
|
@ -1116,7 +1115,7 @@ def delete_opt_completer(ctxt, cass):
|
|||
@completer_for('deleteSelector', 'delcol')
|
||||
def delete_delcol_completer(ctxt, cass):
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
return map(maybe_escape_name, regular_column_names(layout))
|
||||
return list(map(maybe_escape_name, regular_column_names(layout)))
|
||||
|
||||
|
||||
syntax_rules += r'''
|
||||
|
|
@ -1197,7 +1196,7 @@ syntax_rules += r'''
|
|||
|
||||
@completer_for('cfamOrdering', 'ordercol')
|
||||
def create_cf_clustering_order_colname_completer(ctxt, cass):
|
||||
colnames = map(dequote_name, ctxt.get_binding('newcolname', ()))
|
||||
colnames = list(map(dequote_name, ctxt.get_binding('newcolname', ())))
|
||||
# Definitely some of these aren't valid for ordering, but I'm not sure
|
||||
# precisely which are. This is good enough for now
|
||||
return colnames
|
||||
|
|
@ -1227,7 +1226,7 @@ def create_cf_ks_dot_completer(ctxt, cass):
|
|||
def create_cf_pkdef_declaration_completer(ctxt, cass):
|
||||
cols_declared = ctxt.get_binding('newcolname')
|
||||
pieces_already = ctxt.get_binding('ptkey', ())
|
||||
pieces_already = map(dequote_name, pieces_already)
|
||||
pieces_already = list(map(dequote_name, pieces_already))
|
||||
while cols_declared[0] in pieces_already:
|
||||
cols_declared = cols_declared[1:]
|
||||
if len(cols_declared) < 2:
|
||||
|
|
@ -1239,7 +1238,7 @@ def create_cf_pkdef_declaration_completer(ctxt, cass):
|
|||
def create_cf_composite_key_declaration_completer(ctxt, cass):
|
||||
cols_declared = ctxt.get_binding('newcolname')
|
||||
pieces_already = ctxt.get_binding('ptkey', ()) + ctxt.get_binding('pkey', ())
|
||||
pieces_already = map(dequote_name, pieces_already)
|
||||
pieces_already = list(map(dequote_name, pieces_already))
|
||||
while cols_declared[0] in pieces_already:
|
||||
cols_declared = cols_declared[1:]
|
||||
if len(cols_declared) < 2:
|
||||
|
|
@ -1325,9 +1324,9 @@ explain_completion('createUserTypeStatement', 'newcol', '<new_field_name>')
|
|||
def create_index_col_completer(ctxt, cass):
|
||||
""" Return the columns for which an index doesn't exist yet. """
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
idx_targets = [idx.index_options["target"] for idx in layout.indexes.itervalues()]
|
||||
colnames = [cd.name for cd in layout.columns.values() if cd.name not in idx_targets]
|
||||
return map(maybe_escape_name, colnames)
|
||||
idx_targets = [idx.index_options["target"] for idx in layout.indexes.values()]
|
||||
colnames = [cd.name for cd in list(layout.columns.values()) if cd.name not in idx_targets]
|
||||
return list(map(maybe_escape_name, colnames))
|
||||
|
||||
|
||||
syntax_rules += r'''
|
||||
|
|
@ -1385,7 +1384,7 @@ def idx_ks_idx_name_completer(ctxt, cass):
|
|||
if ks is None:
|
||||
return ()
|
||||
raise
|
||||
return map(maybe_escape_name, idxnames)
|
||||
return list(map(maybe_escape_name, idxnames))
|
||||
|
||||
|
||||
syntax_rules += r'''
|
||||
|
|
@ -1413,14 +1412,14 @@ syntax_rules += r'''
|
|||
def alter_table_col_completer(ctxt, cass):
|
||||
layout = get_table_meta(ctxt, cass)
|
||||
cols = [str(md) for md in layout.columns]
|
||||
return map(maybe_escape_name, cols)
|
||||
return list(map(maybe_escape_name, cols))
|
||||
|
||||
|
||||
@completer_for('alterTypeInstructions', 'existcol')
|
||||
def alter_type_field_completer(ctxt, cass):
|
||||
layout = get_ut_layout(ctxt, cass)
|
||||
fields = [tuple[0] for tuple in layout]
|
||||
return map(maybe_escape_name, fields)
|
||||
return list(map(maybe_escape_name, fields))
|
||||
|
||||
|
||||
explain_completion('alterInstructions', 'newcol', '<new_column_name>')
|
||||
|
|
@ -1557,7 +1556,7 @@ def username_name_completer(ctxt, cass):
|
|||
return [Hint('<username>')]
|
||||
|
||||
session = cass.session
|
||||
return [maybe_quote(row.values()[0].replace("'", "''")) for row in session.execute("LIST USERS")]
|
||||
return [maybe_quote(list(row.values())[0].replace("'", "''")) for row in session.execute("LIST USERS")]
|
||||
|
||||
|
||||
@completer_for('rolename', 'role')
|
||||
|
|
@ -1596,7 +1595,7 @@ def get_trigger_names(ctxt, cass):
|
|||
@completer_for('dropTriggerStatement', 'triggername')
|
||||
def drop_trigger_completer(ctxt, cass):
|
||||
names = get_trigger_names(ctxt, cass)
|
||||
return map(maybe_escape_name, names)
|
||||
return list(map(maybe_escape_name, names))
|
||||
|
||||
|
||||
# END SYNTAX/COMPLETION RULE DEFINITIONS
|
||||
|
|
|
|||
|
|
@ -18,8 +18,9 @@
|
|||
# i.e., stuff that's not necessarily cqlsh-specific
|
||||
|
||||
import traceback
|
||||
|
||||
from cassandra.metadata import cql_keywords_reserved
|
||||
from . import pylexotron, util
|
||||
from cqlshlib import pylexotron, util
|
||||
|
||||
Hint = pylexotron.Hint
|
||||
|
||||
|
|
@ -66,7 +67,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
if cass is None:
|
||||
return ()
|
||||
return f(ctxt, cass)
|
||||
completerwrapper.func_name = 'completerwrapper_on_' + f.func_name
|
||||
completerwrapper.__name__ = 'completerwrapper_on_' + f.__name__
|
||||
self.register_completer(completerwrapper, rulename, symname)
|
||||
return completerwrapper
|
||||
return registrator
|
||||
|
|
@ -100,12 +101,6 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
# operations on tokens (like .lower()). See CASSANDRA-9083
|
||||
# for one example of this.
|
||||
str_token = t[1]
|
||||
if isinstance(str_token, unicode):
|
||||
try:
|
||||
str_token = str_token.encode('ascii')
|
||||
t = (t[0], str_token) + t[2:]
|
||||
except UnicodeEncodeError:
|
||||
pass
|
||||
|
||||
curstmt.append(t)
|
||||
if t[0] == 'endtoken':
|
||||
|
|
@ -196,7 +191,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
f = lambda s: s and dequoter(s).lower().startswith(partial)
|
||||
else:
|
||||
f = lambda s: s and dequoter(s).startswith(partial)
|
||||
candidates = filter(f, strcompletes)
|
||||
candidates = list(filter(f, strcompletes))
|
||||
|
||||
if prefix is not None:
|
||||
# dequote, re-escape, strip quotes: gets us the right quoted text
|
||||
|
|
@ -207,7 +202,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
|
||||
# the above process can result in an empty string; this doesn't help for
|
||||
# completions
|
||||
candidates = filter(None, candidates)
|
||||
candidates = [_f for _f in candidates if _f]
|
||||
|
||||
# prefix a space when desirable for pleasant cql formatting
|
||||
if tokens:
|
||||
|
|
@ -249,7 +244,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
init_bindings = {'cassandra_conn': cassandra_conn}
|
||||
if debug:
|
||||
init_bindings['*DEBUG*'] = True
|
||||
print "cql_complete(%r, partial=%r)" % (text, partial)
|
||||
print("cql_complete(%r, partial=%r)" % (text, partial))
|
||||
|
||||
completions, hints = self.cql_complete_single(text, partial, init_bindings,
|
||||
startsymbol=startsymbol)
|
||||
|
|
@ -261,12 +256,12 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
if len(completions) == 1 and len(hints) == 0:
|
||||
c = completions[0]
|
||||
if debug:
|
||||
print "** Got one completion: %r. Checking for further matches...\n" % (c,)
|
||||
print("** Got one completion: %r. Checking for further matches...\n" % (c,))
|
||||
if not c.isspace():
|
||||
new_c = self.cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol)
|
||||
completions = [new_c]
|
||||
if debug:
|
||||
print "** New list of completions: %r" % (completions,)
|
||||
print("** New list of completions: %r" % (completions,))
|
||||
|
||||
return hints + completions
|
||||
|
||||
|
|
@ -277,18 +272,18 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
startsymbol=startsymbol)
|
||||
except Exception:
|
||||
if debug:
|
||||
print "** completion expansion had a problem:"
|
||||
print("** completion expansion had a problem:")
|
||||
traceback.print_exc()
|
||||
return first
|
||||
if hints:
|
||||
if not first[-1].isspace():
|
||||
first += ' '
|
||||
if debug:
|
||||
print "** completion expansion found hints: %r" % (hints,)
|
||||
print("** completion expansion found hints: %r" % (hints,))
|
||||
return first
|
||||
if len(completions) == 1 and completions[0] != '':
|
||||
if debug:
|
||||
print "** Got another completion: %r." % (completions[0],)
|
||||
print("** Got another completion: %r." % (completions[0],))
|
||||
if completions[0][0] in (',', ')', ':') and first[-1] == ' ':
|
||||
first = first[:-1]
|
||||
first += completions[0]
|
||||
|
|
@ -299,10 +294,10 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
|
|||
if common_prefix[0] in (',', ')', ':') and first[-1] == ' ':
|
||||
first = first[:-1]
|
||||
if debug:
|
||||
print "** Got a partial completion: %r." % (common_prefix,)
|
||||
print("** Got a partial completion: %r." % (common_prefix,))
|
||||
return first + common_prefix
|
||||
if debug:
|
||||
print "** New total completion: %r. Checking for further matches...\n" % (first,)
|
||||
print("** New total completion: %r. Checking for further matches...\n" % (first,))
|
||||
return self.cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import cqlhandling
|
||||
|
||||
from cqlshlib import cqlhandling
|
||||
|
||||
# we want the cql parser to understand our cqlsh-specific commands too
|
||||
my_commands_ending_with_newline = (
|
||||
|
|
@ -240,7 +241,7 @@ def complete_source_quoted_filename(ctxt, cqlsh):
|
|||
contents = os.listdir(exhead or '.')
|
||||
except OSError:
|
||||
return ()
|
||||
matches = filter(lambda f: f.startswith(tail), contents)
|
||||
matches = [f for f in contents if f.startswith(tail)]
|
||||
annotated = []
|
||||
for f in matches:
|
||||
match = os.path.join(head, f)
|
||||
|
|
@ -267,7 +268,7 @@ def copy_fname_completer(ctxt, cqlsh):
|
|||
|
||||
@cqlsh_syntax_completer('copyCommand', 'colnames')
|
||||
def complete_copy_column_names(ctxt, cqlsh):
|
||||
existcols = map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ()))
|
||||
existcols = list(map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ())))
|
||||
ks = cqlsh.cql_unprotect_name(ctxt.get_binding('ksname', None))
|
||||
cf = cqlsh.cql_unprotect_name(ctxt.get_binding('cfname'))
|
||||
colnames = cqlsh.get_column_names(ks, cf)
|
||||
|
|
@ -288,7 +289,7 @@ COPY_TO_OPTIONS = ['ENCODING', 'PAGESIZE', 'PAGETIMEOUT', 'BEGINTOKEN', 'ENDTOKE
|
|||
|
||||
@cqlsh_syntax_completer('copyOption', 'optnames')
|
||||
def complete_copy_options(ctxt, cqlsh):
|
||||
optnames = map(str.upper, ctxt.get_binding('optnames', ()))
|
||||
optnames = list(map(str.upper, ctxt.get_binding('optnames', ())))
|
||||
direction = ctxt.get_binding('dir').upper()
|
||||
if direction == 'FROM':
|
||||
opts = set(COPY_COMMON_OPTIONS + COPY_FROM_OPTIONS) - set(optnames)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
RED = '\033[0;1;31m'
|
||||
|
|
@ -41,7 +43,7 @@ def get_str(val):
|
|||
return val
|
||||
|
||||
|
||||
class FormattedValue:
|
||||
class FormattedValue(object):
|
||||
|
||||
def __init__(self, strval, coloredval=None, displaywidth=None):
|
||||
self.strval = strval
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import binascii
|
||||
import calendar
|
||||
import datetime
|
||||
|
|
@ -21,15 +23,17 @@ import math
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import six
|
||||
import platform
|
||||
import wcwidth
|
||||
|
||||
from six import ensure_text
|
||||
|
||||
from collections import defaultdict
|
||||
from displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP
|
||||
|
||||
from cassandra.cqltypes import EMPTY
|
||||
from cassandra.util import datetime_from_timestamp
|
||||
from util import UTC
|
||||
from . import wcwidth
|
||||
from .displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP
|
||||
from .util import UTC
|
||||
|
||||
is_win = platform.system() == 'Windows'
|
||||
|
||||
|
|
@ -204,7 +208,7 @@ class CqlType(object):
|
|||
|
||||
|
||||
def format_value_default(val, colormap, **_):
|
||||
val = str(val)
|
||||
val = ensure_text(str(val))
|
||||
escapedval = val.replace('\\', '\\\\')
|
||||
bval = controlchars_re.sub(_show_control_chars, escapedval)
|
||||
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap)
|
||||
|
|
@ -218,7 +222,6 @@ _formatters = {}
|
|||
def format_value(val, cqltype, **kwargs):
|
||||
if val == EMPTY:
|
||||
return format_value_default('', **kwargs)
|
||||
|
||||
formatter = get_formatter(val, cqltype)
|
||||
return formatter(val, cqltype=cqltype, **kwargs)
|
||||
|
||||
|
|
@ -239,7 +242,7 @@ def formatter_for(typname):
|
|||
|
||||
@formatter_for('bytearray')
|
||||
def format_value_blob(val, colormap, **_):
|
||||
bval = '0x' + binascii.hexlify(val)
|
||||
bval = ensure_text('0x') + ensure_text(binascii.hexlify(val))
|
||||
return colorme(bval, colormap, 'blob')
|
||||
|
||||
|
||||
|
|
@ -248,7 +251,7 @@ formatter_for('blob')(format_value_blob)
|
|||
|
||||
|
||||
def format_python_formatted_type(val, colormap, color, quote=False):
|
||||
bval = str(val)
|
||||
bval = ensure_text(str(val))
|
||||
if quote:
|
||||
bval = "'%s'" % bval
|
||||
return colorme(bval, colormap, color)
|
||||
|
|
@ -318,6 +321,7 @@ formatter_for('double')(format_floating_point_type)
|
|||
def format_integer_type(val, colormap, thousands_sep=None, **_):
|
||||
# base-10 only for now; support others?
|
||||
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
|
||||
bval = ensure_text(bval)
|
||||
return colorme(bval, colormap, 'int')
|
||||
|
||||
|
||||
|
|
@ -352,7 +356,7 @@ def format_value_timestamp(val, colormap, date_time_format, quote=False, **_):
|
|||
if date_time_format.milliseconds_only:
|
||||
bval = round_microseconds(bval)
|
||||
else:
|
||||
bval = str(val)
|
||||
bval = ensure_text(str(val))
|
||||
|
||||
if quote:
|
||||
bval = "'%s'" % bval
|
||||
|
|
@ -378,7 +382,7 @@ def strftime(time_format, seconds, microseconds=0, timezone=None):
|
|||
return '%d' % (seconds * 1000.0)
|
||||
|
||||
|
||||
microseconds_regex = re.compile("(.*)(?:\.(\d{1,6}))(.*)")
|
||||
microseconds_regex = re.compile(r"(.*)(?:\.(\d{1,6}))(.*)")
|
||||
|
||||
|
||||
def round_microseconds(val):
|
||||
|
|
@ -453,14 +457,14 @@ def decode_unsigned_vint(buf):
|
|||
For example, if we need to read 3 more bytes the first byte will start with 1110.
|
||||
"""
|
||||
|
||||
first_byte = buf.next()
|
||||
first_byte = next(buf)
|
||||
if (first_byte >> 7) == 0:
|
||||
return first_byte
|
||||
|
||||
size = number_of_extra_bytes_to_read(first_byte)
|
||||
retval = first_byte & (0xff >> size)
|
||||
for i in range(size):
|
||||
b = buf.next()
|
||||
b = next(buf)
|
||||
retval <<= 8
|
||||
retval |= b & 0xff
|
||||
|
||||
|
|
@ -477,15 +481,14 @@ def decode_zig_zag_64(n):
|
|||
|
||||
@formatter_for('str')
|
||||
def format_value_text(val, encoding, colormap, quote=False, **_):
|
||||
escapedval = val.replace(u'\\', u'\\\\')
|
||||
escapedval = val.replace('\\', '\\\\')
|
||||
if quote:
|
||||
escapedval = escapedval.replace("'", "''")
|
||||
escapedval = unicode_controlchars_re.sub(_show_control_chars, escapedval)
|
||||
bval = escapedval.encode(encoding, 'backslashreplace')
|
||||
bval = escapedval
|
||||
if quote:
|
||||
bval = "'%s'" % bval
|
||||
|
||||
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval.decode(encoding)))
|
||||
bval = "'{}'".format(bval)
|
||||
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval))
|
||||
|
||||
|
||||
# name alias
|
||||
|
|
@ -532,7 +535,7 @@ def format_value_tuple(val, cqltype, encoding, colormap, date_time_format, float
|
|||
@formatter_for('set')
|
||||
def format_value_set(val, cqltype, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
return format_simple_collection(sorted(val), cqltype, '{', '}', encoding, colormap,
|
||||
return format_simple_collection(val, cqltype, '{', '}', encoding, colormap,
|
||||
date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles)
|
||||
|
||||
|
|
@ -584,7 +587,7 @@ def format_value_utype(val, cqltype, encoding, colormap, date_time_format, float
|
|||
def format_field_name(name):
|
||||
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)
|
||||
|
||||
subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(val._asdict().items(),
|
||||
subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(list(val._asdict().items()),
|
||||
cqltype.sub_types)]
|
||||
bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}'
|
||||
if colormap is NO_COLOR_MAP:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
# limitations under the License.
|
||||
|
||||
import re
|
||||
from .saferscanner import SaferScanner
|
||||
|
||||
from cqlshlib.saferscanner import SaferScanner
|
||||
|
||||
|
||||
class LexingError(Exception):
|
||||
|
|
@ -107,15 +108,6 @@ class ParseContext:
|
|||
return ' '.join([t[1] for t in tokens])
|
||||
# low end of span for first token, to high end of span for last token
|
||||
orig_text = orig[tokens[0][2][0]:tokens[-1][2][1]]
|
||||
|
||||
# Convert all unicode tokens to ascii, where possible. This
|
||||
# helps avoid problems with performing unicode-incompatible
|
||||
# operations on tokens (like .lower()). See CASSANDRA-9083
|
||||
# for one example of this.
|
||||
try:
|
||||
orig_text = orig_text.encode('ascii')
|
||||
except UnicodeEncodeError:
|
||||
pass
|
||||
return orig_text
|
||||
|
||||
def __repr__(self):
|
||||
|
|
@ -146,7 +138,7 @@ class matcher:
|
|||
except KeyError:
|
||||
return False
|
||||
if debugging:
|
||||
print "Trying completer %r with %r" % (completer, ctxt)
|
||||
print("Trying completer %r with %r" % (completer, ctxt))
|
||||
try:
|
||||
new_compls = completer(ctxt)
|
||||
except Exception:
|
||||
|
|
@ -155,7 +147,7 @@ class matcher:
|
|||
traceback.print_exc()
|
||||
return False
|
||||
if debugging:
|
||||
print "got %r" % (new_compls,)
|
||||
print("got %r" % (new_compls,))
|
||||
completions.update(new_compls)
|
||||
return True
|
||||
|
||||
|
|
@ -284,7 +276,7 @@ class text_match(terminal_matcher):
|
|||
try:
|
||||
terminal_matcher.__init__(self, eval(text))
|
||||
except SyntaxError:
|
||||
print "bad syntax %r" % (text,)
|
||||
print("bad syntax %r" % (text,))
|
||||
|
||||
def match(self, ctxt, completions):
|
||||
if ctxt.remainder:
|
||||
|
|
@ -359,7 +351,7 @@ class ParsingRuleSet:
|
|||
(r'[@()|?*;]', lambda s, t: t),
|
||||
(r'\s+', None),
|
||||
(r'#[^\n]*', None),
|
||||
], re.I | re.S)
|
||||
], re.I | re.S | re.U)
|
||||
|
||||
def __init__(self):
|
||||
self.ruleset = {}
|
||||
|
|
@ -382,7 +374,7 @@ class ParsingRuleSet:
|
|||
tokeniter = iter(tokens)
|
||||
for t in tokeniter:
|
||||
if isinstance(t, tuple) and t[0] in ('reference', 'junk'):
|
||||
assign = tokeniter.next()
|
||||
assign = next(tokeniter)
|
||||
if assign != '::=':
|
||||
raise ValueError('Unexpected token %r; expected "::="' % (assign,))
|
||||
name = t[1]
|
||||
|
|
@ -405,7 +397,7 @@ class ParsingRuleSet:
|
|||
|
||||
@classmethod
|
||||
def read_rule_tokens_until(cls, endtoks, tokeniter):
|
||||
if isinstance(endtoks, basestring):
|
||||
if isinstance(endtoks, str):
|
||||
endtoks = (endtoks,)
|
||||
counttarget = None
|
||||
if isinstance(endtoks, int):
|
||||
|
|
@ -419,7 +411,7 @@ class ParsingRuleSet:
|
|||
if t in endtoks:
|
||||
if len(mybranches) == 1:
|
||||
return cls.mkrule(mybranches[0])
|
||||
return choice(map(cls.mkrule, mybranches))
|
||||
return choice(list(map(cls.mkrule, mybranches)))
|
||||
if isinstance(t, tuple):
|
||||
if t[0] == 'reference':
|
||||
t = rule_reference(t[1])
|
||||
|
|
@ -441,7 +433,7 @@ class ParsingRuleSet:
|
|||
elif t == '*':
|
||||
t = repeat(myrules.pop(-1))
|
||||
elif t == '@':
|
||||
x = tokeniter.next()
|
||||
x = next(tokeniter)
|
||||
if not isinstance(x, tuple) or x[0] != 'litstring':
|
||||
raise ValueError("Unexpected token %r following '@'" % (x,))
|
||||
t = case_match(x[1])
|
||||
|
|
@ -455,7 +447,7 @@ class ParsingRuleSet:
|
|||
if countsofar == counttarget:
|
||||
if len(mybranches) == 1:
|
||||
return cls.mkrule(mybranches[0])
|
||||
return choice(map(cls.mkrule, mybranches))
|
||||
return choice(list(map(cls.mkrule, mybranches)))
|
||||
raise ValueError('Unexpected end of rule tokens')
|
||||
|
||||
def append_rules(self, rulestr):
|
||||
|
|
@ -474,7 +466,7 @@ class ParsingRuleSet:
|
|||
return None
|
||||
return lambda s, t: (name, t, s.match.span())
|
||||
regexes = [(p.pattern(), make_handler(name)) for (name, p) in self.terminals]
|
||||
return SaferScanner(regexes, re.I | re.S).scan
|
||||
return SaferScanner(regexes, re.I | re.S | re.U).scan
|
||||
|
||||
def lex(self, text):
|
||||
if self.scanner is None:
|
||||
|
|
|
|||
|
|
@ -19,24 +19,11 @@
|
|||
# regex in-pattern flags. Any of those can break correct operation of Scanner.
|
||||
|
||||
import re
|
||||
import six
|
||||
from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS
|
||||
|
||||
|
||||
class SaferScanner(re.Scanner):
|
||||
|
||||
def __init__(self, lexicon, flags=0):
|
||||
self.lexicon = lexicon
|
||||
p = []
|
||||
s = re.sre_parse.Pattern()
|
||||
s.flags = flags
|
||||
for phrase, action in lexicon:
|
||||
p.append(re.sre_parse.SubPattern(s, [
|
||||
(SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))),
|
||||
]))
|
||||
s.groups = len(p) + 1
|
||||
p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
|
||||
self.p = p
|
||||
self.scanner = re.sre_compile.compile(p)
|
||||
class SaferScannerBase(re.Scanner):
|
||||
|
||||
@classmethod
|
||||
def subpat(cls, phrase, flags):
|
||||
|
|
@ -60,3 +47,39 @@ class SaferScanner(re.Scanner):
|
|||
if sub.pattern.flags ^ flags:
|
||||
raise ValueError("RE flag setting not allowed in SaferScanner lexicon (%s)" % (bin(sub.pattern.flags),))
|
||||
return re.sre_parse.SubPattern(sub.pattern, scrubbedsub)
|
||||
|
||||
|
||||
class Py2SaferScanner(SaferScannerBase):
|
||||
|
||||
def __init__(self, lexicon, flags=0):
|
||||
self.lexicon = lexicon
|
||||
p = []
|
||||
s = re.sre_parse.Pattern()
|
||||
s.flags = flags
|
||||
for phrase, action in lexicon:
|
||||
p.append(re.sre_parse.SubPattern(s, [
|
||||
(SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))),
|
||||
]))
|
||||
s.groups = len(p) + 1
|
||||
p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
|
||||
self.p = p
|
||||
self.scanner = re.sre_compile.compile(p)
|
||||
|
||||
|
||||
class Py36SaferScanner(SaferScannerBase):
|
||||
|
||||
def __init__(self, lexicon, flags=0):
|
||||
self.lexicon = lexicon
|
||||
p = []
|
||||
s = re.sre_parse.Pattern()
|
||||
s.flags = flags
|
||||
for phrase, action in lexicon:
|
||||
gid = s.opengroup()
|
||||
p.append(re.sre_parse.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re.sre_parse.parse(phrase, flags))), ]))
|
||||
s.closegroup(gid, p[-1])
|
||||
p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))])
|
||||
self.p = p
|
||||
self.scanner = re.sre_compile.compile(p)
|
||||
|
||||
|
||||
SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner
|
||||
|
|
|
|||
|
|
@ -16,9 +16,10 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import ConfigParser
|
||||
import ssl
|
||||
|
||||
from six.moves import configparser
|
||||
|
||||
|
||||
def ssl_settings(host, config_file, env=os.environ):
|
||||
"""
|
||||
|
|
@ -38,13 +39,13 @@ def ssl_settings(host, config_file, env=os.environ):
|
|||
either in the config file or as an environment variable.
|
||||
Environment variables override any options set in cqlsh config file.
|
||||
"""
|
||||
configs = ConfigParser.SafeConfigParser()
|
||||
configs = configparser.SafeConfigParser()
|
||||
configs.read(config_file)
|
||||
|
||||
def get_option(section, option):
|
||||
try:
|
||||
return configs.get(section, option)
|
||||
except ConfigParser.Error:
|
||||
except configparser.Error:
|
||||
return None
|
||||
|
||||
ssl_validate = env.get('SSL_VALIDATE')
|
||||
|
|
|
|||
|
|
@ -15,6 +15,3 @@
|
|||
# limitations under the License.
|
||||
|
||||
from .cassconnect import create_db, remove_db
|
||||
|
||||
setUp = create_db
|
||||
tearDown = remove_db
|
||||
|
|
|
|||
|
|
@ -14,31 +14,35 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import six
|
||||
|
||||
LIGHT = 010
|
||||
LIGHT = 0o10
|
||||
|
||||
ansi_CSI = '\033['
|
||||
|
||||
ansi_CSI = '\x1b['
|
||||
ansi_seq = re.compile(re.escape(ansi_CSI) + r'(?P<params>[\x20-\x3f]*)(?P<final>[\x40-\x7e])')
|
||||
ansi_cmd_SGR = 'm' # set graphics rendition
|
||||
|
||||
color_defs = (
|
||||
(000, 'k', 'black'),
|
||||
(001, 'r', 'dark red'),
|
||||
(002, 'g', 'dark green'),
|
||||
(003, 'w', 'brown', 'dark yellow'),
|
||||
(004, 'b', 'dark blue'),
|
||||
(005, 'm', 'dark magenta', 'dark purple'),
|
||||
(006, 'c', 'dark cyan'),
|
||||
(007, 'n', 'light grey', 'light gray', 'neutral', 'dark white'),
|
||||
(010, 'B', 'dark grey', 'dark gray', 'light black'),
|
||||
(011, 'R', 'red', 'light red'),
|
||||
(012, 'G', 'green', 'light green'),
|
||||
(013, 'Y', 'yellow', 'light yellow'),
|
||||
(014, 'B', 'blue', 'light blue'),
|
||||
(015, 'M', 'magenta', 'purple', 'light magenta', 'light purple'),
|
||||
(016, 'C', 'cyan', 'light cyan'),
|
||||
(017, 'W', 'white', 'light white'),
|
||||
(0o01, 'r', 'dark red'),
|
||||
(0o02, 'g', 'dark green'),
|
||||
(0o03, 'w', 'brown', 'dark yellow'),
|
||||
(0o04, 'b', 'dark blue'),
|
||||
(0o05, 'm', 'dark magenta', 'dark purple'),
|
||||
(0o06, 'c', 'dark cyan'),
|
||||
(0o07, 'n', 'light grey', 'light gray', 'neutral', 'dark white'),
|
||||
(0o10, 'B', 'dark grey', 'dark gray', 'light black'),
|
||||
(0o11, 'R', 'red', 'light red'),
|
||||
(0o12, 'G', 'green', 'light green'),
|
||||
(0o13, 'Y', 'yellow', 'light yellow'),
|
||||
(0o14, 'B', 'blue', 'light blue'),
|
||||
(0o15, 'M', 'magenta', 'purple', 'light magenta', 'light purple'),
|
||||
(0o16, 'C', 'cyan', 'light cyan'),
|
||||
(0o17, 'W', 'white', 'light white'),
|
||||
)
|
||||
|
||||
colors_by_num = {}
|
||||
|
|
@ -61,7 +65,7 @@ for colordef in color_defs:
|
|||
for c in nameset:
|
||||
colors_by_name[c] = colorcode
|
||||
|
||||
class ColoredChar:
|
||||
class ColoredChar(object):
|
||||
def __init__(self, c, colorcode):
|
||||
self.c = c
|
||||
self._colorcode = colorcode
|
||||
|
|
@ -76,8 +80,8 @@ class ColoredChar:
|
|||
return getattr(self.c, name)
|
||||
|
||||
def ansi_color(self):
|
||||
clr = str(30 + (07 & self._colorcode))
|
||||
if self._colorcode & 010:
|
||||
clr = str(30 + (0o7 & self._colorcode))
|
||||
if self._colorcode & 0o10:
|
||||
clr = '1;' + clr
|
||||
return clr
|
||||
|
||||
|
|
@ -100,11 +104,11 @@ class ColoredChar:
|
|||
def colortag(self):
|
||||
return lookup_letter_from_code(self._colorcode)
|
||||
|
||||
class ColoredText:
|
||||
class ColoredText(object):
|
||||
def __init__(self, source=''):
|
||||
if isinstance(source, basestring):
|
||||
if isinstance(source, six.text_type):
|
||||
plain, colors = self.parse_ansi_colors(source)
|
||||
self.chars = map(ColoredChar, plain, colors)
|
||||
self.chars = list(map(ColoredChar, plain, colors))
|
||||
else:
|
||||
# expected that source is an iterable of ColoredChars (or duck-typed as such)
|
||||
self.chars = tuple(source)
|
||||
|
|
@ -149,7 +153,7 @@ class ColoredText:
|
|||
@staticmethod
|
||||
def parse_sgr_param(curclr, paramstr):
|
||||
oldclr = curclr
|
||||
args = map(int, paramstr.split(';'))
|
||||
args = list(map(int, paramstr.split(';')))
|
||||
for a in args:
|
||||
if a == 0:
|
||||
curclr = lookup_colorcode('neutral')
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
import os
|
||||
import sys
|
||||
import logging
|
||||
from itertools import izip
|
||||
|
||||
from os.path import dirname, join, normpath
|
||||
|
||||
cqlshlog = logging.getLogger('test_cqlsh')
|
||||
|
|
@ -46,13 +46,27 @@ TEST_PORT = int(os.environ.get('CQL_TEST_PORT', 9042))
|
|||
|
||||
class BaseTestCase(unittest.TestCase):
|
||||
def assertNicelyFormattedTableHeader(self, line, msg=None):
|
||||
return self.assertRegexpMatches(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg)
|
||||
return self.assertRegex(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg)
|
||||
|
||||
def assertNicelyFormattedTableRule(self, line, msg=None):
|
||||
return self.assertRegexpMatches(line, r'^-+(\+-+)*\s*$', msg=msg)
|
||||
return self.assertRegex(line, r'^-+(\+-+)*\s*$', msg=msg)
|
||||
|
||||
def assertNicelyFormattedTableData(self, line, msg=None):
|
||||
return self.assertRegexpMatches(line, r'^ .* \| ', msg=msg)
|
||||
return self.assertRegex(line, r'^ .* \| ', msg=msg)
|
||||
|
||||
def assertRegex(self, text, regex, msg=None):
|
||||
"""Call assertRegexpMatches() if in Python 2"""
|
||||
if hasattr(unittest.TestCase, 'assertRegex'):
|
||||
return super().assertRegex(text, regex, msg)
|
||||
else:
|
||||
return self.assertRegexpMatches(text, regex, msg)
|
||||
|
||||
def assertNotRegex(self, text, regex, msg=None):
|
||||
"""Call assertNotRegexpMatches() if in Python 2"""
|
||||
if hasattr(unittest.TestCase, 'assertNotRegex'):
|
||||
return super().assertNotRegex(text, regex, msg)
|
||||
else:
|
||||
return self.assertNotRegexpMatches(text, regex, msg)
|
||||
|
||||
def dedent(s):
|
||||
lines = [ln.rstrip() for ln in s.splitlines()]
|
||||
|
|
@ -63,4 +77,4 @@ def dedent(s):
|
|||
return '\n'.join(line[minspace:] for line in lines)
|
||||
|
||||
def at_a_time(i, num):
|
||||
return izip(*([iter(i)] * num))
|
||||
return zip(*([iter(i)] * num))
|
||||
|
|
|
|||
|
|
@ -14,12 +14,13 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
import contextlib
|
||||
import tempfile
|
||||
import io
|
||||
import os.path
|
||||
from .basecase import cql, cqlsh, cqlshlog, TEST_HOST, TEST_PORT, rundir, policy, quote_name
|
||||
import tempfile
|
||||
|
||||
from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, rundir
|
||||
from .run_cqlsh import run_cqlsh, call_cqlsh
|
||||
|
||||
test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql')
|
||||
|
|
@ -43,12 +44,12 @@ def make_ks_name():
|
|||
return os.path.basename(tempfile.mktemp(prefix='CqlshTests_'))
|
||||
|
||||
def create_keyspace(cursor):
|
||||
ksname = make_ks_name()
|
||||
ksname = make_ks_name().lower()
|
||||
qksname = quote_name(ksname)
|
||||
cursor.execute('''
|
||||
CREATE KEYSPACE %s WITH replication =
|
||||
{'class': 'SimpleStrategy', 'replication_factor': 1};
|
||||
''' % quote_name(ksname))
|
||||
''' % qksname)
|
||||
cursor.execute('USE %s;' % qksname)
|
||||
TEST_KEYSPACES_CREATED.append(ksname)
|
||||
return ksname
|
||||
|
|
@ -63,11 +64,11 @@ def split_cql_commands(source):
|
|||
|
||||
def execute_cql_commands(cursor, source, logprefix='INIT: '):
|
||||
for cql in split_cql_commands(source):
|
||||
cqlshlog.debug(logprefix + cql)
|
||||
cqlshlog.debug((logprefix + cql).encode("utf-8"))
|
||||
cursor.execute(cql)
|
||||
|
||||
def execute_cql_file(cursor, fname):
|
||||
with open(fname) as f:
|
||||
with io.open(fname, "r", encoding="utf-8") as f:
|
||||
return execute_cql_commands(cursor, f.read())
|
||||
|
||||
def create_db():
|
||||
|
|
@ -116,8 +117,6 @@ def cassandra_cursor(cql_version=None, ks=''):
|
|||
c = conn.connect(ks)
|
||||
# increase default timeout to fix flacky tests, see CASSANDRA-12481
|
||||
c.default_timeout = 60.0
|
||||
# if ks is not None:
|
||||
# c.execute('USE %s;' % quote_name(c, ks))
|
||||
yield c
|
||||
finally:
|
||||
conn.shutdown()
|
||||
|
|
@ -137,4 +136,6 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs):
|
|||
def testcall_cqlsh(keyspace=None, **kwargs):
|
||||
if keyspace is None:
|
||||
keyspace = get_keyspace()
|
||||
if ('input' in kwargs.keys() and isinstance(kwargs['input'], str)):
|
||||
kwargs['input'] = kwargs['input'].encode('utf-8')
|
||||
return call_cqlsh(keyspace=keyspace, **kwargs)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@
|
|||
|
||||
# NOTE: this testing tool is *nix specific
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
|
|
@ -32,7 +34,7 @@ def is_win():
|
|||
return sys.platform in ("cygwin", "win32")
|
||||
|
||||
if is_win():
|
||||
from winpty import WinPty
|
||||
from .winpty import WinPty
|
||||
DEFAULT_PREFIX = ''
|
||||
else:
|
||||
import pty
|
||||
|
|
@ -41,6 +43,24 @@ else:
|
|||
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> '
|
||||
DEFAULT_CQLSH_TERM = 'xterm'
|
||||
|
||||
def get_smm_sequence(term='xterm'):
|
||||
"""
|
||||
Return the set meta mode (smm) sequence, if any.
|
||||
On more recent Linux systems, xterm emits the smm sequence
|
||||
before each prompt.
|
||||
"""
|
||||
result = ''
|
||||
if not is_win():
|
||||
tput_proc = subprocess.Popen(['tput', '-T{}'.format(term), 'smm'], stdout=subprocess.PIPE)
|
||||
tput_stdout = tput_proc.communicate()[0]
|
||||
if (tput_stdout and (tput_stdout != b'')):
|
||||
result = tput_stdout
|
||||
if isinstance(result, bytes):
|
||||
result = result.decode("utf-8")
|
||||
return result
|
||||
|
||||
DEFAULT_SMM_SEQUENCE = get_smm_sequence()
|
||||
|
||||
cqlshlog = basecase.cqlshlog
|
||||
|
||||
def set_controlling_pty(master, slave):
|
||||
|
|
@ -103,7 +123,7 @@ def timing_out_alarm(seconds):
|
|||
if is_win():
|
||||
try:
|
||||
import eventlet
|
||||
except ImportError, e:
|
||||
except ImportError as e:
|
||||
sys.exit("evenlet library required to run cqlshlib tests on Windows")
|
||||
|
||||
def timing_out(seconds):
|
||||
|
|
@ -171,22 +191,33 @@ class ProcRunner:
|
|||
return self.proc.wait()
|
||||
|
||||
def send_tty(self, data):
|
||||
if not isinstance(data, bytes):
|
||||
data = data.encode("utf-8")
|
||||
os.write(self.childpty, data)
|
||||
|
||||
def send_pipe(self, data):
|
||||
self.proc.stdin.write(data)
|
||||
|
||||
def read_tty(self, blksize, timeout=None):
|
||||
return os.read(self.childpty, blksize)
|
||||
buf = os.read(self.childpty, blksize)
|
||||
if isinstance(buf, bytes):
|
||||
buf = buf.decode("utf-8")
|
||||
return buf
|
||||
|
||||
def read_pipe(self, blksize, timeout=None):
|
||||
return self.proc.stdout.read(blksize)
|
||||
buf = self.proc.stdout.read(blksize)
|
||||
if isinstance(buf, bytes):
|
||||
buf = buf.decode("utf-8")
|
||||
return buf
|
||||
|
||||
def read_winpty(self, blksize, timeout=None):
|
||||
return self.winpty.read(blksize, timeout)
|
||||
buf = self.winpty.read(blksize, timeout)
|
||||
if isinstance(buf, bytes):
|
||||
buf = buf.decode("utf-8")
|
||||
return buf
|
||||
|
||||
def read_until(self, until, blksize=4096, timeout=None,
|
||||
flags=0, ptty_timeout=None):
|
||||
flags=0, ptty_timeout=None, replace=[]):
|
||||
if not isinstance(until, re._pattern_type):
|
||||
until = re.compile(until, flags)
|
||||
|
||||
|
|
@ -196,6 +227,9 @@ class ProcRunner:
|
|||
with timing_out(timeout):
|
||||
while True:
|
||||
val = self.read(blksize, ptty_timeout)
|
||||
for replace_target in replace:
|
||||
if (replace_target != ''):
|
||||
val = val.replace(replace_target, '')
|
||||
cqlshlog.debug("read %r from subproc" % (val,))
|
||||
if val == '':
|
||||
raise EOFError("'until' pattern %r not found" % (until.pattern,))
|
||||
|
|
@ -252,16 +286,21 @@ class CqlshRunner(ProcRunner):
|
|||
env.setdefault('TERM', 'xterm')
|
||||
env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', ''))
|
||||
env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', ''))
|
||||
coverage = False
|
||||
if ('CQLSH_COVERAGE' in env.keys()):
|
||||
coverage = True
|
||||
args = tuple(args) + (host, str(port))
|
||||
if cqlver is not None:
|
||||
args += ('--cqlversion', str(cqlver))
|
||||
if keyspace is not None:
|
||||
args += ('--keyspace', keyspace)
|
||||
args += ('--keyspace', keyspace.lower())
|
||||
if tty and is_win():
|
||||
args += ('--tty',)
|
||||
args += ('--encoding', 'utf-8')
|
||||
if win_force_colors:
|
||||
args += ('--color',)
|
||||
if coverage:
|
||||
args += ('--coverage',)
|
||||
self.keyspace = keyspace
|
||||
ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs)
|
||||
self.prompt = prompt
|
||||
|
|
@ -270,8 +309,8 @@ class CqlshRunner(ProcRunner):
|
|||
else:
|
||||
self.output_header = self.read_to_next_prompt()
|
||||
|
||||
def read_to_next_prompt(self):
|
||||
return self.read_until(self.prompt, timeout=10.0, ptty_timeout=3)
|
||||
def read_to_next_prompt(self, timeout=10.0):
|
||||
return self.read_until(self.prompt, timeout=timeout, ptty_timeout=3, replace=[DEFAULT_SMM_SEQUENCE,])
|
||||
|
||||
def read_up_to_timeout(self, timeout, blksize=4096):
|
||||
output = ProcRunner.read_up_to_timeout(self, timeout, blksize=blksize)
|
||||
|
|
@ -309,4 +348,6 @@ def call_cqlsh(**kwargs):
|
|||
c = CqlshRunner(**kwargs)
|
||||
output, _ = c.proc.communicate(proginput)
|
||||
result = c.close()
|
||||
if isinstance(output, bytes):
|
||||
output = output.decode("utf-8")
|
||||
return output, result
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
|
||||
import unittest
|
||||
|
||||
from cassandra.metadata import MIN_LONG, Murmur3Token, TokenMap
|
||||
from cassandra.policies import SimpleConvictionPolicy
|
||||
from cassandra.pool import Host
|
||||
from unittest.mock import Mock
|
||||
|
||||
from cqlshlib.copyutil import ExportTask
|
||||
|
||||
|
||||
class CopyTaskTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# set up default test data
|
||||
self.ks = 'testks'
|
||||
self.table = 'testtable'
|
||||
self.columns = ['a', 'b']
|
||||
self.fname = 'test_fname'
|
||||
self.opts = {}
|
||||
self.protocol_version = 0
|
||||
self.config_file = 'test_config'
|
||||
self.hosts = [
|
||||
Host('10.0.0.1', SimpleConvictionPolicy, 9000),
|
||||
Host('10.0.0.2', SimpleConvictionPolicy, 9000),
|
||||
Host('10.0.0.3', SimpleConvictionPolicy, 9000),
|
||||
Host('10.0.0.4', SimpleConvictionPolicy, 9000)
|
||||
]
|
||||
|
||||
def mock_shell(self):
|
||||
"""
|
||||
Set up a mock Shell so we can unit test ExportTask internals
|
||||
"""
|
||||
shell = Mock()
|
||||
shell.conn = Mock()
|
||||
shell.conn.get_control_connection_host.return_value = self.hosts[0]
|
||||
shell.get_column_names.return_value = self.columns
|
||||
shell.debug = False
|
||||
return shell
|
||||
|
||||
|
||||
class TestExportTask(CopyTaskTest):
|
||||
|
||||
def _test_get_ranges_murmur3_base(self, opts, expected_ranges):
|
||||
"""
|
||||
Set up a mock shell with a simple token map to test the ExportTask get_ranges function.
|
||||
"""
|
||||
shell = self.mock_shell()
|
||||
shell.conn.metadata.partitioner = 'Murmur3Partitioner'
|
||||
# token range for a cluster of 4 nodes with replication factor 3
|
||||
shell.get_ring.return_value = {
|
||||
Murmur3Token(-9223372036854775808): self.hosts[0:3],
|
||||
Murmur3Token(-4611686018427387904): self.hosts[1:4],
|
||||
Murmur3Token(0): [self.hosts[2], self.hosts[3], self.hosts[0]],
|
||||
Murmur3Token(4611686018427387904): [self.hosts[3], self.hosts[0], self.hosts[1]]
|
||||
}
|
||||
# merge override options with standard options
|
||||
overridden_opts = dict(self.opts)
|
||||
for k,v in opts.items():
|
||||
overridden_opts[k] = v
|
||||
export_task = ExportTask(shell, self.ks, self.table, self.columns, self.fname, overridden_opts, self.protocol_version, self.config_file)
|
||||
assert export_task.get_ranges() == expected_ranges
|
||||
|
||||
def test_get_ranges_murmur3(self):
|
||||
"""
|
||||
Test behavior of ExportTask internal get_ranges function
|
||||
"""
|
||||
|
||||
# return empty dict and print error if begin_token < min_token
|
||||
self._test_get_ranges_murmur3_base({'begintoken': MIN_LONG - 1}, {})
|
||||
|
||||
# return empty dict and print error if begin_token < min_token
|
||||
self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': -1}, {})
|
||||
|
||||
# simple case of a single range
|
||||
expected_ranges = {(1,2): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}}
|
||||
self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': 2}, expected_ranges)
|
||||
|
||||
# simple case of two contiguous ranges
|
||||
expected_ranges = {
|
||||
(-4611686018427387903,0): {'hosts': ('10.0.0.3', '10.0.0.4', '10.0.0.1'), 'attempts': 0, 'rows': 0, 'workerno': -1},
|
||||
(0,1): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}
|
||||
}
|
||||
self._test_get_ranges_murmur3_base({'begintoken': -4611686018427387903, 'endtoken': 1}, expected_ranges)
|
||||
|
||||
# specify a begintoken only (endtoken defaults to None)
|
||||
expected_ranges = {
|
||||
(4611686018427387905,None): {'hosts': ('10.0.0.1', '10.0.0.2', '10.0.0.3'), 'attempts': 0, 'rows': 0, 'workerno': -1}
|
||||
}
|
||||
self._test_get_ranges_murmur3_base({'begintoken': 4611686018427387905}, expected_ranges)
|
||||
|
||||
# specify an endtoken only (begintoken defaults to None)
|
||||
expected_ranges = {
|
||||
(None, MIN_LONG + 1): {'hosts': ('10.0.0.2', '10.0.0.3', '10.0.0.4'), 'attempts': 0, 'rows': 0, 'workerno': -1}
|
||||
}
|
||||
self._test_get_ranges_murmur3_base({'endtoken': MIN_LONG + 1}, expected_ranges)
|
||||
|
||||
|
|
@ -20,7 +20,7 @@
|
|||
from unittest import TestCase
|
||||
from operator import itemgetter
|
||||
|
||||
from ..cql3handling import CqlRuleSet
|
||||
from cqlshlib.cql3handling import CqlRuleSet
|
||||
|
||||
|
||||
class TestCqlParsing(TestCase):
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
from .basecase import BaseTestCase, cqlsh
|
||||
|
||||
class TestCqlshCommands(BaseTestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_show(self):
|
||||
pass
|
||||
|
||||
def test_describe(self):
|
||||
pass
|
||||
|
||||
def test_exit(self):
|
||||
pass
|
||||
|
||||
def test_help(self):
|
||||
pass
|
||||
|
|
@ -17,11 +17,13 @@
|
|||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
from __future__ import with_statement
|
||||
|
||||
import locale
|
||||
import os
|
||||
import re
|
||||
from .basecase import BaseTestCase, cqlsh
|
||||
from .cassconnect import testrun_cqlsh
|
||||
from .basecase import BaseTestCase, cqlsh, cqlshlog
|
||||
from .cassconnect import create_db, remove_db, testrun_cqlsh
|
||||
from .run_cqlsh import TimeoutError
|
||||
import unittest
|
||||
import sys
|
||||
|
||||
|
|
@ -41,8 +43,22 @@ completion_separation_re = re.compile(r'\s+')
|
|||
@unittest.skipIf(sys.platform == "win32", 'Tab completion tests not supported on Windows')
|
||||
class CqlshCompletionCase(BaseTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
create_db()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
remove_db()
|
||||
|
||||
def setUp(self):
|
||||
self.cqlsh_runner = testrun_cqlsh(cqlver=None, env={'COLUMNS': '100000'})
|
||||
env = os.environ
|
||||
env['COLUMNS'] = '100000'
|
||||
if (locale.getpreferredencoding() != 'UTF-8'):
|
||||
env['LC_CTYPE'] = 'en_US.utf8'
|
||||
if ('PATH' in os.environ.keys()):
|
||||
env['PATH'] = os.environ['PATH']
|
||||
self.cqlsh_runner = testrun_cqlsh(cqlver=None, env=env)
|
||||
self.cqlsh = self.cqlsh_runner.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -81,14 +97,14 @@ class CqlshCompletionCase(BaseTestCase):
|
|||
prompt_regex = self.cqlsh.prompt.lstrip() + re.escape(inputstring)
|
||||
msg = ('Double-tab completion '
|
||||
'does not print prompt for input "{}"'.format(inputstring))
|
||||
self.assertRegexpMatches(choice_lines[-1], prompt_regex, msg=msg)
|
||||
self.assertRegex(choice_lines[-1], prompt_regex, msg=msg)
|
||||
|
||||
choice_lines = [line.strip() for line in choice_lines[:-1]]
|
||||
choice_lines = [line for line in choice_lines if line]
|
||||
|
||||
if split_completed_lines:
|
||||
completed_lines = map(set, (completion_separation_re.split(line.strip())
|
||||
for line in choice_lines))
|
||||
completed_lines = list(map(set, (completion_separation_re.split(line.strip())
|
||||
for line in choice_lines)))
|
||||
|
||||
if not completed_lines:
|
||||
return set()
|
||||
|
|
@ -132,8 +148,14 @@ class CqlshCompletionCase(BaseTestCase):
|
|||
other_choices_ok=other_choices_ok,
|
||||
split_completed_lines=split_completed_lines)
|
||||
finally:
|
||||
self.cqlsh.send(CTRL_C) # cancel any current line
|
||||
self.cqlsh.read_to_next_prompt()
|
||||
try:
|
||||
self.cqlsh.send(CTRL_C) # cancel any current line
|
||||
self.cqlsh.read_to_next_prompt(timeout=1.0)
|
||||
except TimeoutError:
|
||||
# retry once
|
||||
self.cqlsh.send(CTRL_C)
|
||||
self.cqlsh.read_to_next_prompt(timeout=10.0)
|
||||
|
||||
|
||||
def strategies(self):
|
||||
return self.module.CqlRuleSet.replication_strategies
|
||||
|
|
@ -345,8 +367,6 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
choices=[',', 'WHERE'])
|
||||
self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ",
|
||||
choices=['TOKEN(', 'lonelykey'])
|
||||
self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ",
|
||||
choices=['TOKEN(', 'lonelykey'])
|
||||
|
||||
self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE lonel",
|
||||
immediate='ykey ')
|
||||
|
|
@ -392,7 +412,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'twenty_rows_composite_table',
|
||||
'utf8_with_special_chars',
|
||||
'system_traces.', 'songs',
|
||||
'"' + self.cqlsh.keyspace + '".'],
|
||||
self.cqlsh.keyspace + '.'],
|
||||
other_choices_ok=True)
|
||||
|
||||
self.trycompletions('DELETE FROM ',
|
||||
|
|
@ -407,7 +427,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'system_traces.', 'songs',
|
||||
'system_auth.', 'system_distributed.',
|
||||
'system_schema.', 'system_traces.',
|
||||
'"' + self.cqlsh.keyspace + '".'],
|
||||
self.cqlsh.keyspace + '.'],
|
||||
other_choices_ok=True)
|
||||
self.trycompletions('DELETE FROM twenty_rows_composite_table ',
|
||||
choices=['USING', 'WHERE'])
|
||||
|
|
@ -529,13 +549,13 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
self.trycompletions('DROP K', immediate='EYSPACE ')
|
||||
quoted_keyspace = '"' + self.cqlsh.keyspace + '"'
|
||||
self.trycompletions('DROP KEYSPACE ',
|
||||
choices=['IF', quoted_keyspace])
|
||||
choices=['IF', self.cqlsh.keyspace])
|
||||
|
||||
self.trycompletions('DROP KEYSPACE ' + quoted_keyspace,
|
||||
choices=[';'])
|
||||
|
||||
self.trycompletions('DROP KEYSPACE I',
|
||||
immediate='F EXISTS ' + quoted_keyspace + ';')
|
||||
immediate='F EXISTS ' + self.cqlsh.keyspace + ' ;')
|
||||
|
||||
def create_columnfamily_table_template(self, name):
|
||||
"""Parameterized test for CREATE COLUMNFAMILY and CREATE TABLE. Since
|
||||
|
|
@ -544,11 +564,11 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
prefix = 'CREATE ' + name + ' '
|
||||
quoted_keyspace = '"' + self.cqlsh.keyspace + '"'
|
||||
self.trycompletions(prefix + '',
|
||||
choices=['IF', quoted_keyspace, '<new_table_name>'])
|
||||
choices=['IF', self.cqlsh.keyspace, '<new_table_name>'])
|
||||
self.trycompletions(prefix + 'IF ',
|
||||
immediate='NOT EXISTS ')
|
||||
self.trycompletions(prefix + 'IF NOT EXISTS ',
|
||||
choices=['<new_table_name>', quoted_keyspace])
|
||||
choices=['<new_table_name>', self.cqlsh.keyspace])
|
||||
self.trycompletions(prefix + 'IF NOT EXISTS new_table ',
|
||||
immediate='( ')
|
||||
|
||||
|
|
@ -594,7 +614,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'memtable_flush_period_in_ms',
|
||||
'CLUSTERING',
|
||||
'COMPACT', 'caching', 'comment',
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc'])
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
|
||||
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ',
|
||||
choices=['bloom_filter_fp_chance', 'compaction',
|
||||
'compression',
|
||||
|
|
@ -603,7 +623,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'memtable_flush_period_in_ms',
|
||||
'CLUSTERING',
|
||||
'COMPACT', 'caching', 'comment',
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc'])
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
|
||||
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance ',
|
||||
immediate='= ')
|
||||
self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance = ',
|
||||
|
|
@ -650,7 +670,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'memtable_flush_period_in_ms',
|
||||
'CLUSTERING',
|
||||
'COMPACT', 'caching', 'comment',
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc'])
|
||||
'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair'])
|
||||
self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = "
|
||||
+ "{'class': 'DateTieredCompactionStrategy', '",
|
||||
choices=['base_time_seconds', 'max_sstable_age_days',
|
||||
|
|
@ -694,7 +714,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'utf8_with_special_chars',
|
||||
'system_traces.', 'songs',
|
||||
'system_distributed.',
|
||||
'"' + self.cqlsh.keyspace + '".'],
|
||||
self.cqlsh.keyspace + '.'],
|
||||
other_choices_ok=True)
|
||||
|
||||
self.trycompletions('DESC TYPE ',
|
||||
|
|
@ -717,7 +737,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'fbestsong',
|
||||
'fmax',
|
||||
'fmin',
|
||||
'"' + self.cqlsh.keyspace + '".'],
|
||||
self.cqlsh.keyspace + '.'],
|
||||
other_choices_ok=True)
|
||||
|
||||
self.trycompletions('DESC AGGREGATE ',
|
||||
|
|
@ -727,7 +747,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
|
|||
'system_distributed.',
|
||||
'aggmin',
|
||||
'aggmax',
|
||||
'"' + self.cqlsh.keyspace + '".'],
|
||||
self.cqlsh.keyspace + '.'],
|
||||
other_choices_ok=True)
|
||||
|
||||
# Unfortunately these commented tests will not work. This is due to the keyspace name containing quotes;
|
||||
|
|
|
|||
|
|
@ -1,78 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
from .basecase import BaseTestCase
|
||||
|
||||
class TestCqlshInvocation(BaseTestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_normal_run(self):
|
||||
pass
|
||||
|
||||
def test_python_interpreter_location(self):
|
||||
pass
|
||||
|
||||
def test_color_capability_detection(self):
|
||||
pass
|
||||
|
||||
def test_colored_output(self):
|
||||
pass
|
||||
|
||||
def test_color_cmdline_option(self):
|
||||
pass
|
||||
|
||||
def test_debug_option(self):
|
||||
pass
|
||||
|
||||
def test_connection_args(self):
|
||||
pass
|
||||
|
||||
def test_connection_config(self):
|
||||
pass
|
||||
|
||||
def test_connection_envvars(self):
|
||||
pass
|
||||
|
||||
def test_command_history(self):
|
||||
pass
|
||||
|
||||
def test_missing_dependencies(self):
|
||||
pass
|
||||
|
||||
def test_completekey_config(self):
|
||||
pass
|
||||
|
||||
def test_ctrl_c(self):
|
||||
pass
|
||||
|
||||
def test_eof(self):
|
||||
pass
|
||||
|
||||
def test_output_encoding_detection(self):
|
||||
pass
|
||||
|
||||
def test_output_encoding(self):
|
||||
pass
|
||||
|
||||
def test_retries(self):
|
||||
pass
|
||||
|
|
@ -17,35 +17,55 @@
|
|||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
from __future__ import with_statement
|
||||
from __future__ import unicode_literals, with_statement
|
||||
|
||||
import locale
|
||||
import os
|
||||
import re
|
||||
from itertools import izip
|
||||
from .basecase import (BaseTestCase, cqlshlog, dedent, at_a_time, cqlsh,
|
||||
TEST_HOST, TEST_PORT)
|
||||
from .cassconnect import (get_keyspace, testrun_cqlsh, testcall_cqlsh,
|
||||
cassandra_cursor, split_cql_commands, quote_name)
|
||||
from .ansi_colors import (ColoredText, lookup_colorcode, lookup_colorname,
|
||||
lookup_colorletter, ansi_seq)
|
||||
import unittest
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT,
|
||||
at_a_time, cqlsh, cqlshlog, dedent)
|
||||
from .cassconnect import (cassandra_cursor, create_db, get_keyspace,
|
||||
quote_name, remove_db, split_cql_commands,
|
||||
testcall_cqlsh, testrun_cqlsh)
|
||||
from .ansi_colors import (ColoredText, ansi_seq, lookup_colorcode,
|
||||
lookup_colorname, lookup_colorletter)
|
||||
|
||||
CONTROL_C = '\x03'
|
||||
CONTROL_D = '\x04'
|
||||
|
||||
class TestCqlshOutput(BaseTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
create_db()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
remove_db()
|
||||
|
||||
def setUp(self):
|
||||
pass
|
||||
env = os.environ
|
||||
env['COLUMNS'] = '100000'
|
||||
# carry forward or override locale LC_CTYPE for UTF-8 encoding
|
||||
if (locale.getpreferredencoding() != 'UTF-8'):
|
||||
env['LC_CTYPE'] = 'en_US.utf8'
|
||||
else:
|
||||
env['LC_CTYPE'] = os.environ.get('LC_CTYPE', 'en_US.utf8')
|
||||
if ('PATH' in os.environ.keys()):
|
||||
env['PATH'] = os.environ['PATH']
|
||||
self.default_env = env
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def assertNoHasColors(self, text, msg=None):
|
||||
self.assertNotRegexpMatches(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text)
|
||||
self.assertNotRegex(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text)
|
||||
|
||||
def assertHasColors(self, text, msg=None):
|
||||
self.assertRegexpMatches(text, ansi_seq, msg=msg)
|
||||
self.assertRegex(text, ansi_seq, msg=msg)
|
||||
|
||||
def assertColored(self, coloredtext, colorname):
|
||||
wanted_colorcode = lookup_colorcode(colorname)
|
||||
|
|
@ -57,7 +77,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
% (coloredtext, num, lookup_colorname(ccolor), colorname))
|
||||
|
||||
def assertColorFromTags(self, coloredtext, tags):
|
||||
for (char, tag) in izip(coloredtext, tags):
|
||||
for (char, tag) in zip(coloredtext, tags):
|
||||
if char.isspace():
|
||||
continue
|
||||
if tag.isspace():
|
||||
|
|
@ -67,12 +87,14 @@ class TestCqlshOutput(BaseTestCase):
|
|||
'Actually got: %s\ncolor code: %s'
|
||||
% (tags, coloredtext.colored_version(), coloredtext.colortags()))
|
||||
|
||||
def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, **kwargs):
|
||||
def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, env=None, **kwargs):
|
||||
"""
|
||||
Allow queries and expected output to be specified in structured tuples,
|
||||
along with expected color information.
|
||||
"""
|
||||
with testrun_cqlsh(tty=True, **kwargs) as c:
|
||||
if env is None:
|
||||
env = self.default_env
|
||||
with testrun_cqlsh(tty=True, env=env, **kwargs) as c:
|
||||
for query, expected in queries_and_expected_outputs:
|
||||
cqlshlog.debug('Testing %r' % (query,))
|
||||
output = c.cmd_and_response(query).lstrip("\r\n")
|
||||
|
|
@ -83,10 +105,25 @@ class TestCqlshOutput(BaseTestCase):
|
|||
self.assertEqual(outputline.plain().rstrip(), plain)
|
||||
self.assertColorFromTags(outputline, colorcodes)
|
||||
|
||||
def strip_read_repair_chance(self, describe_statement):
|
||||
"""
|
||||
Remove read_repair_chance and dclocal_read_repair_chance options
|
||||
from output of DESCRIBE statements. The resulting string may be
|
||||
reused as a CREATE statement.
|
||||
Useful after CASSANDRA-13910, which removed read_repair_chance
|
||||
options from CREATE statements but did not remove them completely
|
||||
from the system.
|
||||
"""
|
||||
describe_statement = re.sub(r"( AND)? (dclocal_)?read_repair_chance = [\d\.]+", "", describe_statement)
|
||||
describe_statement = re.sub(r"WITH[\s]*;", "", describe_statement)
|
||||
return describe_statement
|
||||
|
||||
def test_no_color_output(self):
|
||||
env = self.default_env
|
||||
for termname in ('', 'dumb', 'vt100'):
|
||||
cqlshlog.debug('TERM=%r' % termname)
|
||||
with testrun_cqlsh(tty=True, env={'TERM': termname},
|
||||
env['TERM'] = termname
|
||||
with testrun_cqlsh(tty=True, env=env,
|
||||
win_force_colors=False) as c:
|
||||
c.send('select * from has_all_types;\n')
|
||||
self.assertNoHasColors(c.read_to_next_prompt())
|
||||
|
|
@ -96,15 +133,17 @@ class TestCqlshOutput(BaseTestCase):
|
|||
self.assertNoHasColors(c.read_to_next_prompt())
|
||||
|
||||
def test_no_prompt_or_colors_output(self):
|
||||
env = self.default_env
|
||||
for termname in ('', 'dumb', 'vt100', 'xterm'):
|
||||
cqlshlog.debug('TERM=%r' % termname)
|
||||
env['TERM'] = termname
|
||||
query = 'select * from has_all_types limit 1;'
|
||||
output, result = testcall_cqlsh(prompt=None, env={'TERM': termname},
|
||||
output, result = testcall_cqlsh(prompt=None, env=env,
|
||||
tty=False, input=query + '\n')
|
||||
output = output.splitlines()
|
||||
for line in output:
|
||||
self.assertNoHasColors(line)
|
||||
self.assertNotRegexpMatches(line, r'^cqlsh\S*>')
|
||||
self.assertNotRegex(line, r'^cqlsh\S*>')
|
||||
self.assertEqual(len(output), 6,
|
||||
msg='output: %r' % '\n'.join(output))
|
||||
self.assertEqual(output[0], '')
|
||||
|
|
@ -115,9 +154,12 @@ class TestCqlshOutput(BaseTestCase):
|
|||
self.assertEqual(output[5].strip(), '(1 rows)')
|
||||
|
||||
def test_color_output(self):
|
||||
env = self.default_env
|
||||
for termname in ('xterm', 'unknown-garbage'):
|
||||
cqlshlog.debug('TERM=%r' % termname)
|
||||
with testrun_cqlsh(tty=True, env={'TERM': termname}) as c:
|
||||
env['TERMNAME'] = termname
|
||||
env['TERM'] = termname
|
||||
with testrun_cqlsh(tty=True, env=env) as c:
|
||||
c.send('select * from has_all_types;\n')
|
||||
self.assertHasColors(c.read_to_next_prompt())
|
||||
c.send('select count(*) from has_all_types;\n')
|
||||
|
|
@ -349,6 +391,8 @@ class TestCqlshOutput(BaseTestCase):
|
|||
))
|
||||
|
||||
def test_timestamp_output(self):
|
||||
env = self.default_env
|
||||
env['TZ'] = 'Etc/UTC'
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
('''select timestampcol from has_all_types where num = 0;''', """
|
||||
timestampcol
|
||||
|
|
@ -362,9 +406,10 @@ class TestCqlshOutput(BaseTestCase):
|
|||
(1 rows)
|
||||
nnnnnnnn
|
||||
"""),
|
||||
), env={'TZ': 'Etc/UTC'})
|
||||
), env=env)
|
||||
try:
|
||||
import pytz # test only if pytz is available on PYTHONPATH
|
||||
env['TZ'] = 'America/Sao_Paulo'
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
('''select timestampcol from has_all_types where num = 0;''', """
|
||||
timestampcol
|
||||
|
|
@ -378,7 +423,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
(1 rows)
|
||||
nnnnnnnn
|
||||
"""),
|
||||
), env={'TZ': 'America/Sao_Paulo'})
|
||||
), env=env)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
|
@ -470,8 +515,10 @@ class TestCqlshOutput(BaseTestCase):
|
|||
# terminals, but the color-checking machinery here will still treat
|
||||
# it as one character, so those won't seem to line up visually either.
|
||||
|
||||
env = self.default_env
|
||||
env['LANG'] = 'en_US.UTF-8'
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", u"""
|
||||
("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", """
|
||||
k | val
|
||||
R MMM
|
||||
---+-------------------------------
|
||||
|
|
@ -494,8 +541,8 @@ class TestCqlshOutput(BaseTestCase):
|
|||
|
||||
(7 rows)
|
||||
nnnnnnnn
|
||||
""".encode('utf-8')),
|
||||
), env={'LANG': 'en_US.UTF-8'})
|
||||
"""),
|
||||
), env=env)
|
||||
|
||||
def test_blob_output(self):
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
|
|
@ -516,11 +563,11 @@ class TestCqlshOutput(BaseTestCase):
|
|||
|
||||
(4 rows)
|
||||
nnnnnnnn
|
||||
"""),
|
||||
""", ),
|
||||
))
|
||||
|
||||
def test_prompt(self):
|
||||
with testrun_cqlsh(tty=True, keyspace=None) as c:
|
||||
with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
|
||||
self.assertTrue(c.output_header.splitlines()[-1].endswith('cqlsh> '))
|
||||
|
||||
c.send('\n')
|
||||
|
|
@ -552,7 +599,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
"RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR")
|
||||
|
||||
def test_describe_keyspace_output(self):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
ks = get_keyspace()
|
||||
qks = quote_name(ks)
|
||||
for cmd in ('describe keyspace', 'desc keyspace'):
|
||||
|
|
@ -572,27 +619,27 @@ class TestCqlshOutput(BaseTestCase):
|
|||
with cassandra_cursor() as curs:
|
||||
try:
|
||||
for stmt in statements:
|
||||
stmt = self.strip_read_repair_chance(stmt)
|
||||
cqlshlog.debug('TEST EXEC: %s' % stmt)
|
||||
curs.execute(stmt)
|
||||
finally:
|
||||
curs.execute('use system')
|
||||
if do_drop:
|
||||
curs.execute('drop keyspace %s' % quote_name(new_ks_name))
|
||||
curs.execute('drop keyspace {}'.format(new_ks_name))
|
||||
|
||||
def check_describe_keyspace_output(self, output, qksname):
|
||||
expected_bits = [r'(?im)^CREATE KEYSPACE %s WITH\b' % re.escape(qksname),
|
||||
r';\s*$',
|
||||
r'\breplication = {\'class\':']
|
||||
for expr in expected_bits:
|
||||
self.assertRegexpMatches(output, expr)
|
||||
self.assertRegex(output, expr)
|
||||
|
||||
def test_describe_columnfamily_output(self):
|
||||
# we can change these to regular expressions if/when it makes sense
|
||||
# to do so; these will likely be subject to lots of adjustments.
|
||||
|
||||
|
||||
# note columns are now comparator-ordered instead of original-order.
|
||||
table_desc3 = dedent("""
|
||||
|
||||
CREATE TABLE %s.has_all_types (
|
||||
num int PRIMARY KEY,
|
||||
asciicol ascii,
|
||||
|
|
@ -615,37 +662,35 @@ class TestCqlshOutput(BaseTestCase):
|
|||
AND cdc = false
|
||||
AND comment = ''
|
||||
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
|
||||
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
|
||||
AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
|
||||
AND crc_check_chance = 1.0
|
||||
AND dclocal_read_repair_chance = 0.0
|
||||
AND default_time_to_live = 0
|
||||
AND gc_grace_seconds = 864000
|
||||
AND max_index_interval = 2048
|
||||
AND memtable_flush_period_in_ms = 0
|
||||
AND min_index_interval = 128
|
||||
AND speculative_retry = '99PERCENTILE'
|
||||
AND additional_write_policy = '99PERCENTILE';
|
||||
AND read_repair_chance = 0.0
|
||||
AND speculative_retry = '99p';
|
||||
""" % quote_name(get_keyspace()))
|
||||
|
||||
""" % quote_name(get_keyspace()))
|
||||
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
for cmdword in ('describe table', 'desc columnfamily'):
|
||||
for semicolon in (';', ''):
|
||||
output = c.cmd_and_response('%s has_all_types%s' % (cmdword, semicolon))
|
||||
self.assertNoHasColors(output)
|
||||
self.assertSequenceEqual(output.split('\n'), table_desc3.split('\n'))
|
||||
self.assertSequenceEqual(dedent(output).split('\n'), table_desc3.split('\n'))
|
||||
|
||||
def test_describe_columnfamilies_output(self):
|
||||
output_re = r'''
|
||||
\n
|
||||
Keyspace [ ] (?P<ksname> \S+ ) \n
|
||||
\n? Keyspace [ ] (?P<ksname> \S+ ) \n
|
||||
-----------* \n
|
||||
(?P<cfnames> .*? )
|
||||
\n
|
||||
'''
|
||||
(?P<cfnames> ( ( ["']?\w+["']? [^\w\n]* )+ \n )* )
|
||||
'''
|
||||
|
||||
ks = get_keyspace()
|
||||
|
||||
with testrun_cqlsh(tty=True, keyspace=None) as c:
|
||||
with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
|
||||
|
||||
# when not in a keyspace
|
||||
for cmdword in ('DESCRIBE COLUMNFAMILIES', 'desc tables'):
|
||||
|
|
@ -653,7 +698,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
ksnames = []
|
||||
output = c.cmd_and_response(cmdword + semicolon)
|
||||
self.assertNoHasColors(output)
|
||||
self.assertRegexpMatches(output, '(?xs) ^ ( %s )+ $' % output_re)
|
||||
self.assertRegex(output, '(?xs) ^ ( %s )+ $' % output_re)
|
||||
|
||||
for section in re.finditer('(?xs)' + output_re, output):
|
||||
ksname = section.group('ksname')
|
||||
|
|
@ -696,13 +741,13 @@ class TestCqlshOutput(BaseTestCase):
|
|||
\n
|
||||
'''
|
||||
|
||||
with testrun_cqlsh(tty=True, keyspace=None) as c:
|
||||
with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c:
|
||||
|
||||
# not in a keyspace
|
||||
for semicolon in ('', ';'):
|
||||
output = c.cmd_and_response('describe cluster' + semicolon)
|
||||
self.assertNoHasColors(output)
|
||||
self.assertRegexpMatches(output, output_re + '$')
|
||||
self.assertRegex(output, output_re + '$')
|
||||
|
||||
c.send('USE %s;\n' % quote_name(get_keyspace()))
|
||||
c.read_to_next_prompt()
|
||||
|
|
@ -710,32 +755,32 @@ class TestCqlshOutput(BaseTestCase):
|
|||
for semicolon in ('', ';'):
|
||||
output = c.cmd_and_response('describe cluster' + semicolon)
|
||||
self.assertNoHasColors(output)
|
||||
self.assertRegexpMatches(output, output_re + ringinfo_re + '$')
|
||||
self.assertRegex(output, output_re + ringinfo_re + '$')
|
||||
|
||||
def test_describe_schema_output(self):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
for semicolon in ('', ';'):
|
||||
output = c.cmd_and_response('desc full schema' + semicolon)
|
||||
self.assertNoHasColors(output)
|
||||
self.assertRegexpMatches(output, '^\nCREATE KEYSPACE')
|
||||
self.assertRegex(output, '^\nCREATE KEYSPACE')
|
||||
self.assertIn("\nCREATE KEYSPACE system WITH replication = {'class': 'LocalStrategy'} AND durable_writes = true;\n",
|
||||
output)
|
||||
self.assertRegexpMatches(output, ';\s*$')
|
||||
self.assertRegex(output, ';\s*$')
|
||||
|
||||
def test_show_output(self):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
output = c.cmd_and_response('show version;')
|
||||
self.assertRegexpMatches(output,
|
||||
self.assertRegex(output,
|
||||
'^\[cqlsh \S+ \| Cassandra \S+ \| CQL spec \S+ \| Native protocol \S+\]$')
|
||||
|
||||
output = c.cmd_and_response('show host;')
|
||||
self.assertHasColors(output)
|
||||
self.assertRegexpMatches(output, '^Connected to .* at %s:%d\.$'
|
||||
self.assertRegex(output, '^Connected to .* at %s:%d\.$'
|
||||
% (re.escape(TEST_HOST), TEST_PORT))
|
||||
|
||||
@unittest.skipIf(sys.platform == "win32", 'EOF signaling not supported on Windows')
|
||||
def test_eof_prints_newline(self):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
c.send(CONTROL_D)
|
||||
out = c.read_lines(1)[0].replace('\r', '')
|
||||
self.assertEqual(out, '\n')
|
||||
|
|
@ -745,7 +790,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
|
||||
def test_exit_prints_no_newline(self):
|
||||
for semicolon in ('', ';'):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
cmd = 'exit%s\n' % semicolon
|
||||
c.send(cmd)
|
||||
if c.realtty:
|
||||
|
|
@ -756,7 +801,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
self.assertIn(type(cm.exception), (EOFError, OSError))
|
||||
|
||||
def test_help_types(self):
|
||||
with testrun_cqlsh(tty=True) as c:
|
||||
with testrun_cqlsh(tty=True, env=self.default_env) as c:
|
||||
c.cmd_and_response('help types')
|
||||
|
||||
def test_help(self):
|
||||
|
|
@ -800,6 +845,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
nnnnnnnn
|
||||
"""),
|
||||
))
|
||||
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
("select phone_numbers from users;", r"""
|
||||
phone_numbers
|
||||
|
|
@ -832,6 +878,7 @@ class TestCqlshOutput(BaseTestCase):
|
|||
nnnnnnnn
|
||||
"""),
|
||||
))
|
||||
|
||||
self.assertQueriesGiveColoredOutput((
|
||||
("select tags from songs;", r"""
|
||||
tags
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# to configure behavior, define $CQL_TEST_HOST to the destination address
|
||||
# and $CQL_TEST_PORT to the associated port.
|
||||
|
||||
from unittest import TestCase
|
||||
|
||||
|
||||
class TestCqlshParsing(TestCase):
|
||||
def test_describe(self):
|
||||
pass
|
||||
|
||||
|
|
@ -15,11 +15,11 @@
|
|||
# limitations under the License.
|
||||
|
||||
from threading import Thread
|
||||
from cStringIO import StringIO
|
||||
from Queue import Queue, Empty
|
||||
from six import StringIO
|
||||
from six.moves.queue import Queue, Empty
|
||||
|
||||
|
||||
class WinPty:
|
||||
class WinPty(object):
|
||||
|
||||
def __init__(self, stdin):
|
||||
self._s = stdin
|
||||
|
|
@ -47,4 +47,4 @@ class WinPty:
|
|||
count = count + 1
|
||||
except Empty:
|
||||
pass
|
||||
return buf.getvalue()
|
||||
return buf.getvalue()
|
||||
|
|
@ -14,11 +14,12 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from cqlshlib.displaying import MAGENTA
|
||||
from datetime import datetime, timedelta
|
||||
from formatting import CqlType
|
||||
import time
|
||||
|
||||
from cassandra.query import QueryTrace, TraceUnavailable
|
||||
from cqlshlib.displaying import MAGENTA
|
||||
from cqlshlib.formatting import CqlType
|
||||
|
||||
|
||||
def print_trace_session(shell, session, session_id, partial_session=False):
|
||||
|
|
@ -45,8 +46,8 @@ def print_trace(shell, trace):
|
|||
return
|
||||
names = ['activity', 'timestamp', 'source', 'source_elapsed', 'client']
|
||||
|
||||
formatted_names = map(shell.myformat_colname, names)
|
||||
formatted_values = [map(shell.myformat_value, row) for row in rows]
|
||||
formatted_names = list(map(shell.myformat_colname, names))
|
||||
formatted_values = [list(map(shell.myformat_value, row)) for row in rows]
|
||||
|
||||
shell.writeresult('')
|
||||
shell.writeresult('Tracing session: ', color=MAGENTA, newline=False)
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ import cProfile
|
|||
import codecs
|
||||
import pstats
|
||||
|
||||
from itertools import izip
|
||||
|
||||
from datetime import timedelta, tzinfo
|
||||
from StringIO import StringIO
|
||||
from six import StringIO
|
||||
|
||||
try:
|
||||
from line_profiler import LineProfiler
|
||||
|
|
@ -77,7 +77,7 @@ def find_common_prefix(strs):
|
|||
"""
|
||||
|
||||
common = []
|
||||
for cgroup in izip(*strs):
|
||||
for cgroup in zip(*strs):
|
||||
if all(x == cgroup[0] for x in cgroup[1:]):
|
||||
common.append(cgroup[0])
|
||||
else:
|
||||
|
|
@ -161,6 +161,6 @@ def profile_off(pr, file_name):
|
|||
ret = s.getvalue()
|
||||
if file_name:
|
||||
with open(file_name, 'w') as f:
|
||||
print "Writing to %s\n" % (f.name, )
|
||||
print("Writing to %s\n" % (f.name, ))
|
||||
f.write(ret)
|
||||
return ret
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def bisearch(ucs, table):
|
|||
if ucs < table[0][0] or ucs > table[max][1]:
|
||||
return 0
|
||||
while max >= min:
|
||||
mid = (min + max) / 2
|
||||
mid = int((min + max) / 2)
|
||||
if ucs > table[mid][1]:
|
||||
min = mid + 1
|
||||
elif ucs < table[mid][0]:
|
||||
|
|
@ -251,20 +251,21 @@ def mk_wcwidth(ucs):
|
|||
|
||||
# if we arrive here, ucs is not a combining or C0/C1 control character
|
||||
|
||||
return 1 + \
|
||||
int(ucs >= 0x1100 and
|
||||
(ucs <= 0x115f or # Hangul Jamo init. consonants
|
||||
ucs == 0x2329 or ucs == 0x232a or
|
||||
(ucs >= 0x2e80 and ucs <= 0xa4cf and
|
||||
ucs != 0x303f) or # CJK ... Yi
|
||||
(ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables
|
||||
(ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs
|
||||
(ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms
|
||||
(ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms
|
||||
(ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms
|
||||
(ucs >= 0xffe0 and ucs <= 0xffe6) or
|
||||
(ucs >= 0x20000 and ucs <= 0x2fffd) or
|
||||
(ucs >= 0x30000 and ucs <= 0x3fffd)))
|
||||
return 1 + int(
|
||||
ucs >= 0x1100
|
||||
and (ucs <= 0x115f # Hangul Jamo init. consonants
|
||||
or ucs == 0x2329 or ucs == 0x232a
|
||||
or (ucs >= 0x2e80 and ucs <= 0xa4cf
|
||||
and ucs != 0x303f) # CJK ... Yi
|
||||
or (ucs >= 0xac00 and ucs <= 0xd7a3) # Hangul Syllables
|
||||
or (ucs >= 0xf900 and ucs <= 0xfaff) # CJK Compatibility Ideographs
|
||||
or (ucs >= 0xfe10 and ucs <= 0xfe19) # Vertical forms
|
||||
or (ucs >= 0xfe30 and ucs <= 0xfe6f) # CJK Compatibility Forms
|
||||
or (ucs >= 0xff00 and ucs <= 0xff60) # Fullwidth Forms
|
||||
or (ucs >= 0xffe0 and ucs <= 0xffe6)
|
||||
or (ucs >= 0x20000 and ucs <= 0x2fffd)
|
||||
or (ucs >= 0x30000 and ucs <= 0x3fffd))
|
||||
)
|
||||
|
||||
|
||||
def mk_wcswidth(pwcs):
|
||||
|
|
@ -313,7 +314,7 @@ def wcwidth(c):
|
|||
|
||||
|
||||
def wcswidth(s):
|
||||
return mk_wcswidth(map(ord, s))
|
||||
return mk_wcswidth(list(map(ord, s)))
|
||||
|
||||
|
||||
def wcwidth_cjk(c):
|
||||
|
|
@ -321,7 +322,7 @@ def wcwidth_cjk(c):
|
|||
|
||||
|
||||
def wcswidth_cjk(s):
|
||||
return mk_wcswidth_cjk(map(ord, s))
|
||||
return mk_wcswidth_cjk(list(map(ord, s)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -341,7 +342,7 @@ if __name__ == "__main__":
|
|||
('COMBINING PALATALIZED HOOK BELOW', 0),
|
||||
('COMBINING GRAVE ACCENT', 0),
|
||||
)
|
||||
nonprinting = u'\r\n\t\a\b\f\v\x7f'
|
||||
nonprinting = '\r\n\t\a\b\f\v\x7f'
|
||||
|
||||
import unicodedata
|
||||
|
||||
|
|
@ -366,13 +367,13 @@ if __name__ == "__main__":
|
|||
assert mk_wcwidth(0x10ffff) == 1
|
||||
assert mk_wcwidth(0x3fffd) == 2
|
||||
|
||||
teststr = u'B\0ig br\u00f8wn moose\ub143\u200b'
|
||||
teststr = 'B\0ig br\u00f8wn moose\ub143\u200b'
|
||||
calculatedwidth = wcswidth(teststr)
|
||||
assert calculatedwidth == 17, 'expected 17, got %d' % calculatedwidth
|
||||
|
||||
calculatedwidth = wcswidth_cjk(teststr)
|
||||
assert calculatedwidth == 18, 'expected 18, got %d' % calculatedwidth
|
||||
|
||||
assert wcswidth(u'foobar\u200b\a') < 0
|
||||
assert wcswidth('foobar\u200b\a') < 0
|
||||
|
||||
print 'tests pass.'
|
||||
print('tests pass.')
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
# cythonizing the driver, perhaps only on old pips.
|
||||
# http://datastax.github.io/python-driver/installation.html#cython-based-extensions
|
||||
futures
|
||||
six
|
||||
six>=0.12.0
|
||||
-e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver
|
||||
# Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
|
||||
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm
|
||||
coverage
|
||||
cql
|
||||
decorator
|
||||
docopt
|
||||
|
|
@ -17,5 +18,4 @@ nose-test-select
|
|||
parse
|
||||
pycodestyle
|
||||
psutil
|
||||
pycassa
|
||||
thrift==0.9.3
|
||||
|
|
|
|||
Loading…
Reference in New Issue