diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 650b9a0bc5..8bb7c00920 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -138,6 +138,46 @@ j8_with_dtests_jobs: &j8_with_dtests_jobs - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - j8_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - start_j8_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - j8_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes j11_with_dtests_jobs: &j11_with_dtests_jobs jobs: @@ -164,6 +204,26 @@ j11_with_dtests_jobs: &j11_with_dtests_jobs - j11_dtests-no-vnodes: requires: - start_j11_dtests + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes j8_with_dtest_jobs_only: &j8_with_dtest_jobs_only jobs: @@ -355,7 +415,7 @@ jobs: - create_venv - create_dtest_containers: file_tag: j8_with_vnodes - run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'" - run_dtests: file_tag: j8_with_vnodes pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' @@ -370,7 +430,7 @@ jobs: - create_venv - create_dtest_containers: file_tag: j11_with_vnodes - run_dtests_extra_args: '--use-vnodes --skip-resource-intensive-tests' + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql'" - run_dtests: file_tag: j11_with_vnodes pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' @@ -384,7 +444,7 @@ jobs: - create_venv - create_dtest_containers: file_tag: j8_without_vnodes - run_dtests_extra_args: '--skip-resource-intensive-tests' + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'" - run_dtests: file_tag: j8_without_vnodes pytest_extra_args: '--skip-resource-intensive-tests' @@ -399,7 +459,7 @@ jobs: - create_venv - create_dtest_containers: file_tag: j11_without_vnodes - run_dtests_extra_args: '--skip-resource-intensive-tests' + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k not cql'" - run_dtests: file_tag: j11_without_vnodes pytest_extra_args: '--skip-resource-intensive-tests' @@ -421,6 +481,126 @@ jobs: extra_env_args: 'RUN_STATIC_UPGRADE_MATRIX=true' pytest_extra_args: '--execute-upgrade-tests' + j8_cqlsh-dtests-py2-with-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-with-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j8_cqlsh-dtests-py2-no-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j8_cqlsh-dtests-py3-no-vnodes: + <<: *j8_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j8_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j8_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py2-with-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-with-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_with_vnodes + run_dtests_extra_args: "--use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_with_vnodes + pytest_extra_args: '--use-vnodes --num-tokens=32 --skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + + j11_cqlsh-dtests-py2-no-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python2.7' + + j11_cqlsh-dtests-py3-no-vnodes: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - clone_dtest + - create_venv + - create_dtest_containers: + file_tag: j11_without_vnodes + run_dtests_extra_args: "--skip-resource-intensive-tests --pytest-options '-k cql'" + - run_dtests: + file_tag: j11_without_vnodes + pytest_extra_args: '--skip-resource-intensive-tests' + extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6' + commands: log_environment: steps: @@ -635,6 +815,7 @@ commands: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze create_dtest_containers: diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e07fd285a..dcf444f56d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,82 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_unit_tests: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -173,6 +249,389 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -223,7 +682,7 @@ jobs: - run: name: Determine Tests to Run (j11_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - run: name: Run dtests (j11_with_vnodes) no_output_timeout: 15m @@ -281,7 +740,7 @@ jobs: - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m @@ -707,6 +1166,83 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -735,7 +1271,7 @@ jobs: - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m @@ -930,6 +1466,82 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -1070,7 +1682,7 @@ jobs: - run: name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: name: Run dtests (j11_without_vnodes) no_output_timeout: 15m @@ -1336,6 +1948,46 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - j8_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - start_j8_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - j8_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes java11_build_and_run_tests: jobs: - j11_build @@ -1359,3 +2011,23 @@ workflows: - j11_dtests-no-vnodes: requires: - start_j11_dtests + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes diff --git a/.circleci/config.yml.HIGHRES b/.circleci/config.yml.HIGHRES index 47350fded2..5626d7898e 100644 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@ -86,6 +86,83 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_unit_tests: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -173,6 +250,394 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -219,11 +684,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j11_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - run: name: Run dtests (j11_with_vnodes) no_output_timeout: 15m @@ -277,11 +743,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m @@ -334,6 +801,7 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_upgradetests_without_vnodes) @@ -707,6 +1175,84 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -731,11 +1277,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m @@ -930,6 +1477,83 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -1066,11 +1690,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: name: Run dtests (j11_without_vnodes) no_output_timeout: 15m @@ -1336,6 +1961,46 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - j8_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - start_j8_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - j8_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes java11_build_and_run_tests: jobs: - j11_build @@ -1359,3 +2024,23 @@ workflows: - j11_dtests-no-vnodes: requires: - start_j11_dtests + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes diff --git a/.circleci/config.yml.LOWRES b/.circleci/config.yml.LOWRES index 6e07fd285a..88dce2a052 100644 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@ -86,6 +86,83 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 j11_unit_tests: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -173,6 +250,394 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_cqlsh-dtests-py3-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + - run: + name: Run dtests (j8_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_cqlsh-dtests-py2-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j11_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11:20181210 @@ -219,11 +684,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j11_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" - run: name: Run dtests (j11_with_vnodes) no_output_timeout: 15m @@ -277,11 +743,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" - run: name: Run dtests (j8_without_vnodes) no_output_timeout: 15m @@ -334,6 +801,7 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_upgradetests_without_vnodes) @@ -707,6 +1175,84 @@ jobs: - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 - CASSANDRA_USE_JDK11: true + j11_cqlsh-dtests-py2-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then + export CQLSH_PYTHON=/usr/bin/python2.7 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -731,11 +1277,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j8_with_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_with_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_with_vnodes_raw /tmp/all_dtest_tests_j8_with_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_with_vnodes_raw > /tmp/all_dtest_tests_j8_with_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_with_vnodes > /tmp/split_dtest_tests_j8_with_vnodes.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_with_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_with_vnodes_final.txt\n" - run: name: Run dtests (j8_with_vnodes) no_output_timeout: 15m @@ -930,6 +1477,83 @@ jobs: - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_cqlsh-dtests-py3-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh + pip3 freeze + - run: + name: Determine Tests to Run (j8_without_vnodes) + no_output_timeout: 5m + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j8_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j8_without_vnodes_raw /tmp/all_dtest_tests_j8_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j8_without_vnodes_raw > /tmp/all_dtest_tests_j8_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j8_without_vnodes > /tmp/split_dtest_tests_j8_without_vnodes.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j8_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j8_without_vnodes_final.txt\n" + - run: + name: Run dtests (j8_without_vnodes) + no_output_timeout: 15m + command: | + echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" + cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt + + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + if [ -n 'CQLSH_PYTHON=/usr/bin/python3.6' ]; then + export CQLSH_PYTHON=/usr/bin/python3.6 + fi + + java -version + cd ~/cassandra-dtest + mkdir -p /tmp/dtest + + echo "env: $(env)" + echo "** done env" + mkdir -p /tmp/results/dtests + # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee + export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` + set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level="INFO" --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j8_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j8_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 utests_long: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@ -1066,11 +1690,12 @@ jobs: source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 uninstall -y cqlsh pip3 freeze - run: name: Determine Tests to Run (j11_without_vnodes) no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" + command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-intensive-tests --pytest-options '-k not cql' --dtest-print-tests-only --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" - run: name: Run dtests (j11_without_vnodes) no_output_timeout: 15m @@ -1336,6 +1961,46 @@ workflows: - j8_upgradetests-no-vnodes: requires: - start_upgrade_tests + - start_j8_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - j8_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j8_cqlsh_tests-with-vnodes + - start_j8_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j8_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - j8_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j8_cqlsh_tests-no-vnodes + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j8_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes java11_build_and_run_tests: jobs: - j11_build @@ -1359,3 +2024,23 @@ workflows: - j11_dtests-no-vnodes: requires: - start_j11_dtests + - start_j11_cqlsh_tests-with-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - j11_cqlsh-dtests-py3-with-vnodes: + requires: + - start_j11_cqlsh_tests-with-vnodes + - start_j11_cqlsh_tests-no-vnodes: + type: approval + requires: + - j11_build + - j11_cqlsh-dtests-py2-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes + - j11_cqlsh-dtests-py3-no-vnodes: + requires: + - start_j11_cqlsh_tests-no-vnodes diff --git a/CHANGES.txt b/CHANGES.txt index 6d3f2091f6..8762a02dc3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0-alpha4 + * Make cqlsh and cqlshlib Python 2 & 3 compatible (CASSANDRA-10190) * Added documentation for Full Query Logging (CASSANDRA-15475) * Added documentation for backups (CASSANDRA-15479) * Documentation gives the wrong instruction to activate remote jmx (CASSANDRA-15535) diff --git a/bin/cqlsh b/bin/cqlsh index 82a4a533a8..b7de04ab2c 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -15,12 +15,79 @@ # See the License for the specific language governing permissions and # limitations under the License. -# bash code here; finds a suitable python interpreter and execs this file. -# prefer unqualified "python" if suitable: -python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \ - && exec python "`python -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@" -for pyver in 2.7; do - which python$pyver > /dev/null 2>&1 && exec python$pyver "`python$pyver -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@" +# shell script to find a suitable Python interpreter and run cqlsh.py + +# parse arguments +PARAMS="" + +# Use the Python that is specified in the env +if [ -n "$CQLSH_PYTHON" ]; then + USER_SPECIFIED_PYTHON="$CQLSH_PYTHON" +fi + +while [ $# -gt 0 ]; do + case "$1" in + --python) + if [ $# -lt 2 ]; then + echo "You must specify a python interpreter path with the --python option" + exit 1 + fi + USER_SPECIFIED_PYTHON="$2" + shift + shift + ;; + --) + shift + break + ;; + *) + PARAMS="$PARAMS $1" + shift + ;; + esac done -echo "No appropriate python interpreter found." >&2 + +# get a version string for a Python interpreter +get_python_version() { + interpreter=$1 + version=$(command -v "$interpreter" > /dev/null 2>&1 && $interpreter -c "import os; print('{}.{}'.format(os.sys.version_info.major, os.sys.version_info.minor))") + echo "$version" +} + +# test whether a version string matches one of the supported versions for cqlsh +is_supported_version() { + version=$1 + if [ "$version" = "3.6" ] || [ "$version" = "2.7" ]; then + echo "supported" + else + echo "unsupported" + fi +} + +run_if_supported_version() { + interpreter="$1" + params="$2" + version=$(get_python_version "$interpreter") + if [ -n "$version" ]; then + if [ "$(is_supported_version "$version")" = "supported" ]; then + # We need the params to be unquoted, otherwise the shell will just interpret it as one giant string + # shellcheck disable=SC2086 + exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" $params + exit + fi + fi +} + + +if [ "$USER_SPECIFIED_PYTHON" != "" ]; then + # run a user specified Python interpreter + run_if_supported_version "$USER_SPECIFIED_PYTHON" "$PARAMS" +else + # try unqualified python first, then python3, then python2.7 + for interpreter in python python3 python2.7; do + run_if_supported_version "$interpreter" "$PARAMS" + done +fi + +echo "No appropriate Python interpreter found." >&2 exit 1 diff --git a/bin/cqlsh.py b/bin/cqlsh.py index e59d3c168d..6fc5fed9ef 100644 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -19,21 +19,21 @@ """:" # bash code here; finds a suitable python interpreter and execs this file. +# this implementation of cqlsh is compatible with both Python 3 and Python 2.7. # prefer unqualified "python" if suitable: -python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \ +python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion))' 2>/dev/null \ && exec python "$0" "$@" -for pyver in 2.7; do +for pyver in 3 2.7; do which python$pyver > /dev/null 2>&1 && exec python$pyver "$0" "$@" done echo "No appropriate python interpreter found." >&2 exit 1 ":""" -from __future__ import with_statement +from __future__ import division, unicode_literals import cmd import codecs -import ConfigParser import csv import getpass import optparse @@ -43,13 +43,12 @@ import sys import traceback import warnings import webbrowser -from StringIO import StringIO from contextlib import contextmanager from glob import glob from uuid import UUID -if sys.version_info[0] != 2 or sys.version_info[1] != 7: - sys.exit("\nCQL Shell supports only Python 2.7\n") +if sys.version_info.major != 3 and (sys.version_info.major == 2 and sys.version_info.minor != 7): + sys.exit("\nCQL Shell supports only Python 3 or Python 2.7\n") # see CASSANDRA-10428 if platform.python_implementation().startswith('Jython'): @@ -140,10 +139,17 @@ for lib in third_parties: if lib_zip: sys.path.insert(0, lib_zip) +# We cannot import six until we add its location to sys.path so the Python +# interpreter can find it. Do not move this to the top. +import six + +from six.moves import configparser, input +from six import StringIO, ensure_text, ensure_str + warnings.filterwarnings("ignore", r".*blist.*") try: import cassandra -except ImportError, e: +except ImportError as e: sys.exit("\nPython Cassandra driver not installed, or not on PYTHONPATH.\n" 'You might try "pip install cassandra-driver".\n\n' 'Python: %s\n' @@ -216,8 +222,10 @@ parser.add_option('-k', '--keyspace', help='Authenticate to the given keyspace.' parser.add_option("-f", "--file", help="Execute commands from FILE, then exit") parser.add_option('--debug', action='store_true', help='Show additional debugging information') -parser.add_option("--encoding", help="Specify a non-default encoding for output." + - " (Default: %s)" % (UTF8,)) +parser.add_option('--coverage', action='store_true', + help='Collect coverage data') +parser.add_option("--encoding", help="Specify a non-default encoding for output." + + " (Default: %s)" % (UTF8,)) parser.add_option("--cqlshrc", help="Specify an alternative cqlshrc file location.") parser.add_option('--cqlversion', default=None, help='Specify a particular CQL version, ' @@ -243,7 +251,7 @@ HISTORY_DIR = os.path.expanduser(os.path.join('~', '.cassandra')) if hasattr(options, 'cqlshrc'): CONFIG_FILE = options.cqlshrc if not os.path.exists(CONFIG_FILE): - print '\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, HISTORY_DIR) + print('\nWarning: Specified cqlshrc location `%s` does not exist. Using `%s` instead.\n' % (CONFIG_FILE, HISTORY_DIR)) CONFIG_FILE = os.path.join(HISTORY_DIR, 'cqlshrc') else: CONFIG_FILE = os.path.join(HISTORY_DIR, 'cqlshrc') @@ -253,16 +261,16 @@ if not os.path.exists(HISTORY_DIR): try: os.mkdir(HISTORY_DIR) except OSError: - print '\nWarning: Cannot create directory at `%s`. Command history will not be saved.\n' % HISTORY_DIR + print('\nWarning: Cannot create directory at `%s`. Command history will not be saved.\n' % HISTORY_DIR) OLD_CONFIG_FILE = os.path.expanduser(os.path.join('~', '.cqlshrc')) if os.path.exists(OLD_CONFIG_FILE): if os.path.exists(CONFIG_FILE): - print '\nWarning: cqlshrc config files were found at both the old location (%s) and \ - the new location (%s), the old config file will not be migrated to the new \ - location, and the new location will be used for now. You should manually \ - consolidate the config files at the new location and remove the old file.' \ - % (OLD_CONFIG_FILE, CONFIG_FILE) + print('\nWarning: cqlshrc config files were found at both the old location ({0})' + + ' and the new location ({1}), the old config file will not be migrated to the new' + + ' location, and the new location will be used for now. You should manually' + + ' consolidate the config files at the new location and remove the old file.' + .format(OLD_CONFIG_FILE, CONFIG_FILE)) else: os.rename(OLD_CONFIG_FILE, CONFIG_FILE) OLD_HISTORY = os.path.expanduser(os.path.join('~', '.cqlsh_history')) @@ -351,7 +359,7 @@ def full_cql_version(ver): while ver.count('.') < 2: ver += '.0' ver_parts = ver.split('-', 1) + [''] - vertuple = tuple(map(int, ver_parts[0].split('.')) + [ver_parts[1]]) + vertuple = tuple(list(map(int, ver_parts[0].split('.'))) + [ver_parts[1]]) return ver, vertuple @@ -404,34 +412,18 @@ def insert_driver_hooks(): cassandra.cqltypes.CassandraType.support_empty_values = True -class FrozenType(cassandra.cqltypes._ParameterizedType): - """ - Needed until the bundled python driver adds FrozenType. - """ - typename = "frozen" - num_subtypes = 1 - - @classmethod - def deserialize_safe(cls, byts, protocol_version): - subtype, = cls.subtypes - return subtype.from_binary(byts) - - @classmethod - def serialize_safe(cls, val, protocol_version): - subtype, = cls.subtypes - return subtype.to_binary(val, protocol_version) - - class Shell(cmd.Cmd): custom_prompt = os.getenv('CQLSH_PROMPT', '') if custom_prompt is not '': custom_prompt += "\n" default_prompt = custom_prompt + "cqlsh> " continue_prompt = " ... " - keyspace_prompt = custom_prompt + "cqlsh:%s> " - keyspace_continue_prompt = "%s ... " + keyspace_prompt = custom_prompt + "cqlsh:{}> " + keyspace_continue_prompt = "{} ... " show_line_nums = False debug = False + coverage = False + coveragerc_path = None stop = False last_hist = None shunted_query_out = None @@ -459,7 +451,8 @@ class Shell(cmd.Cmd): request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS, protocol_version=None, connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS, - allow_server_port_discovery=False): + allow_server_port_discovery=False, + is_subshell=False): cmd.Cmd.__init__(self, completekey=completekey) self.hostname = hostname self.port = port @@ -538,7 +531,7 @@ class Shell(cmd.Cmd): if tty: self.reset_prompt() self.report_connection() - print 'Use HELP for help.' + print('Use HELP for help.') else: self.show_line_nums = True self.stdin = stdin @@ -549,6 +542,7 @@ class Shell(cmd.Cmd): self.empty_lines = 0 self.statement_error = False self.single_statement = single_statement + self.is_subshell = is_subshell @property def is_using_utf8(self): @@ -583,7 +577,7 @@ class Shell(cmd.Cmd): return format_value(val, cqltype=cqltype, encoding=self.output_codec.name, addcolor=self.color, date_time_format=dtformats, float_precision=precision, **kwargs) - except Exception, e: + except Exception as e: err = FormatError(val, e) self.decoding_errors.append(err) return format_value(err, cqltype=cqltype, encoding=self.output_codec.name, addcolor=self.color) @@ -605,10 +599,10 @@ class Shell(cmd.Cmd): self.show_version() def show_host(self): - print "Connected to %s at %s:%d." % \ - (self.applycolor(self.get_cluster_name(), BLUE), - self.hostname, - self.port) + print("Connected to {0} at {1}:{2}." + .format(self.applycolor(self.get_cluster_name(), BLUE), + self.hostname, + self.port)) def show_version(self): vers = self.connection_versions.copy() @@ -616,7 +610,7 @@ class Shell(cmd.Cmd): # system.Versions['cql'] apparently does not reflect changes with # set_cql_version. vers['cql'] = self.cql_version - print "[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers + print("[cqlsh %(shver)s | Cassandra %(build)s | CQL spec %(cql)s | Native protocol v%(protocol)s]" % vers) def show_session(self, sessionid, partial_session=False): print_trace_session(self, self.session, sessionid, partial_session) @@ -625,7 +619,7 @@ class Shell(cmd.Cmd): result, = self.session.execute("select * from system.local where key = 'local'") vers = { 'build': result['release_version'], - 'protocol': self.conn.protocol_version, + 'protocol': result['native_protocol_version'], 'cql': result['cql_version'], } self.connection_versions = vers @@ -634,37 +628,37 @@ class Shell(cmd.Cmd): # TODO remove after virtual tables are added to connection metadata if self.virtual_keyspaces is None: self.init_virtual_keyspaces_meta() - return map(str, self.conn.metadata.keyspaces.keys() + self.virtual_keyspaces.keys()) + return list(map(str, list(self.conn.metadata.keyspaces.keys()))) def get_columnfamily_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return map(str, self.get_keyspace_meta(ksname).tables.keys()) + return list(map(str, list(self.get_keyspace_meta(ksname).tables.keys()))) def get_materialized_view_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return map(str, self.get_keyspace_meta(ksname).views.keys()) + return list(map(str, list(self.get_keyspace_meta(ksname).views.keys()))) def get_index_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return map(str, self.get_keyspace_meta(ksname).indexes.keys()) + return list(map(str, list(self.get_keyspace_meta(ksname).indexes.keys()))) def get_column_names(self, ksname, cfname): if ksname is None: ksname = self.current_keyspace layout = self.get_table_meta(ksname, cfname) - return [unicode(col) for col in layout.columns] + return [str(col) for col in layout.columns] def get_usertype_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return self.get_keyspace_meta(ksname).user_types.keys() + return list(self.get_keyspace_meta(ksname).user_types.keys()) def get_usertype_layout(self, ksname, typename): if ksname is None: @@ -675,21 +669,21 @@ class Shell(cmd.Cmd): try: user_type = ks_meta.user_types[typename] except KeyError: - raise UserTypeNotFound("User type %r not found" % typename) + raise UserTypeNotFound("User type {!r} not found".format(typename)) - return zip(user_type.field_names, user_type.field_types) + return list(zip(user_type.field_names, user_type.field_types)) def get_userfunction_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return map(lambda f: f.name, self.get_keyspace_meta(ksname).functions.values()) + return [f.name for f in list(self.get_keyspace_meta(ksname).functions.values())] def get_useraggregate_names(self, ksname=None): if ksname is None: ksname = self.current_keyspace - return map(lambda f: f.name, self.get_keyspace_meta(ksname).aggregates.values()) + return [f.name for f in list(self.get_keyspace_meta(ksname).aggregates.values())] def get_cluster_name(self): return self.conn.metadata.cluster_name @@ -734,7 +728,7 @@ class Shell(cmd.Cmd): def fetch_virtual_tables(self, keyspace_name): tables = [] - result = self.session.execute("SELECT * FROM system_virtual_schema.tables WHERE keyspace_name = '{}';".format(keyspace_name)) + result = self.session.execute("SELECT * FROM system_virtual_schema.tables WHERE keyspace_name = '{}'".format(keyspace_name)) for row in result: name = row['table_name'] table = TableMetadata(keyspace_name, name) @@ -768,11 +762,11 @@ class Shell(cmd.Cmd): partition_key_columns.sort(key=lambda t: t[0]) clustering_columns.sort(key=lambda t: t[0]) - table.partition_key = map(lambda t: t[1], partition_key_columns) + table.partition_key = map(lambda t: t[1], partition_key_columns) table.clustering_key = map(lambda t: t[1], clustering_columns) def get_keyspaces(self): - return self.conn.metadata.keyspaces.values() + return list(self.conn.metadata.keyspaces.values()) def get_ring(self, ks): self.conn.metadata.token_map.rebuild_keyspace(ks, build_if_absent=True) @@ -786,7 +780,7 @@ class Shell(cmd.Cmd): if ksname == 'system_auth' and tablename in ['roles', 'role_permissions']: self.get_fake_auth_table_meta(ksname, tablename) else: - raise ColumnFamilyNotFound("Column family %r not found" % tablename) + raise ColumnFamilyNotFound("Column family {} not found".format(tablename)) else: return ksmeta.tables[tablename] @@ -807,7 +801,7 @@ class Shell(cmd.Cmd): table_meta.columns['resource'] = ColumnMetadata(table_meta, 'resource', cassandra.cqltypes.UTF8Type) table_meta.columns['permission'] = ColumnMetadata(table_meta, 'permission', cassandra.cqltypes.UTF8Type) else: - raise ColumnFamilyNotFound("Column family %r not found" % tablename) + raise ColumnFamilyNotFound("Column family {} not found".format(tablename)) def get_index_meta(self, ksname, idxname): if ksname is None: @@ -815,7 +809,7 @@ class Shell(cmd.Cmd): ksmeta = self.get_keyspace_meta(ksname) if idxname not in ksmeta.indexes: - raise IndexNotFound("Index %r not found" % idxname) + raise IndexNotFound("Index {} not found".format(idxname)) return ksmeta.indexes[idxname] @@ -825,7 +819,7 @@ class Shell(cmd.Cmd): ksmeta = self.get_keyspace_meta(ksname) if viewname not in ksmeta.views: - raise MaterializedViewNotFound("Materialized view %r not found" % viewname) + raise MaterializedViewNotFound("Materialized view '{}' not found".format(viewname)) return ksmeta.views[viewname] def get_object_meta(self, ks, name): @@ -833,7 +827,7 @@ class Shell(cmd.Cmd): if ks and ks in self.conn.metadata.keyspaces: return self.conn.metadata.keyspaces[ks] elif self.current_keyspace is None: - raise ObjectNotFound("%r not found in keyspaces" % (ks)) + raise ObjectNotFound("'{}' not found in keyspaces".format(ks)) else: name = ks ks = self.current_keyspace @@ -850,7 +844,7 @@ class Shell(cmd.Cmd): elif name in ksmeta.views: return ksmeta.views[name] - raise ObjectNotFound("%r not found in keyspace %r" % (name, ks)) + raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks)) def get_usertypes_meta(self): data = self.session.execute("select * from system.schema_usertypes") @@ -864,19 +858,20 @@ class Shell(cmd.Cmd): ksname = self.current_keyspace return [trigger.name - for table in self.get_keyspace_meta(ksname).tables.values() - for trigger in table.triggers.values()] + for table in list(self.get_keyspace_meta(ksname).tables.values()) + for trigger in list(table.triggers.values())] def reset_statement(self): self.reset_prompt() self.statement.truncate(0) + self.statement.seek(0) self.empty_lines = 0 def reset_prompt(self): if self.current_keyspace is None: self.set_prompt(self.default_prompt, True) else: - self.set_prompt(self.keyspace_prompt % self.current_keyspace, True) + self.set_prompt(self.keyspace_prompt.format(self.current_keyspace), True) def set_continue_prompt(self): if self.empty_lines >= 3: @@ -887,7 +882,7 @@ class Shell(cmd.Cmd): self.set_prompt(self.continue_prompt) else: spaces = ' ' * len(str(self.current_keyspace)) - self.set_prompt(self.keyspace_continue_prompt % spaces) + self.set_prompt(self.keyspace_continue_prompt.format(spaces)) self.empty_lines = self.empty_lines + 1 if not self.lastcmd else 0 @contextmanager @@ -898,7 +893,7 @@ class Shell(cmd.Cmd): import readline except ImportError: if is_win: - print "WARNING: pyreadline dependency missing. Install to enable tab completion." + print("WARNING: pyreadline dependency missing. Install to enable tab completion.") pass else: old_completer = readline.get_completer() @@ -909,20 +904,34 @@ class Shell(cmd.Cmd): readline.parse_and_bind("bind ^R em-inc-search-prev") else: readline.parse_and_bind(self.completekey + ": complete") + # start coverage collection if requested, unless in subshell + if self.coverage and not self.is_subshell: + # check for coveragerc file, write it if missing + if os.path.exists(HISTORY_DIR): + self.coveragerc_path = os.path.join(HISTORY_DIR, '.coveragerc') + covdata_path = os.path.join(HISTORY_DIR, '.coverage') + if not os.path.isfile(self.coveragerc_path): + with open(self.coveragerc_path, 'w') as f: + f.writelines(["[run]\n", + "concurrency = multiprocessing\n", + "data_file = {}\n".format(covdata_path), + "parallel = true\n"] + ) + # start coverage + import coverage + self.cov = coverage.Coverage(config_file=self.coveragerc_path) + self.cov.start() try: yield finally: if readline is not None: readline.set_completer(old_completer) + if self.coverage and not self.is_subshell: + self.stop_coverage() def get_input_line(self, prompt=''): if self.tty: - try: - self.lastcmd = raw_input(prompt).decode(self.encoding) - except UnicodeDecodeError: - self.lastcmd = '' - traceback.print_exc() - self.check_windows_encoding() + self.lastcmd = input(prompt) line = self.lastcmd + '\n' else: self.lastcmd = self.stdin.readline() @@ -930,6 +939,7 @@ class Shell(cmd.Cmd): if not len(line): raise EOFError self.lineno += 1 + line = ensure_text(line) return line def use_stdin_reader(self, until='', prompt=''): @@ -962,11 +972,11 @@ class Shell(cmd.Cmd): self.reset_statement() except EOFError: self.handle_eof() - except CQL_ERRORS, cqlerr: - self.printerr(cqlerr.message.decode(encoding='utf-8')) + except CQL_ERRORS as cqlerr: + self.printerr(cqlerr.message) except KeyboardInterrupt: self.reset_statement() - print + print('') def onecmd(self, statementtext): """ @@ -976,15 +986,15 @@ class Shell(cmd.Cmd): try: statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext) - except pylexotron.LexingError, e: + except pylexotron.LexingError as e: if self.show_line_nums: - self.printerr('Invalid syntax at char %d' % (e.charnum,)) + self.printerr('Invalid syntax at line {0}, char {1}' + .format(e.linenum, e.charnum)) else: - self.printerr('Invalid syntax at line %d, char %d' - % (e.linenum, e.charnum)) + self.printerr('Invalid syntax at char {0}'.format(e.charnum)) statementline = statementtext.split('\n')[e.linenum - 1] - self.printerr(' %s' % statementline) - self.printerr(' %s^' % (' ' * e.charnum)) + self.printerr(' {0}'.format(statementline)) + self.printerr(' {0}^'.format(' ' * e.charnum)) return True while statements and not statements[-1]: @@ -997,7 +1007,7 @@ class Shell(cmd.Cmd): for st in statements: try: self.handle_statement(st, statementtext) - except Exception, e: + except Exception as e: if self.debug: traceback.print_exc() else: @@ -1006,7 +1016,7 @@ class Shell(cmd.Cmd): def handle_eof(self): if self.tty: - print + print('') statement = self.statement.getvalue() if statement.strip(): if not self.onecmd(statement): @@ -1021,7 +1031,7 @@ class Shell(cmd.Cmd): new_hist = srcstr.replace("\n", " ").rstrip() if nl_count > 1 and self.last_hist != new_hist: - readline.add_history(new_hist.encode(self.encoding)) + readline.add_history(new_hist) self.last_hist = new_hist cmdword = tokens[0][1] @@ -1069,6 +1079,8 @@ class Shell(cmd.Cmd): self.tracing_enabled = tracing_was_enabled def perform_statement(self, statement): + statement = ensure_str(statement) + stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None) success, future = self.perform_simple_statement(stmt) @@ -1085,7 +1097,7 @@ class Shell(cmd.Cmd): self.writeresult(msg, color=RED) for trace_id in future.get_query_trace_ids(): self.show_session(trace_id, partial_session=True) - except Exception, err: + except Exception as err: self.printerr("Unable to fetch query trace: %s" % (str(err),)) return success @@ -1103,7 +1115,7 @@ class Shell(cmd.Cmd): try: return self.get_view_meta(ks, name) except MaterializedViewNotFound: - raise ObjectNotFound("%r not found in keyspace %r" % (name, ks)) + raise ObjectNotFound("'{}' not found in keyspace '{}'".format(name, ks)) def parse_for_update_meta(self, query_string): try: @@ -1122,8 +1134,9 @@ class Shell(cmd.Cmd): result = None try: result = future.result() - except CQL_ERRORS, err: - self.printerr(unicode(err.__class__.__name__) + u": " + err.message.decode(encoding='utf-8')) + except CQL_ERRORS as err: + err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err)) + self.printerr(str(err.__class__.__name__) + ": " + err_msg) except Exception: import traceback self.printerr(traceback.format_exc()) @@ -1166,7 +1179,7 @@ class Shell(cmd.Cmd): if result.has_more_pages: if self.shunted_query_out is None: # Only pause when not capturing. - raw_input("---MORE---") + input("---MORE---") result.fetch_next_page() else: break @@ -1186,7 +1199,7 @@ class Shell(cmd.Cmd): if not result.column_names and not table_meta: return - column_names = result.column_names or table_meta.columns.keys() + column_names = result.column_names or list(table_meta.columns.keys()) formatted_names = [self.myformat_colname(name, table_meta) for name in column_names] if not result.current_rows: # print header only @@ -1199,7 +1212,7 @@ class Shell(cmd.Cmd): ks_meta = self.conn.metadata.keyspaces.get(ks_name, None) cql_types = [CqlType(cql_typename(t), ks_meta) for t in result.column_types] - formatted_values = [map(self.myformat_value, [row[c] for c in column_names], cql_types) for row in result.current_rows] + formatted_values = [list(map(self.myformat_value, [row[c] for c in column_names], cql_types)) for row in result.current_rows] if self.expand_enabled: self.print_formatted_result_vertically(formatted_names, formatted_values) @@ -1290,7 +1303,7 @@ class Shell(cmd.Cmd): def set_prompt(self, prompt, prepend_user=False): if prepend_user and self.username: - self.prompt = "%s@%s" % (self.username, prompt) + self.prompt = "{0}@{1}".format(self.username, prompt) return self.prompt = prompt @@ -1348,161 +1361,161 @@ class Shell(cmd.Cmd): out.write("\n") def describe_keyspaces(self): - print + print('') cmd.Cmd.columnize(self, protect_names(self.get_keyspace_names())) - print + print('') def describe_keyspace(self, ksname): - print + print('') self.print_recreate_keyspace(self.get_keyspace_meta(ksname), sys.stdout) - print + print('') def describe_columnfamily(self, ksname, cfname): if ksname is None: ksname = self.current_keyspace if ksname is None: raise NoKeyspaceError("No keyspace specified and no current keyspace") - print + print('') self.print_recreate_columnfamily(ksname, cfname, sys.stdout) - print + print('') def describe_index(self, ksname, idxname): - print + print('') self.print_recreate_index(ksname, idxname, sys.stdout) - print + print('') def describe_materialized_view(self, ksname, viewname): if ksname is None: ksname = self.current_keyspace if ksname is None: raise NoKeyspaceError("No keyspace specified and no current keyspace") - print + print('') self.print_recreate_materialized_view(ksname, viewname, sys.stdout) - print + print('') def describe_object(self, ks, name): - print + print('') self.print_recreate_object(ks, name, sys.stdout) - print + print('') def describe_columnfamilies(self, ksname): - print + print('') if ksname is None: for k in self.get_keyspaces(): name = protect_name(k.name) - print 'Keyspace %s' % (name,) - print '---------%s' % ('-' * len(name)) + print('Keyspace %s' % (name,)) + print('---------%s' % ('-' * len(name))) cmd.Cmd.columnize(self, protect_names(self.get_columnfamily_names(k.name))) - print + print('') else: cmd.Cmd.columnize(self, protect_names(self.get_columnfamily_names(ksname))) - print + print('') def describe_functions(self, ksname): - print + print('') if ksname is None: for ksmeta in self.get_keyspaces(): name = protect_name(ksmeta.name) - print 'Keyspace %s' % (name,) - print '---------%s' % ('-' * len(name)) - self._columnize_unicode(ksmeta.functions.keys()) + print('Keyspace %s' % (name,)) + print('---------%s' % ('-' * len(name))) + self._columnize_unicode(list(ksmeta.functions.keys())) else: ksmeta = self.get_keyspace_meta(ksname) - self._columnize_unicode(ksmeta.functions.keys()) + self._columnize_unicode(list(ksmeta.functions.keys())) def describe_function(self, ksname, functionname): if ksname is None: ksname = self.current_keyspace if ksname is None: raise NoKeyspaceError("No keyspace specified and no current keyspace") - print + print('') ksmeta = self.get_keyspace_meta(ksname) - functions = filter(lambda f: f.name == functionname, ksmeta.functions.values()) + functions = [f for f in list(ksmeta.functions.values()) if f.name == functionname] if len(functions) == 0: - raise FunctionNotFound("User defined function %r not found" % functionname) - print "\n\n".join(func.export_as_string() for func in functions) - print + raise FunctionNotFound("User defined function {} not found".format(functionname)) + print("\n\n".join(func.export_as_string() for func in functions)) + print('') def describe_aggregates(self, ksname): - print + print('') if ksname is None: for ksmeta in self.get_keyspaces(): name = protect_name(ksmeta.name) - print 'Keyspace %s' % (name,) - print '---------%s' % ('-' * len(name)) - self._columnize_unicode(ksmeta.aggregates.keys()) + print('Keyspace %s' % (name,)) + print('---------%s' % ('-' * len(name))) + self._columnize_unicode(list(ksmeta.aggregates.keys())) else: ksmeta = self.get_keyspace_meta(ksname) - self._columnize_unicode(ksmeta.aggregates.keys()) + self._columnize_unicode(list(ksmeta.aggregates.keys())) def describe_aggregate(self, ksname, aggregatename): if ksname is None: ksname = self.current_keyspace if ksname is None: raise NoKeyspaceError("No keyspace specified and no current keyspace") - print + print('') ksmeta = self.get_keyspace_meta(ksname) - aggregates = filter(lambda f: f.name == aggregatename, ksmeta.aggregates.values()) + aggregates = [f for f in list(ksmeta.aggregates.values()) if f.name == aggregatename] if len(aggregates) == 0: - raise FunctionNotFound("User defined aggregate %r not found" % aggregatename) - print "\n\n".join(aggr.export_as_string() for aggr in aggregates) - print + raise FunctionNotFound("User defined aggregate {} not found".format(aggregatename)) + print("\n\n".join(aggr.export_as_string() for aggr in aggregates)) + print('') def describe_usertypes(self, ksname): - print + print('') if ksname is None: for ksmeta in self.get_keyspaces(): name = protect_name(ksmeta.name) - print 'Keyspace %s' % (name,) - print '---------%s' % ('-' * len(name)) - self._columnize_unicode(ksmeta.user_types.keys(), quote=True) + print('Keyspace %s' % (name,)) + print('---------%s' % ('-' * len(name))) + self._columnize_unicode(list(ksmeta.user_types.keys()), quote=True) else: ksmeta = self.get_keyspace_meta(ksname) - self._columnize_unicode(ksmeta.user_types.keys(), quote=True) + self._columnize_unicode(list(ksmeta.user_types.keys()), quote=True) def describe_usertype(self, ksname, typename): if ksname is None: ksname = self.current_keyspace if ksname is None: raise NoKeyspaceError("No keyspace specified and no current keyspace") - print + print('') ksmeta = self.get_keyspace_meta(ksname) try: usertype = ksmeta.user_types[typename] except KeyError: - raise UserTypeNotFound("User type %r not found" % typename) - print usertype.export_as_string() + raise UserTypeNotFound("User type {} not found".format(typename)) + print(usertype.export_as_string()) def _columnize_unicode(self, name_list, quote=False): """ Used when columnizing identifiers that may contain unicode """ - names = [n.encode('utf-8') for n in name_list] + names = [n for n in name_list] if quote: names = protect_names(names) cmd.Cmd.columnize(self, names) - print + print('') def describe_cluster(self): - print '\nCluster: %s' % self.get_cluster_name() + print('\nCluster: %s' % self.get_cluster_name()) p = trim_if_present(self.get_partitioner(), 'org.apache.cassandra.dht.') - print 'Partitioner: %s\n' % p + print('Partitioner: %s\n' % p) # TODO: snitch? # snitch = trim_if_present(self.get_snitch(), 'org.apache.cassandra.locator.') # print 'Snitch: %s\n' % snitch if self.current_keyspace is not None and self.current_keyspace != 'system': - print "Range ownership:" + print("Range ownership:") ring = self.get_ring(self.current_keyspace) - for entry in ring.items(): - print ' %39s [%s]' % (str(entry[0].value), ', '.join([host.address for host in entry[1]])) - print + for entry in list(ring.items()): + print(' %39s [%s]' % (str(entry[0].value), ', '.join([host.address for host in entry[1]]))) + print('') def describe_schema(self, include_system=False): - print + print('') for k in self.get_keyspaces(): if include_system or k.name not in cql3handling.SYSTEM_KEYSPACES: self.print_recreate_keyspace(k, sys.stdout) - print + print('') def do_describe(self, parsed): """ @@ -1745,7 +1758,7 @@ class Shell(cmd.Cmd): table = self.cql_unprotect_name(parsed.get_binding('cfname')) columns = parsed.get_binding('colnames', None) if columns is not None: - columns = map(self.cql_unprotect_name, columns) + columns = list(map(self.cql_unprotect_name, columns)) else: # default to all known columns columns = self.get_column_names(ks, table) @@ -1754,9 +1767,9 @@ class Shell(cmd.Cmd): if fname is not None: fname = self.cql_unprotect_value(fname) - copyoptnames = map(str.lower, parsed.get_binding('optnames', ())) - copyoptvals = map(self.cql_unprotect_value, parsed.get_binding('optvals', ())) - opts = dict(zip(copyoptnames, copyoptvals)) + copyoptnames = list(map(six.text_type.lower, parsed.get_binding('optnames', ()))) + copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ()))) + opts = dict(list(zip(copyoptnames, copyoptvals))) direction = parsed.get_binding('dir').upper() if direction == 'FROM': @@ -1827,7 +1840,7 @@ class Shell(cmd.Cmd): encoding, bom_size = get_file_encoding_bomsize(fname) f = codecs.open(fname, 'r', encoding) f.seek(bom_size) - except IOError, e: + except IOError as e: self.printerr('Could not open %r: %s' % (fname, e)) return username = self.auth_provider.username if self.auth_provider else None @@ -1846,7 +1859,12 @@ class Shell(cmd.Cmd): max_trace_wait=self.max_trace_wait, ssl=self.ssl, request_timeout=self.session.default_timeout, connect_timeout=self.conn.connect_timeout, - allow_server_port_discovery=self.allow_server_port_discovery) + allow_server_port_discovery=self.allow_server_port_discovery, + is_subshell=True) + # duplicate coverage related settings in subshell + if self.coverage: + subshell.coverage = True + subshell.coveragerc_path = self.coveragerc_path subshell.cmdloop() f.close() @@ -1880,9 +1898,9 @@ class Shell(cmd.Cmd): fname = parsed.get_binding('fname') if fname is None: if self.shunted_query_out is not None: - print "Currently capturing query output to %r." % (self.query_out.name,) + print("Currently capturing query output to %r." % (self.query_out.name,)) else: - print "Currently not capturing query output." + print("Currently not capturing query output.") return if fname.upper() == 'OFF': @@ -1904,14 +1922,14 @@ class Shell(cmd.Cmd): fname = os.path.expanduser(self.cql_unprotect_value(fname)) try: f = open(fname, 'a') - except IOError, e: + except IOError as e: self.printerr('Could not open %r for append: %s' % (fname, e)) return self.shunted_query_out = self.query_out self.shunted_color = self.color self.query_out = f self.color = False - print 'Now capturing query output to %r.' % (fname,) + print('Now capturing query output to %r.' % (fname,)) def do_tracing(self, parsed): """ @@ -1975,11 +1993,11 @@ class Shell(cmd.Cmd): """ level = parsed.get_binding('level') if level is None: - print 'Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level]) + print('Current consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.consistency_level])) return self.consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()] - print 'Consistency level set to %s.' % (level.upper(),) + print('Consistency level set to %s.' % (level.upper(),)) def do_serial(self, parsed): """ @@ -2001,11 +2019,11 @@ class Shell(cmd.Cmd): """ level = parsed.get_binding('level') if level is None: - print 'Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level]) + print('Current serial consistency level is %s.' % (cassandra.ConsistencyLevel.value_to_name[self.serial_consistency_level])) return self.serial_consistency_level = cassandra.ConsistencyLevel.name_to_value[level.upper()] - print 'Serial consistency level set to %s.' % (level.upper(),) + print('Serial consistency level set to %s.' % (level.upper(),)) def do_login(self, parsed): """ @@ -2158,7 +2176,7 @@ class Shell(cmd.Cmd): if self.use_paging and requested_page_size is not None: self.page_size = requested_page_size if self.use_paging: - print("Page size: {}".format(self.page_size)) + print(("Page size: {}".format(self.page_size))) else: self.page_size = self.default_page_size @@ -2172,13 +2190,11 @@ class Shell(cmd.Cmd): out = self.query_out # convert Exceptions, etc to text - if not isinstance(text, (unicode, str)): - text = unicode(text) - - if isinstance(text, unicode): - text = text.encode(self.encoding) + if not isinstance(text, six.text_type): + text = "{}".format(text) to_write = self.applycolor(text, color) + ('\n' if newline else '') + to_write = ensure_str(to_write) out.write(to_write) def flush_output(self): @@ -2192,6 +2208,12 @@ class Shell(cmd.Cmd): text = '%s:%d:%s' % (self.stdin.name, self.lineno, text) self.writeresult(text, color, newline=newline, out=sys.stderr) + def stop_coverage(self): + if self.coverage and self.cov is not None: + self.cov.stop() + self.cov.save() + self.cov = None + class SwitchCommand(object): command = None @@ -2205,11 +2227,11 @@ class SwitchCommand(object): switch = parsed.get_binding('switch') if switch is None: if state: - print "%s is currently enabled. Use %s OFF to disable" \ - % (self.description, self.command) + print("%s is currently enabled. Use %s OFF to disable" + % (self.description, self.command)) else: - print "%s is currently disabled. Use %s ON to enable." \ - % (self.description, self.command) + print("%s is currently disabled. Use %s ON to enable." + % (self.description, self.command)) return state if switch.upper() == 'ON': @@ -2217,14 +2239,14 @@ class SwitchCommand(object): printerr('%s is already enabled. Use %s OFF to disable.' % (self.description, self.command)) return state - print 'Now %s is enabled' % (self.description,) + print('Now %s is enabled' % (self.description,)) return True if switch.upper() == 'OFF': if not state: printerr('%s is not enabled.' % (self.description,)) return state - print 'Disabled %s.' % (self.description,) + print('Disabled %s.' % (self.description,)) return False @@ -2256,7 +2278,7 @@ class SwitchCommandWithValue(SwitchCommand): def option_with_default(cparser_getter, section, option, default=None): try: return cparser_getter(section, option) - except ConfigParser.Error: + except configparser.Error: return default @@ -2267,7 +2289,7 @@ def raw_option_with_default(configs, section, option, default=None): """ try: return configs.get(section, option, raw=True) - except ConfigParser.Error: + except configparser.Error: return default @@ -2290,10 +2312,10 @@ def should_use_color(): def read_options(cmdlineargs, environment): - configs = ConfigParser.SafeConfigParser() + configs = configparser.SafeConfigParser() configs.read(CONFIG_FILE) - rawconfigs = ConfigParser.RawConfigParser() + rawconfigs = configparser.RawConfigParser() rawconfigs.read(CONFIG_FILE) optvalues = optparse.Values() @@ -2320,6 +2342,11 @@ def read_options(cmdlineargs, environment): optvalues.timezone = option_with_default(configs.get, 'ui', 'timezone', None) optvalues.debug = False + + optvalues.coverage = False + if 'CQLSH_COVERAGE' in environment.keys(): + optvalues.coverage = True + optvalues.file = None optvalues.ssl = option_with_default(configs.getboolean, 'connection', 'ssl', DEFAULT_SSL) optvalues.encoding = option_with_default(configs.get, 'ui', 'encoding', UTF8) @@ -2431,7 +2458,7 @@ def main(options, hostname, port): encoding, bom_size = get_file_encoding_bomsize(options.file) stdin = codecs.open(options.file, 'r', encoding) stdin.seek(bom_size) - except IOError, e: + except IOError as e: sys.exit("Can't open %r: %s" % (options.file, e)) if options.debug: @@ -2467,8 +2494,10 @@ def main(options, hostname, port): # we silently ignore and fallback to UTC unless a custom timestamp format (which likely # does contain a TZ part) was specified if options.time_format != DEFAULT_TIMESTAMP_FORMAT: - sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, but local timezone could not be detected.\n" + - "Either install Python 'tzlocal' module for auto-detection or specify client timezone in your cqlshrc.\n\n") + sys.stderr.write("Warning: custom timestamp format specified in cqlshrc, " + + "but local timezone could not be detected.\n" + + "Either install Python 'tzlocal' module for auto-detection " + + "or specify client timezone in your cqlshrc.\n\n") try: shell = Shell(hostname, @@ -2498,15 +2527,25 @@ def main(options, hostname, port): allow_server_port_discovery=options.allow_server_port_discovery) except KeyboardInterrupt: sys.exit('Connection aborted.') - except CQL_ERRORS, e: + except CQL_ERRORS as e: sys.exit('Connection error: %s' % (e,)) - except VersionNotSupported, e: + except VersionNotSupported as e: sys.exit('Unsupported CQL version: %s' % (e,)) if options.debug: shell.debug = True + if options.coverage: + shell.coverage = True + import signal + + def handle_sighup(): + shell.stop_coverage() + shell.do_exit() + + signal.signal(signal.SIGHUP, handle_sighup) shell.cmdloop() save_history() + batch_mode = options.file or options.execute if batch_mode and shell.statement_error: sys.exit(2) diff --git a/lib/six-1.12.0-py2.py3-none-any.zip b/lib/six-1.12.0-py2.py3-none-any.zip new file mode 100644 index 0000000000..78ba90328c Binary files /dev/null and b/lib/six-1.12.0-py2.py3-none-any.zip differ diff --git a/lib/six-1.7.3-py2.py3-none-any.zip b/lib/six-1.7.3-py2.py3-none-any.zip deleted file mode 100644 index e077898589..0000000000 Binary files a/lib/six-1.7.3-py2.py3-none-any.zip and /dev/null differ diff --git a/pylib/Dockerfile.ubuntu.py2 b/pylib/Dockerfile.ubuntu.py2 new file mode 100644 index 0000000000..93016f016a --- /dev/null +++ b/pylib/Dockerfile.ubuntu.py2 @@ -0,0 +1,2 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y python-minimal diff --git a/pylib/Dockerfile.ubuntu.py3 b/pylib/Dockerfile.ubuntu.py3 new file mode 100644 index 0000000000..f40281737f --- /dev/null +++ b/pylib/Dockerfile.ubuntu.py3 @@ -0,0 +1,2 @@ +FROM ubuntu:bionic +RUN apt-get update && apt-get install -y python3-minimal diff --git a/pylib/README.asc b/pylib/README.asc new file mode 100644 index 0000000000..42478d98bd --- /dev/null +++ b/pylib/README.asc @@ -0,0 +1,31 @@ +== Overview + +This directory contains code primarily for cqlsh. cqlsh uses cqlshlib in this directory. +Currently, cqlshlib supports Python 2 as well as Python 3. Support for Python 3 is relatively +new. + +== Requirements +. Python 3 and 2.7 (for cqlsh) +. virtualenv +. Docker (optional) + +== Running tests + +In order to run tests for cqlshlib, run cassandra-cqlsh-tests.sh in this directory. It will +automatically setup a virtualenv with the appropriate version of Python and run tests inside it. + +There are Dockerfiles that can be used to test whether cqlsh works with a default, barebones +Python installation. Assuming Cassandra's source is checked out at `$CASSANDRA_DIR`. To test, first +build the Docker image containing the barebones Python installation - + + $ docker build . --file Dockerfile.ubuntu.py3 -t ubuntu-lts-py3 + +Next, run cqlsh inside the newly built image - + + $ docker run -v $CASSANDRA_DIR:/code -it ubuntu-lts-py3:latest /code/cassandra/bin/cqlsh host.docker.internal + +This will try to spawn a cqlsh instance inside the Docker container running Ubuntu LTS (18.04) +with minimal Python installation. It will try to connect to the Cassandra instance running on the +Docker host at port 9042. If you have Cassandra running elsewhere, replace host.docker.internal +with the IP / hostname as usual. Please ensure that the IP / host is accessible from _within_ the +Docker container. \ No newline at end of file diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index 1fb3aa0c85..d3057596cd 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -7,12 +7,22 @@ ################################ WORKSPACE=$1 +PYTHON_VERSION=$2 if [ "${WORKSPACE}" = "" ]; then echo "Specify Cassandra source directory" exit fi +if [ "${PYTHON_VERSION}" = "" ]; then + PYTHON_VERSION=python3 +fi + +if [ "${PYTHON_VERSION}" != "python3" -a "${PYTHON_VERSION}" != "python2" ]; then + echo "Specify Python version python3 or python2" + exit +fi + export PYTHONIOENCODING="utf-8" export PYTHONUNBUFFERED=true export CASS_DRIVER_NO_EXTENSIONS=true @@ -23,6 +33,10 @@ export CCM_CONFIG_DIR=${WORKSPACE}/.ccm export NUM_TOKENS="32" export CASSANDRA_DIR=${WORKSPACE} +if [ -z "$CASSANDRA_USE_JDK11" ]; then + export CASSANDRA_USE_JDK11=true +fi + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. for x in $(seq 1 3); do ant -buildfile ${CASSANDRA_DIR}/build.xml realclean jar @@ -39,8 +53,9 @@ fi # Set up venv with dtest dependencies set -e # enable immediate exit if venv setup fails -virtualenv --python=python2 --no-site-packages venv +virtualenv --python=$PYTHON_VERSION --no-site-packages venv source venv/bin/activate + pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt pip freeze @@ -59,10 +74,11 @@ fi ccm remove test || true # in case an old ccm cluster is left behind ccm create test -n 1 --install-dir=${CASSANDRA_DIR} ccm updateconf "enable_user_defined_functions: true" +ccm updateconf "enable_scripted_user_defined_functions: true" version_from_build=$(ccm node1 versionfrombuild) export pre_or_post_cdc=$(python -c """from distutils.version import LooseVersion -print \"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\" +print (\"postcdc\" if LooseVersion(\"${version_from_build}\") >= \"3.8\" else \"precdc\") """) case "${pre_or_post_cdc}" in postcdc) diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py index 7f97b49ecf..3bed2ad8df 100644 --- a/pylib/cqlshlib/copyutil.py +++ b/pylib/cqlshlib/copyutil.py @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from __future__ import unicode_literals import csv import datetime import json @@ -26,6 +26,8 @@ import os import platform import random import re +import signal +import six import struct import sys import threading @@ -36,25 +38,30 @@ from bisect import bisect_right from calendar import timegm from collections import defaultdict, namedtuple from decimal import Decimal -from Queue import Queue from random import randint -from StringIO import StringIO +from io import BytesIO, StringIO from select import select from uuid import UUID -from util import profile_on, profile_off +from .util import profile_on, profile_off + +from six import ensure_str, ensure_text +from six.moves import configparser +from six.moves import range +from six.moves.queue import Queue from cassandra import OperationTimedOut from cassandra.cluster import Cluster, DefaultConnection -from cassandra.cqltypes import ReversedType, UserType +from cassandra.cqltypes import ReversedType, UserType, BytesType from cassandra.metadata import protect_name, protect_names, protect_value from cassandra.policies import RetryPolicy, WhiteListRoundRobinPolicy, DCAwareRoundRobinPolicy, FallthroughRetryPolicy from cassandra.query import BatchStatement, BatchType, SimpleStatement, tuple_factory from cassandra.util import Date, Time +from cqlshlib.util import profile_on, profile_off -from cql3handling import CqlRuleSet -from displaying import NO_COLOR_MAP -from formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter -from sslhandling import ssl_settings +from cqlshlib.cql3handling import CqlRuleSet +from cqlshlib.displaying import NO_COLOR_MAP +from cqlshlib.formatting import format_value_default, CqlType, DateTimeFormat, EMPTY, get_formatter +from cqlshlib.sslhandling import ssl_settings PROFILE_ON = False STRACE_ON = False @@ -79,7 +86,7 @@ def printdebugmsg(msg): def printmsg(msg, eol='\n', encoding='utf8'): - sys.stdout.write(msg.encode(encoding)) + sys.stdout.write(msg) sys.stdout.write(eol) sys.stdout.flush() @@ -138,7 +145,7 @@ class SendingChannel(object): try: msg = self.pending_messages.get() self.pipe.send(msg) - except Exception, e: + except Exception as e: printmsg('%s: %s' % (e.__class__.__name__, e.message)) feeding_thread = threading.Thread(target=feed) @@ -160,7 +167,7 @@ class SendingChannels(object): A group of one way channels for sending messages. """ def __init__(self, num_channels): - self.pipes = [OneWayPipe() for _ in xrange(num_channels)] + self.pipes = [OneWayPipe() for _ in range(num_channels)] self.channels = [SendingChannel(p) for p in self.pipes] self.num_channels = num_channels @@ -177,7 +184,7 @@ class ReceivingChannels(object): A group of one way channels for receiving messages. """ def __init__(self, num_channels): - self.pipes = [OneWayPipe() for _ in xrange(num_channels)] + self.pipes = [OneWayPipe() for _ in range(num_channels)] self.channels = [ReceivingChannel(p) for p in self.pipes] self._readers = [p.reader for p in self.pipes] self._rlocks = [p.rlock for p in self.pipes] @@ -276,7 +283,7 @@ class CopyTask(object): if not os.path.isfile(config_file): return opts - configs = ConfigParser.RawConfigParser() + configs = configparser.RawConfigParser() configs.readfp(open(config_file)) ret = dict() @@ -305,8 +312,8 @@ class CopyTask(object): """ Convert all option values to valid string literals unless they are path names """ - return dict([(k, v.decode('string_escape') if k not in ['errfile', 'ratefile'] else v) - for k, v, in opts.iteritems()]) + return dict([(k, v if k not in ['errfile', 'ratefile'] else v) + for k, v, in opts.items()]) def parse_options(self, opts, direction): """ @@ -319,9 +326,9 @@ class CopyTask(object): opts = self.clean_options(self.maybe_read_config_file(opts, direction)) dialect_options = dict() - dialect_options['quotechar'] = opts.pop('quote', '"') - dialect_options['escapechar'] = opts.pop('escape', '\\') - dialect_options['delimiter'] = opts.pop('delimiter', ',') + dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"')) + dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\')) + dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ',')) if dialect_options['quotechar'] == dialect_options['escapechar']: dialect_options['doublequote'] = True del dialect_options['escapechar'] @@ -329,7 +336,7 @@ class CopyTask(object): dialect_options['doublequote'] = False copy_options = dict() - copy_options['nullval'] = opts.pop('null', '') + copy_options['nullval'] = ensure_str(opts.pop('null', '')) copy_options['header'] = bool(opts.pop('header', '').lower() == 'true') copy_options['encoding'] = opts.pop('encoding', 'utf8') copy_options['maxrequests'] = int(opts.pop('maxrequests', 6)) @@ -351,7 +358,7 @@ class CopyTask(object): copy_options['consistencylevel'] = shell.consistency_level copy_options['decimalsep'] = opts.pop('decimalsep', '.') copy_options['thousandssep'] = opts.pop('thousandssep', '') - copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')] + copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')] copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16))) copy_options['begintoken'] = opts.pop('begintoken', '') copy_options['endtoken'] = opts.pop('endtoken', '') @@ -494,7 +501,9 @@ class CopyTask(object): cql_version=shell.conn.cql_version, config_file=self.config_file, protocol_version=self.protocol_version, - debug=shell.debug + debug=shell.debug, + coverage=shell.coverage, + coveragerc_path=shell.coveragerc_path ) def validate_columns(self): @@ -534,7 +543,7 @@ class ExportWriter(object): self.columns = columns self.options = options self.header = options.copy['header'] - self.max_output_size = long(options.copy['maxoutputsize']) + self.max_output_size = int(options.copy['maxoutputsize']) self.current_dest = None self.num_files = 0 @@ -577,10 +586,10 @@ class ExportWriter(object): return CsvDest(output=sys.stdout, close=False) else: try: - ret = CsvDest(output=open(source_name, 'wb'), close=True) + ret = CsvDest(output=open(source_name, 'w'), close=True) self.num_files += 1 return ret - except IOError, e: + except IOError as e: self.shell.printerr("Can't open %r for writing: %s" % (source_name, e)) return None @@ -604,7 +613,7 @@ class ExportWriter(object): if (self.num_written + num) > self.max_output_size: num_remaining = self.max_output_size - self.num_written last_switch = 0 - for i, row in enumerate(filter(None, data.split(os.linesep))): + for i, row in enumerate([_f for _f in data.split(os.linesep) if _f]): if i == num_remaining: self._next_dest() last_switch = i @@ -625,8 +634,8 @@ class ExportTask(CopyTask): CopyTask.__init__(self, shell, ks, table, columns, fname, opts, protocol_version, config_file, 'to') options = self.options - self.begin_token = long(options.copy['begintoken']) if options.copy['begintoken'] else None - self.end_token = long(options.copy['endtoken']) if options.copy['endtoken'] else None + self.begin_token = int(options.copy['begintoken']) if options.copy['begintoken'] else None + self.end_token = int(options.copy['endtoken']) if options.copy['endtoken'] else None self.writer = ExportWriter(fname, shell, columns, options) def run(self): @@ -637,7 +646,7 @@ class ExportTask(CopyTask): shell = self.shell if self.options.unrecognized: - shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(self.options.unrecognized.keys())) + shell.printerr('Unrecognized COPY TO options: %s' % ', '.join(list(self.options.unrecognized.keys()))) return if not self.validate_columns(): @@ -650,11 +659,11 @@ class ExportTask(CopyTask): if not self.writer.open(): return 0 - columns = u"[" + u", ".join(self.columns) + u"]" - self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding) + columns = "[" + ", ".join(self.columns) + "]" + self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding) params = self.make_params() - for i in xrange(self.num_processes): + for i in range(self.num_processes): self.processes.append(ExportProcess(self.update_params(params, i))) self.start_processes() @@ -695,15 +704,15 @@ class ExportTask(CopyTask): """ ret = (prev, curr) if begin_token: - if ret[1] < begin_token: + if curr < begin_token: return None - elif ret[0] < begin_token: - ret = (begin_token, ret[1]) + elif (prev is None) or (prev < begin_token): + ret = (begin_token, curr) if end_token: - if ret[0] > end_token: + if (ret[0] is not None) and (ret[0] > end_token): return None - elif ret[1] > end_token: + elif (curr is not None) and (curr > end_token): ret = (ret[0], end_token) return ret @@ -730,7 +739,7 @@ class ExportTask(CopyTask): ranges[(begin_token, end_token)] = make_range_data() return ranges - ring = shell.get_ring(self.ks).items() + ring = list(shell.get_ring(self.ks).items()) ring.sort() if not ring: @@ -760,6 +769,9 @@ class ExportTask(CopyTask): # For the last ring interval we query the same replicas that hold the first token in the ring if previous is not None and (not end_token or previous < end_token): ranges[(previous, end_token)] = first_range_data + elif previous is None and (not end_token or previous < end_token): + previous = begin_token if begin_token else min_token + ranges[(previous, end_token)] = first_range_data if not ranges: shell.printerr('Found no ranges to query, check begin and end tokens: %s - %s' % (begin_token, end_token)) @@ -809,7 +821,7 @@ class ExportTask(CopyTask): total_requests = len(ranges) max_attempts = self.options.copy['maxattempts'] - self.send_work(ranges, ranges.keys()) + self.send_work(ranges, list(ranges.keys())) num_processes = len(processes) succeeded = 0 @@ -881,8 +893,8 @@ class FilesReader(object): """ def make_source(fname): try: - return open(fname, 'rb') - except IOError, e: + return open(fname, 'r') + except IOError as e: raise IOError("Can't open %r for reading: %s" % (fname, e)) for path in paths.split(','): @@ -913,14 +925,14 @@ class FilesReader(object): self.close_current_source() while self.current_source is None: try: - self.current_source = self.sources.next() + self.current_source = next(self.sources) if self.current_source: self.num_sources += 1 except StopIteration: return False if self.header: - self.current_source.next() + next(self.current_source) return True @@ -939,9 +951,9 @@ class FilesReader(object): return [] rows = [] - for i in xrange(min(max_rows, self.chunk_size)): + for i in range(min(max_rows, self.chunk_size)): try: - row = self.current_source.next() + row = next(self.current_source) self.num_read += 1 if 0 <= self.max_rows < self.num_read: @@ -955,7 +967,7 @@ class FilesReader(object): self.next_source() break - return filter(None, rows) + return [_f for _f in rows if _f] class PipeReader(object): @@ -977,7 +989,7 @@ class PipeReader(object): def read_rows(self, max_rows): rows = [] - for i in xrange(min(max_rows, self.chunk_size)): + for i in range(min(max_rows, self.chunk_size)): row = self.inpipe.recv() if row is None: self.exhausted = True @@ -1143,14 +1155,14 @@ class ImportTask(CopyTask): shell = self.shell if self.options.unrecognized: - shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(self.options.unrecognized.keys())) + shell.printerr('Unrecognized COPY FROM options: %s' % ', '.join(list(self.options.unrecognized.keys()))) return if not self.validate_columns(): return 0 - columns = u"[" + u", ".join(self.valid_columns) + u"]" - self.printmsg(u"\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding) + columns = "[" + ", ".join(self.valid_columns) + "]" + self.printmsg("\nStarting copy of %s.%s with columns %s." % (self.ks, self.table, columns), encoding=self.encoding) try: params = self.make_params() @@ -1172,8 +1184,8 @@ class ImportTask(CopyTask): if pr: profile_off(pr, file_name='parent_profile_%d.txt' % (os.getpid(),)) - except Exception, exc: - shell.printerr(unicode(exc)) + except Exception as exc: + shell.printerr(str(exc)) if shell.debug: traceback.print_exc() return 0 @@ -1195,7 +1207,7 @@ class ImportTask(CopyTask): self.outmsg.channels[-1].send(None) if shell.tty: - print + print() def import_records(self): """ @@ -1280,7 +1292,7 @@ class FeedingProcess(mp.Process): A process that reads from import sources and sends chunks to worker processes. """ def __init__(self, inpipe, outpipe, worker_pipes, fname, options, parent_cluster): - mp.Process.__init__(self, target=self.run) + super(FeedingProcess, self).__init__(target=self.run) self.inpipe = inpipe self.outpipe = outpipe self.worker_pipes = worker_pipes @@ -1329,7 +1341,7 @@ class FeedingProcess(mp.Process): reader = self.reader try: reader.start() - except IOError, exc: + except IOError as exc: self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message)) channels = self.worker_channels @@ -1338,7 +1350,7 @@ class FeedingProcess(mp.Process): failed_attempts = 0 while not reader.exhausted: - channels_eligible = filter(lambda c: c.num_pending() < max_pending_chunks, channels) + channels_eligible = [c for c in channels if c.num_pending() < max_pending_chunks] if not channels_eligible: failed_attempts += 1 delay = randint(1, pow(2, failed_attempts)) @@ -1358,7 +1370,7 @@ class FeedingProcess(mp.Process): rows = reader.read_rows(max_rows) if rows: sent += self.send_chunk(ch, rows) - except Exception, exc: + except Exception as exc: self.outmsg.send(ImportTaskError(exc.__class__.__name__, exc.message)) if reader.exhausted: @@ -1392,7 +1404,7 @@ class ChildProcess(mp.Process): """ def __init__(self, params, target): - mp.Process.__init__(self, target=target) + super(ChildProcess, self).__init__(target=target) self.inpipe = params['inpipe'] self.outpipe = params['outpipe'] self.inmsg = None # must be initialized after fork on Windows @@ -1425,6 +1437,12 @@ class ChildProcess(mp.Process): self.test_failures = json.loads(os.environ.get('CQLSH_COPY_TEST_FAILURES', '')) else: self.test_failures = None + # attributes for coverage + self.coverage = params['coverage'] + self.coveragerc_path = params['coveragerc_path'] + self.coverage_collection = None + self.sigterm_handler = None + self.sighup_handler = None def on_fork(self): """ @@ -1442,6 +1460,31 @@ class ChildProcess(mp.Process): self.inmsg.close() self.outmsg.close() + def start_coverage(self): + import coverage + self.coverage_collection = coverage.Coverage(config_file=self.coveragerc_path) + self.coverage_collection.start() + + # save current handlers for SIGTERM and SIGHUP + self.sigterm_handler = signal.getsignal(signal.SIGTERM) + self.sighup_handler = signal.getsignal(signal.SIGTERM) + + def handle_sigterm(): + self.stop_coverage() + self.close() + self.terminate() + + # set custom handler for SIGHUP and SIGTERM + # needed to make sure coverage data is saved + signal.signal(signal.SIGTERM, handle_sigterm) + signal.signal(signal.SIGHUP, handle_sigterm) + + def stop_coverage(self): + self.coverage_collection.stop() + self.coverage_collection.save() + signal.signal(signal.SIGTERM, self.sigterm_handler) + signal.signal(signal.SIGHUP, self.sighup_handler) + class ExpBackoffRetryPolicy(RetryPolicy): """ @@ -1547,9 +1590,13 @@ class ExportProcess(ChildProcess): self.options = options def run(self): + if self.coverage: + self.start_coverage() try: self.inner_run() finally: + if self.coverage: + self.stop_coverage() self.close() def inner_run(self): @@ -1582,7 +1629,7 @@ class ExportProcess(ChildProcess): if print_traceback and sys.exc_info()[1] == err: traceback.print_exc() else: - msg = unicode(err) + msg = str(err) return msg def report_error(self, err, token_range): @@ -1606,7 +1653,7 @@ class ExportProcess(ChildProcess): self.attach_callbacks(token_range, future, session) def num_requests(self): - return sum(session.num_requests() for session in self.hosts_to_sessions.values()) + return sum(session.num_requests() for session in list(self.hosts_to_sessions.values())) def get_session(self, hosts, token_range): """ @@ -1626,7 +1673,7 @@ class ExportProcess(ChildProcess): for host in hosts: try: ret = self.connect(host) - except Exception, e: + except Exception as e: errors.append(self.get_error_message(e)) if ret: @@ -1639,7 +1686,7 @@ class ExportProcess(ChildProcess): return None def connect(self, host): - if host in self.hosts_to_sessions.keys(): + if host in list(self.hosts_to_sessions.keys()): session = self.hosts_to_sessions[host] session.add_request() return session @@ -1687,17 +1734,18 @@ class ExportProcess(ChildProcess): return # no rows in this range try: - output = StringIO() + output = StringIO() if six.PY3 else BytesIO() writer = csv.writer(output, **self.options.dialect) for row in rows: - writer.writerow(map(self.format_value, row, cql_types)) + print("cqlshlib.copyutil.ExportProcess.write_rows_to_csv(): writing row") + writer.writerow(list(map(self.format_value, row, cql_types))) data = (output.getvalue(), len(rows)) self.send((token_range, data)) output.close() - except Exception, e: + except Exception as e: self.report_error(e, token_range) def format_value(self, val, cqltype): @@ -1712,15 +1760,16 @@ class ExportProcess(ChildProcess): if not hasattr(cqltype, 'precision'): cqltype.precision = self.double_precision if cqltype.type_name == 'double' else self.float_precision - return formatter(val, cqltype=cqltype, - encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format, - float_precision=cqltype.precision, nullval=self.nullval, quote=False, - decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep, - boolean_styles=self.boolean_styles) + formatted = formatter(val, cqltype=cqltype, + encoding=self.encoding, colormap=NO_COLOR_MAP, date_time_format=self.date_time_format, + float_precision=cqltype.precision, nullval=self.nullval, quote=False, + decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep, + boolean_styles=self.boolean_styles) + return formatted if six.PY3 else formatted.encode('utf8') def close(self): ChildProcess.close(self) - for session in self.hosts_to_sessions.values(): + for session in list(self.hosts_to_sessions.values()): session.shutdown() def prepare_query(self, partition_key, token_range, attempts): @@ -1782,6 +1831,22 @@ class ParseError(Exception): pass +class ImmutableDict(frozenset): + """ + Immutable dictionary implementation to represent map types. + We need to pass BoundStatement.bind() a dict() because it calls iteritems(), + except we can't create a dict with another dict as the key, hence we use a class + that adds iteritems to a frozen set of tuples (which is how dict are normally made + immutable in python). + Must be declared in the top level of the module to be available for pickling. + """ + iteritems = frozenset.__iter__ + + def items(self): + for k, v in self.iteritems(): + yield k, v + + class ImportConversion(object): """ A class for converting strings to values when importing from csv, used by ImportProcess, @@ -1840,7 +1905,7 @@ class ImportConversion(object): select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks), protect_name(parent.table), where_clause) - return parent.session.prepare(select_query) + return parent.session.prepare(ensure_str(select_query)) @staticmethod def unprotect(v): @@ -1871,20 +1936,20 @@ class ImportConversion(object): return bytearray.fromhex(v[2:]) def convert_text(v, **_): - return v + return ensure_str(v) def convert_uuid(v, **_): return UUID(v) def convert_bool(v, **_): - return True if v.lower() == self.boolean_styles[0].lower() else False + return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False def get_convert_integer_fcn(adapter=int): """ Return a slow and a fast integer conversion function depending on self.thousands_sep """ if self.thousands_sep: - return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '')) + return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str(''))) else: return lambda v, ct=cql_type: adapter(v) @@ -1892,12 +1957,14 @@ class ImportConversion(object): """ Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep """ + empty_str = ensure_str('') + dot_str = ensure_str('.') if self.thousands_sep and self.decimal_sep: - return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '').replace(self.decimal_sep, '.')) + return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str)) elif self.thousands_sep: - return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '')) + return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str)) elif self.decimal_sep: - return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, '.')) + return lambda v, ct=cql_type: adapter(v.replace(self.decimal_sep, dot_str)) else: return lambda v, ct=cql_type: adapter(v) @@ -1949,14 +2016,20 @@ class ImportConversion(object): return ret # this should match all possible CQL and CQLSH datetime formats - p = re.compile("(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" + # YYYY-MM-DD[( |'T')] - "(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" + # [HH:MM[:SS[.NNNNNN]]] - "(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]] + p = re.compile(r"(\d{4})\-(\d{2})\-(\d{2})\s?(?:'T')?" # YYYY-MM-DD[( |'T')] + + r"(?:(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?))?" # [HH:MM[:SS[.NNNNNN]]] + + r"(?:([+\-])(\d{2}):?(\d{2}))?") # [(+|-)HH[:]MM]] def convert_datetime(val, **_): try: - tval = time.strptime(val, self.date_time_format) - return timegm(tval) * 1e3 # scale seconds to millis for the raw value + if six.PY2: + # Python 2 implementation + tval = time.strptime(val, self.date_time_format) + return timegm(tval) * 1e3 # scale seconds to millis for the raw value + else: + # Python 3 implementation + dtval = datetime.datetime.strptime(val, self.date_time_format) + return dtval.timestamp() * 1000 except ValueError: pass # if it's not in the default format we try CQL formats @@ -1986,7 +2059,7 @@ class ImportConversion(object): offset = -time.timezone # scale seconds to millis for the raw value - return ((timegm(tval) + offset) * 1e3) + milliseconds + return ((timegm(tval) + offset) * 1000) + milliseconds def convert_date(v, **_): return Date(v) @@ -2005,16 +2078,12 @@ class ImportConversion(object): def convert_map(val, ct=cql_type): """ - We need to pass to BoundStatement.bind() a dict() because it calls iteritems(), - except we can't create a dict with another dict as the key, hence we use a class - that adds iteritems to a frozen set of tuples (which is how dict are normally made - immutable in python). + See ImmutableDict above for a discussion of why a special object is needed here. """ - class ImmutableDict(frozenset): - iteritems = frozenset.__iter__ - + split_format_str = ensure_str('{%s}') + sep = ensure_str(':') return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1])) - for v in [split('{%s}' % vv, sep=':') for vv in split(val)])) + for v in [split(split_format_str % vv, sep=sep) for vv in split(val)])) def convert_user_type(val, ct=cql_type): """ @@ -2025,7 +2094,9 @@ class ImportConversion(object): Also note that it is possible that the subfield names in the csv are in the wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959. """ - vals = [v for v in [split('{%s}' % vv, sep=':') for vv in split(val)]] + split_format_str = ensure_str('{%s}') + sep = ensure_str(':') + vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]] dict_vals = dict((unprotect(v[0]), v[1]) for v in vals) sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val()) for n, t in zip(ct.fieldnames, ct.subtypes)] @@ -2053,11 +2124,11 @@ class ImportConversion(object): 'ascii': convert_text, 'float': get_convert_decimal_fcn(), 'double': get_convert_decimal_fcn(), - 'bigint': get_convert_integer_fcn(adapter=long), + 'bigint': get_convert_integer_fcn(adapter=int), 'int': get_convert_integer_fcn(), 'varint': get_convert_integer_fcn(), 'inet': convert_text, - 'counter': get_convert_integer_fcn(adapter=long), + 'counter': get_convert_integer_fcn(adapter=int), 'timestamp': convert_datetime, 'timeuuid': convert_uuid, 'date': convert_date, @@ -2082,7 +2153,7 @@ class ImportConversion(object): or "NULL" otherwise. Note that for counters we never use prepared statements, so we only check is_counter when use_prepared_statements is false. """ - return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL") + return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL")) def convert_row(self, row): """ @@ -2102,7 +2173,7 @@ class ImportConversion(object): def convert(c, v): try: return c(v) if v != self.nullval else self.get_null_val() - except Exception, e: + except Exception as e: # if we could not convert an empty string, then self.nullval has been set to a marker # because the user needs to import empty strings, except that the converters for some types # will fail to convert an empty string, in this case the null value should be inserted @@ -2112,7 +2183,7 @@ class ImportConversion(object): if self.debug: traceback.print_exc() - raise ParseError("Failed to parse %s : %s" % (val, e.message)) + raise ParseError("Failed to parse %s : %s" % (v, e.message if hasattr(e, 'message') else str(e))) return [convert(conv, val) for conv, val in zip(converters, row)] @@ -2149,6 +2220,7 @@ class ImportConversion(object): val = serialize(i, row[i]) length = len(val) pk_values.append(struct.pack(">H%dsB" % length, length, val, 0)) + return b"".join(pk_values) if len(partition_key_indexes) == 1: @@ -2204,7 +2276,7 @@ class TokenMap(object): def filter_replicas(self, hosts): shuffled = tuple(sorted(hosts, key=lambda k: random.random())) - return filter(lambda r: r.is_up is not False and r.datacenter == self.local_dc, shuffled) if hosts else () + return [r for r in shuffled if r.is_up is not False and r.datacenter == self.local_dc] if hosts else () class FastTokenAwarePolicy(DCAwareRoundRobinPolicy): @@ -2237,7 +2309,7 @@ class FastTokenAwarePolicy(DCAwareRoundRobinPolicy): return conn.in_flight < min(conn.max_request_id, self.max_inflight_messages) return True - for i in xrange(self.max_backoff_attempts): + for i in range(self.max_backoff_attempts): for r in filter(replica_is_not_overloaded, replicas): yield r @@ -2269,7 +2341,7 @@ class ImportProcess(ChildProcess): ChildProcess.__init__(self, params=params, target=self.run) self.skip_columns = params['skip_columns'] - self.valid_columns = [c.encode(self.encoding) for c in params['valid_columns']] + self.valid_columns = [c for c in params['valid_columns']] self.skip_column_indexes = [i for i, c in enumerate(self.columns) if c in self.skip_columns] options = params['options'] @@ -2312,6 +2384,9 @@ class ImportProcess(ChildProcess): return self._session def run(self): + if self.coverage: + self.start_coverage() + try: pr = profile_on() if PROFILE_ON else None @@ -2321,10 +2396,12 @@ class ImportProcess(ChildProcess): if pr: profile_off(pr, file_name='worker_profile_%d.txt' % (os.getpid(),)) - except Exception, exc: + except Exception as exc: self.report_error(exc) finally: + if self.coverage: + self.stop_coverage() self.close() def close(self): @@ -2361,6 +2438,7 @@ class ImportProcess(ChildProcess): if self.ttl >= 0: query += 'USING TTL %s' % (self.ttl,) make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement) + query = ensure_str(query) conv = ImportConversion(self, table_meta, prepared_statement) tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session) @@ -2399,15 +2477,15 @@ class ImportProcess(ChildProcess): # causes the statement to be None, then we should not report the error so that we can test # the parent process handling missing batches from child processes - except Exception, exc: + except Exception as exc: self.report_error(exc, chunk, chunk['rows']) def wrap_make_statement(self, inner_make_statement): def make_statement(query, conv, chunk, batch, replicas): try: return inner_make_statement(query, conv, batch, replicas) - except Exception, exc: - print "Failed to make batch statement: {}".format(exc) + except Exception as exc: + print("Failed to make batch statement: {}".format(exc)) self.report_error(exc, chunk, batch['rows']) return None @@ -2428,12 +2506,12 @@ class ImportProcess(ChildProcess): set_clause = [] for i, value in enumerate(row): if i in conv.primary_key_indexes: - where_clause.append("%s=%s" % (self.valid_columns[i], value)) + where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value))) else: - set_clause.append("%s=%s+%s" % (self.valid_columns[i], self.valid_columns[i], value)) + set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value))) - full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause)) - statement.add(full_query_text) + full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause)) + statement.add(ensure_str(full_query_text)) return statement def make_prepared_batch_statement(self, query, _, batch, replicas): @@ -2457,7 +2535,8 @@ class ImportProcess(ChildProcess): statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level) statement.replicas = replicas statement.keyspace = self.ks - statement._statements_and_parameters = [(False, query % (','.join(r),), ()) for r in batch['rows']] + field_sep = b',' if six.PY2 else ',' + statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']] return statement def convert_rows(self, conv, chunk): @@ -2477,14 +2556,14 @@ class ImportProcess(ChildProcess): def convert_row(r): try: return conv.convert_row(r) - except Exception, err: - errors[err.message].append(r) + except Exception as err: + errors[err.message if hasattr(err, 'message') else str(err)].append(r) return None - converted_rows = filter(None, [convert_row(r) for r in rows]) + converted_rows = [_f for _f in [convert_row(r) for r in rows] if _f] if errors: - for msg, rows in errors.iteritems(): + for msg, rows in errors.items(): self.report_error(ParseError(msg), chunk, rows) return converted_rows @@ -2549,27 +2628,27 @@ class ImportProcess(ChildProcess): try: pk = get_row_partition_key_values(row) rows_by_ring_pos[get_ring_pos(ring, pk_to_token_value(pk))].append(row) - except Exception, e: + except Exception as e: errors[e.message].append(row) if errors: - for msg, rows in errors.iteritems(): + for msg, rows in errors.items(): self.report_error(ParseError(msg), chunk, rows) replicas = tm.replicas filter_replicas = tm.filter_replicas rows_by_replica = defaultdict(list) - for ring_pos, rows in rows_by_ring_pos.iteritems(): + for ring_pos, rows in rows_by_ring_pos.items(): if len(rows) > min_batch_size: - for i in xrange(0, len(rows), max_batch_size): + for i in range(0, len(rows), max_batch_size): yield filter_replicas(replicas[ring_pos]), make_batch(chunk['id'], rows[i:i + max_batch_size]) else: # select only the first valid replica to guarantee more overlap or none at all - rows_by_replica[filter_replicas(replicas[ring_pos])[:1]].extend(rows) + rows_by_replica[tuple(filter_replicas(replicas[ring_pos])[:1])].extend(rows) # TODO: revisit tuple wrapper # Now send the batches by replica - for replicas, rows in rows_by_replica.iteritems(): - for i in xrange(0, len(rows), max_batch_size): + for replicas, rows in rows_by_replica.items(): + for i in range(0, len(rows), max_batch_size): yield replicas, make_batch(chunk['id'], rows[i:i + max_batch_size]) def result_callback(self, _, batch, chunk): @@ -2590,7 +2669,8 @@ class ImportProcess(ChildProcess): def report_error(self, err, chunk=None, rows=None, attempts=1, final=True): if self.debug and sys.exc_info()[1] == err: traceback.print_exc() - self.outmsg.send(ImportTaskError(err.__class__.__name__, err.message, rows, attempts, final)) + err_msg = err.message if hasattr(err, 'message') else str(err) + self.outmsg.send(ImportTaskError(err.__class__.__name__, err_msg, rows, attempts, final)) if final and chunk is not None: self.update_chunk(rows, chunk) diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index 220e1078f6..eabb059c66 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -14,15 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .cqlhandling import CqlParsingRuleSet, Hint from cassandra.metadata import maybe_escape_name - +from cqlshlib import helptopics +from cqlshlib.cqlhandling import CqlParsingRuleSet, Hint simple_cql_types = set(('ascii', 'bigint', 'blob', 'boolean', 'counter', 'date', 'decimal', 'double', 'duration', 'float', 'inet', 'int', 'smallint', 'text', 'time', 'timestamp', 'timeuuid', 'tinyint', 'uuid', 'varchar', 'varint')) simple_cql_types.difference_update(('set', 'map', 'list')) -from . import helptopics cqldocs = helptopics.CQL3HelpTopics() @@ -442,8 +441,8 @@ def ks_prop_val_mapkey_completer(ctxt, cass): optname = ctxt.get_binding('propname')[-1] if optname != 'replication': return () - keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ())) - valsseen = map(dequote_value, ctxt.get_binding('propmapval', ())) + keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ()))) + valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ()))) for k, v in zip(keysseen, valsseen): if k == 'class': repclass = v @@ -454,7 +453,7 @@ def ks_prop_val_mapkey_completer(ctxt, cass): opts = set(('replication_factor',)) elif repclass == 'NetworkTopologyStrategy': return [Hint('')] - return map(escape_value, opts.difference(keysseen)) + return list(map(escape_value, opts.difference(keysseen))) def ks_prop_val_mapval_completer(ctxt, cass): @@ -463,7 +462,7 @@ def ks_prop_val_mapval_completer(ctxt, cass): return () currentkey = dequote_value(ctxt.get_binding('propmapkey')[-1]) if currentkey == 'class': - return map(escape_value, CqlRuleSet.replication_strategies) + return list(map(escape_value, CqlRuleSet.replication_strategies)) return [Hint('')] @@ -471,8 +470,8 @@ def ks_prop_val_mapender_completer(ctxt, cass): optname = ctxt.get_binding('propname')[-1] if optname != 'replication': return [','] - keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ())) - valsseen = map(dequote_value, ctxt.get_binding('propmapval', ())) + keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ()))) + valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ()))) for k, v in zip(keysseen, valsseen): if k == 'class': repclass = v @@ -488,8 +487,8 @@ def ks_prop_val_mapender_completer(ctxt, cass): def cf_prop_name_completer(ctxt, cass): - return [c[0] for c in (CqlRuleSet.columnfamily_layout_options + - CqlRuleSet.columnfamily_layout_map_options)] + return [c[0] for c in (CqlRuleSet.columnfamily_layout_options + + CqlRuleSet.columnfamily_layout_map_options)] def cf_prop_val_completer(ctxt, cass): @@ -522,13 +521,13 @@ def cf_prop_val_mapkey_completer(ctxt, cass): break else: return () - keysseen = map(dequote_value, ctxt.get_binding('propmapkey', ())) - valsseen = map(dequote_value, ctxt.get_binding('propmapval', ())) - pairsseen = dict(zip(keysseen, valsseen)) + keysseen = list(map(dequote_value, ctxt.get_binding('propmapkey', ()))) + valsseen = list(map(dequote_value, ctxt.get_binding('propmapval', ()))) + pairsseen = dict(list(zip(keysseen, valsseen))) if optname == 'compression': - return map(escape_value, set(subopts).difference(keysseen)) + return list(map(escape_value, set(subopts).difference(keysseen))) if optname == 'caching': - return map(escape_value, set(subopts).difference(keysseen)) + return list(map(escape_value, set(subopts).difference(keysseen))) if optname == 'compaction': opts = set(subopts) try: @@ -545,7 +544,7 @@ def cf_prop_val_mapkey_completer(ctxt, cass): elif csc == 'TimeWindowCompactionStrategy': opts = opts.union(set(CqlRuleSet.time_window_compaction_strategy_options)) - return map(escape_value, opts) + return list(map(escape_value, opts)) return () @@ -554,11 +553,11 @@ def cf_prop_val_mapval_completer(ctxt, cass): key = dequote_value(ctxt.get_binding('propmapkey')[-1]) if opt == 'compaction': if key == 'class': - return map(escape_value, CqlRuleSet.available_compaction_classes) + return list(map(escape_value, CqlRuleSet.available_compaction_classes)) return [Hint('')] elif opt == 'compression': if key == 'sstable_compression': - return map(escape_value, CqlRuleSet.available_compression_classes) + return list(map(escape_value, CqlRuleSet.available_compression_classes)) return [Hint('')] elif opt == 'caching': if key == 'rows_per_partition': @@ -584,19 +583,19 @@ def storagetype_completer(ctxt, cass): @completer_for('keyspaceName', 'ksname') def ks_name_completer(ctxt, cass): - return map(maybe_escape_name, cass.get_keyspace_names()) + return list(map(maybe_escape_name, cass.get_keyspace_names())) @completer_for('nonSystemKeyspaceName', 'ksname') def non_system_ks_name_completer(ctxt, cass): ksnames = [n for n in cass.get_keyspace_names() if n not in SYSTEM_KEYSPACES] - return map(maybe_escape_name, ksnames) + return list(map(maybe_escape_name, ksnames)) @completer_for('alterableKeyspaceName', 'ksname') def alterable_ks_name_completer(ctxt, cass): ksnames = [n for n in cass.get_keyspace_names() if n not in NONALTERBALE_KEYSPACES] - return map(maybe_escape_name, ksnames) + return list(map(maybe_escape_name, ksnames)) def cf_ks_name_completer(ctxt, cass): @@ -629,7 +628,7 @@ def cf_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, cfnames) + return list(map(maybe_escape_name, cfnames)) @completer_for('materializedViewName', 'mvname') @@ -643,7 +642,7 @@ def mv_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, mvnames) + return list(map(maybe_escape_name, mvnames)) completer_for('userTypeName', 'ksname')(cf_ks_name_completer) @@ -661,7 +660,7 @@ def ut_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, utnames) + return list(map(maybe_escape_name, utnames)) completer_for('userTypeName', 'utname')(ut_name_completer) @@ -752,7 +751,7 @@ def udf_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, udfnames) + return list(map(maybe_escape_name, udfnames)) def uda_name_completer(ctxt, cass): @@ -765,7 +764,7 @@ def uda_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, udanames) + return list(map(maybe_escape_name, udanames)) def udf_uda_name_completer(ctxt, cass): @@ -778,7 +777,7 @@ def udf_uda_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, functionnames) + return list(map(maybe_escape_name, functionnames)) def ref_udf_name_completer(ctxt, cass): @@ -786,7 +785,7 @@ def ref_udf_name_completer(ctxt, cass): udanames = cass.get_userfunction_names(None) except Exception: return () - return map(maybe_escape_name, udanames) + return list(map(maybe_escape_name, udanames)) completer_for('functionAggregateName', 'ksname')(cf_ks_name_completer) @@ -844,7 +843,7 @@ def relation_token_subject_completer(ctxt, cass): def select_relation_lhs_completer(ctxt, cass): layout = get_table_meta(ctxt, cass) filterable = set() - already_filtered_on = map(dequote_name, ctxt.get_binding('rel_lhs', ())) + already_filtered_on = list(map(dequote_name, ctxt.get_binding('rel_lhs', ()))) for num in range(0, len(layout.partition_key)): if num == 0 or layout.partition_key[num - 1].name in already_filtered_on: filterable.add(layout.partition_key[num].name) @@ -855,9 +854,9 @@ def select_relation_lhs_completer(ctxt, cass): filterable.add(layout.clustering_key[num].name) else: break - for idx in layout.indexes.itervalues(): + for idx in layout.indexes.values(): filterable.add(idx.index_options["target"]) - return map(maybe_escape_name, filterable) + return list(map(maybe_escape_name, filterable)) explain_completion('selector', 'colname') @@ -880,9 +879,9 @@ syntax_rules += r''' def regular_column_names(table_meta): if not table_meta or not table_meta.columns: return [] - regular_columns = list(set(table_meta.columns.keys()) - - set([key.name for key in table_meta.partition_key]) - - set([key.name for key in table_meta.clustering_key])) + regular_columns = list(set(table_meta.columns.keys()) + - set([key.name for key in table_meta.partition_key]) + - set([key.name for key in table_meta.clustering_key])) return regular_columns @@ -895,13 +894,13 @@ def insert_colname_completer(ctxt, cass): if k.name not in colnames: return [maybe_escape_name(k.name)] normalcols = set(regular_column_names(layout)) - colnames - return map(maybe_escape_name, normalcols) + return list(map(maybe_escape_name, normalcols)) @completer_for('insertStatement', 'newval') def insert_newval_completer(ctxt, cass): layout = get_table_meta(ctxt, cass) - insertcols = map(dequote_name, ctxt.get_binding('colname')) + insertcols = list(map(dequote_name, ctxt.get_binding('colname'))) valuesdone = ctxt.get_binding('newval', ()) if len(valuesdone) >= len(insertcols): return [] @@ -974,7 +973,7 @@ def update_option_completer(ctxt, cass): @completer_for('assignment', 'updatecol') def update_col_completer(ctxt, cass): layout = get_table_meta(ctxt, cass) - return map(maybe_escape_name, regular_column_names(layout)) + return list(map(maybe_escape_name, regular_column_names(layout))) @completer_for('assignment', 'update_rhs') @@ -1116,7 +1115,7 @@ def delete_opt_completer(ctxt, cass): @completer_for('deleteSelector', 'delcol') def delete_delcol_completer(ctxt, cass): layout = get_table_meta(ctxt, cass) - return map(maybe_escape_name, regular_column_names(layout)) + return list(map(maybe_escape_name, regular_column_names(layout))) syntax_rules += r''' @@ -1197,7 +1196,7 @@ syntax_rules += r''' @completer_for('cfamOrdering', 'ordercol') def create_cf_clustering_order_colname_completer(ctxt, cass): - colnames = map(dequote_name, ctxt.get_binding('newcolname', ())) + colnames = list(map(dequote_name, ctxt.get_binding('newcolname', ()))) # Definitely some of these aren't valid for ordering, but I'm not sure # precisely which are. This is good enough for now return colnames @@ -1227,7 +1226,7 @@ def create_cf_ks_dot_completer(ctxt, cass): def create_cf_pkdef_declaration_completer(ctxt, cass): cols_declared = ctxt.get_binding('newcolname') pieces_already = ctxt.get_binding('ptkey', ()) - pieces_already = map(dequote_name, pieces_already) + pieces_already = list(map(dequote_name, pieces_already)) while cols_declared[0] in pieces_already: cols_declared = cols_declared[1:] if len(cols_declared) < 2: @@ -1239,7 +1238,7 @@ def create_cf_pkdef_declaration_completer(ctxt, cass): def create_cf_composite_key_declaration_completer(ctxt, cass): cols_declared = ctxt.get_binding('newcolname') pieces_already = ctxt.get_binding('ptkey', ()) + ctxt.get_binding('pkey', ()) - pieces_already = map(dequote_name, pieces_already) + pieces_already = list(map(dequote_name, pieces_already)) while cols_declared[0] in pieces_already: cols_declared = cols_declared[1:] if len(cols_declared) < 2: @@ -1325,9 +1324,9 @@ explain_completion('createUserTypeStatement', 'newcol', '') def create_index_col_completer(ctxt, cass): """ Return the columns for which an index doesn't exist yet. """ layout = get_table_meta(ctxt, cass) - idx_targets = [idx.index_options["target"] for idx in layout.indexes.itervalues()] - colnames = [cd.name for cd in layout.columns.values() if cd.name not in idx_targets] - return map(maybe_escape_name, colnames) + idx_targets = [idx.index_options["target"] for idx in layout.indexes.values()] + colnames = [cd.name for cd in list(layout.columns.values()) if cd.name not in idx_targets] + return list(map(maybe_escape_name, colnames)) syntax_rules += r''' @@ -1385,7 +1384,7 @@ def idx_ks_idx_name_completer(ctxt, cass): if ks is None: return () raise - return map(maybe_escape_name, idxnames) + return list(map(maybe_escape_name, idxnames)) syntax_rules += r''' @@ -1413,14 +1412,14 @@ syntax_rules += r''' def alter_table_col_completer(ctxt, cass): layout = get_table_meta(ctxt, cass) cols = [str(md) for md in layout.columns] - return map(maybe_escape_name, cols) + return list(map(maybe_escape_name, cols)) @completer_for('alterTypeInstructions', 'existcol') def alter_type_field_completer(ctxt, cass): layout = get_ut_layout(ctxt, cass) fields = [tuple[0] for tuple in layout] - return map(maybe_escape_name, fields) + return list(map(maybe_escape_name, fields)) explain_completion('alterInstructions', 'newcol', '') @@ -1557,7 +1556,7 @@ def username_name_completer(ctxt, cass): return [Hint('')] session = cass.session - return [maybe_quote(row.values()[0].replace("'", "''")) for row in session.execute("LIST USERS")] + return [maybe_quote(list(row.values())[0].replace("'", "''")) for row in session.execute("LIST USERS")] @completer_for('rolename', 'role') @@ -1596,7 +1595,7 @@ def get_trigger_names(ctxt, cass): @completer_for('dropTriggerStatement', 'triggername') def drop_trigger_completer(ctxt, cass): names = get_trigger_names(ctxt, cass) - return map(maybe_escape_name, names) + return list(map(maybe_escape_name, names)) # END SYNTAX/COMPLETION RULE DEFINITIONS diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py index ab18e9c87c..9305dfa96b 100644 --- a/pylib/cqlshlib/cqlhandling.py +++ b/pylib/cqlshlib/cqlhandling.py @@ -18,8 +18,9 @@ # i.e., stuff that's not necessarily cqlsh-specific import traceback + from cassandra.metadata import cql_keywords_reserved -from . import pylexotron, util +from cqlshlib import pylexotron, util Hint = pylexotron.Hint @@ -66,7 +67,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): if cass is None: return () return f(ctxt, cass) - completerwrapper.func_name = 'completerwrapper_on_' + f.func_name + completerwrapper.__name__ = 'completerwrapper_on_' + f.__name__ self.register_completer(completerwrapper, rulename, symname) return completerwrapper return registrator @@ -100,12 +101,6 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): # operations on tokens (like .lower()). See CASSANDRA-9083 # for one example of this. str_token = t[1] - if isinstance(str_token, unicode): - try: - str_token = str_token.encode('ascii') - t = (t[0], str_token) + t[2:] - except UnicodeEncodeError: - pass curstmt.append(t) if t[0] == 'endtoken': @@ -196,7 +191,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): f = lambda s: s and dequoter(s).lower().startswith(partial) else: f = lambda s: s and dequoter(s).startswith(partial) - candidates = filter(f, strcompletes) + candidates = list(filter(f, strcompletes)) if prefix is not None: # dequote, re-escape, strip quotes: gets us the right quoted text @@ -207,7 +202,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): # the above process can result in an empty string; this doesn't help for # completions - candidates = filter(None, candidates) + candidates = [_f for _f in candidates if _f] # prefix a space when desirable for pleasant cql formatting if tokens: @@ -249,7 +244,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): init_bindings = {'cassandra_conn': cassandra_conn} if debug: init_bindings['*DEBUG*'] = True - print "cql_complete(%r, partial=%r)" % (text, partial) + print("cql_complete(%r, partial=%r)" % (text, partial)) completions, hints = self.cql_complete_single(text, partial, init_bindings, startsymbol=startsymbol) @@ -261,12 +256,12 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): if len(completions) == 1 and len(hints) == 0: c = completions[0] if debug: - print "** Got one completion: %r. Checking for further matches...\n" % (c,) + print("** Got one completion: %r. Checking for further matches...\n" % (c,)) if not c.isspace(): new_c = self.cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol) completions = [new_c] if debug: - print "** New list of completions: %r" % (completions,) + print("** New list of completions: %r" % (completions,)) return hints + completions @@ -277,18 +272,18 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): startsymbol=startsymbol) except Exception: if debug: - print "** completion expansion had a problem:" + print("** completion expansion had a problem:") traceback.print_exc() return first if hints: if not first[-1].isspace(): first += ' ' if debug: - print "** completion expansion found hints: %r" % (hints,) + print("** completion expansion found hints: %r" % (hints,)) return first if len(completions) == 1 and completions[0] != '': if debug: - print "** Got another completion: %r." % (completions[0],) + print("** Got another completion: %r." % (completions[0],)) if completions[0][0] in (',', ')', ':') and first[-1] == ' ': first = first[:-1] first += completions[0] @@ -299,10 +294,10 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): if common_prefix[0] in (',', ')', ':') and first[-1] == ' ': first = first[:-1] if debug: - print "** Got a partial completion: %r." % (common_prefix,) + print("** Got a partial completion: %r." % (common_prefix,)) return first + common_prefix if debug: - print "** New total completion: %r. Checking for further matches...\n" % (first,) + print("** New total completion: %r. Checking for further matches...\n" % (first,)) return self.cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol) @staticmethod diff --git a/pylib/cqlshlib/cqlshhandling.py b/pylib/cqlshlib/cqlshhandling.py index 7abd6cef08..193ca8879f 100644 --- a/pylib/cqlshlib/cqlshhandling.py +++ b/pylib/cqlshlib/cqlshhandling.py @@ -15,7 +15,8 @@ # limitations under the License. import os -import cqlhandling + +from cqlshlib import cqlhandling # we want the cql parser to understand our cqlsh-specific commands too my_commands_ending_with_newline = ( @@ -240,7 +241,7 @@ def complete_source_quoted_filename(ctxt, cqlsh): contents = os.listdir(exhead or '.') except OSError: return () - matches = filter(lambda f: f.startswith(tail), contents) + matches = [f for f in contents if f.startswith(tail)] annotated = [] for f in matches: match = os.path.join(head, f) @@ -267,7 +268,7 @@ def copy_fname_completer(ctxt, cqlsh): @cqlsh_syntax_completer('copyCommand', 'colnames') def complete_copy_column_names(ctxt, cqlsh): - existcols = map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ())) + existcols = list(map(cqlsh.cql_unprotect_name, ctxt.get_binding('colnames', ()))) ks = cqlsh.cql_unprotect_name(ctxt.get_binding('ksname', None)) cf = cqlsh.cql_unprotect_name(ctxt.get_binding('cfname')) colnames = cqlsh.get_column_names(ks, cf) @@ -288,7 +289,7 @@ COPY_TO_OPTIONS = ['ENCODING', 'PAGESIZE', 'PAGETIMEOUT', 'BEGINTOKEN', 'ENDTOKE @cqlsh_syntax_completer('copyOption', 'optnames') def complete_copy_options(ctxt, cqlsh): - optnames = map(str.upper, ctxt.get_binding('optnames', ())) + optnames = list(map(str.upper, ctxt.get_binding('optnames', ()))) direction = ctxt.get_binding('dir').upper() if direction == 'FROM': opts = set(COPY_COMMON_OPTIONS + COPY_FROM_OPTIONS) - set(optnames) diff --git a/pylib/cqlshlib/displaying.py b/pylib/cqlshlib/displaying.py index 424d6334b6..ef076f76a9 100644 --- a/pylib/cqlshlib/displaying.py +++ b/pylib/cqlshlib/displaying.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + from collections import defaultdict RED = '\033[0;1;31m' @@ -41,7 +43,7 @@ def get_str(val): return val -class FormattedValue: +class FormattedValue(object): def __init__(self, strval, coloredval=None, displaywidth=None): self.strval = strval diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py index 803ea63f89..7ed6735b12 100644 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + import binascii import calendar import datetime @@ -21,15 +23,17 @@ import math import os import re import sys -import six import platform -import wcwidth + +from six import ensure_text from collections import defaultdict -from displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP + from cassandra.cqltypes import EMPTY from cassandra.util import datetime_from_timestamp -from util import UTC +from . import wcwidth +from .displaying import colorme, get_str, FormattedValue, DEFAULT_VALUE_COLORS, NO_COLOR_MAP +from .util import UTC is_win = platform.system() == 'Windows' @@ -204,7 +208,7 @@ class CqlType(object): def format_value_default(val, colormap, **_): - val = str(val) + val = ensure_text(str(val)) escapedval = val.replace('\\', '\\\\') bval = controlchars_re.sub(_show_control_chars, escapedval) return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap) @@ -218,7 +222,6 @@ _formatters = {} def format_value(val, cqltype, **kwargs): if val == EMPTY: return format_value_default('', **kwargs) - formatter = get_formatter(val, cqltype) return formatter(val, cqltype=cqltype, **kwargs) @@ -239,7 +242,7 @@ def formatter_for(typname): @formatter_for('bytearray') def format_value_blob(val, colormap, **_): - bval = '0x' + binascii.hexlify(val) + bval = ensure_text('0x') + ensure_text(binascii.hexlify(val)) return colorme(bval, colormap, 'blob') @@ -248,7 +251,7 @@ formatter_for('blob')(format_value_blob) def format_python_formatted_type(val, colormap, color, quote=False): - bval = str(val) + bval = ensure_text(str(val)) if quote: bval = "'%s'" % bval return colorme(bval, colormap, color) @@ -318,6 +321,7 @@ formatter_for('double')(format_floating_point_type) def format_integer_type(val, colormap, thousands_sep=None, **_): # base-10 only for now; support others? bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val) + bval = ensure_text(bval) return colorme(bval, colormap, 'int') @@ -352,7 +356,7 @@ def format_value_timestamp(val, colormap, date_time_format, quote=False, **_): if date_time_format.milliseconds_only: bval = round_microseconds(bval) else: - bval = str(val) + bval = ensure_text(str(val)) if quote: bval = "'%s'" % bval @@ -378,7 +382,7 @@ def strftime(time_format, seconds, microseconds=0, timezone=None): return '%d' % (seconds * 1000.0) -microseconds_regex = re.compile("(.*)(?:\.(\d{1,6}))(.*)") +microseconds_regex = re.compile(r"(.*)(?:\.(\d{1,6}))(.*)") def round_microseconds(val): @@ -453,14 +457,14 @@ def decode_unsigned_vint(buf): For example, if we need to read 3 more bytes the first byte will start with 1110. """ - first_byte = buf.next() + first_byte = next(buf) if (first_byte >> 7) == 0: return first_byte size = number_of_extra_bytes_to_read(first_byte) retval = first_byte & (0xff >> size) for i in range(size): - b = buf.next() + b = next(buf) retval <<= 8 retval |= b & 0xff @@ -477,15 +481,14 @@ def decode_zig_zag_64(n): @formatter_for('str') def format_value_text(val, encoding, colormap, quote=False, **_): - escapedval = val.replace(u'\\', u'\\\\') + escapedval = val.replace('\\', '\\\\') if quote: escapedval = escapedval.replace("'", "''") escapedval = unicode_controlchars_re.sub(_show_control_chars, escapedval) - bval = escapedval.encode(encoding, 'backslashreplace') + bval = escapedval if quote: - bval = "'%s'" % bval - - return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval.decode(encoding))) + bval = "'{}'".format(bval) + return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap, wcwidth.wcswidth(bval)) # name alias @@ -532,7 +535,7 @@ def format_value_tuple(val, cqltype, encoding, colormap, date_time_format, float @formatter_for('set') def format_value_set(val, cqltype, encoding, colormap, date_time_format, float_precision, nullval, decimal_sep, thousands_sep, boolean_styles, **_): - return format_simple_collection(sorted(val), cqltype, '{', '}', encoding, colormap, + return format_simple_collection(val, cqltype, '{', '}', encoding, colormap, date_time_format, float_precision, nullval, decimal_sep, thousands_sep, boolean_styles) @@ -584,7 +587,7 @@ def format_value_utype(val, cqltype, encoding, colormap, date_time_format, float def format_field_name(name): return format_value_text(name, encoding=encoding, colormap=colormap, quote=False) - subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(val._asdict().items(), + subs = [(format_field_name(k), format_field_value(v, t)) for ((k, v), t) in zip(list(val._asdict().items()), cqltype.sub_types)] bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}' if colormap is NO_COLOR_MAP: diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py index 7b11eac245..69f31dced7 100644 --- a/pylib/cqlshlib/pylexotron.py +++ b/pylib/cqlshlib/pylexotron.py @@ -15,7 +15,8 @@ # limitations under the License. import re -from .saferscanner import SaferScanner + +from cqlshlib.saferscanner import SaferScanner class LexingError(Exception): @@ -107,15 +108,6 @@ class ParseContext: return ' '.join([t[1] for t in tokens]) # low end of span for first token, to high end of span for last token orig_text = orig[tokens[0][2][0]:tokens[-1][2][1]] - - # Convert all unicode tokens to ascii, where possible. This - # helps avoid problems with performing unicode-incompatible - # operations on tokens (like .lower()). See CASSANDRA-9083 - # for one example of this. - try: - orig_text = orig_text.encode('ascii') - except UnicodeEncodeError: - pass return orig_text def __repr__(self): @@ -146,7 +138,7 @@ class matcher: except KeyError: return False if debugging: - print "Trying completer %r with %r" % (completer, ctxt) + print("Trying completer %r with %r" % (completer, ctxt)) try: new_compls = completer(ctxt) except Exception: @@ -155,7 +147,7 @@ class matcher: traceback.print_exc() return False if debugging: - print "got %r" % (new_compls,) + print("got %r" % (new_compls,)) completions.update(new_compls) return True @@ -284,7 +276,7 @@ class text_match(terminal_matcher): try: terminal_matcher.__init__(self, eval(text)) except SyntaxError: - print "bad syntax %r" % (text,) + print("bad syntax %r" % (text,)) def match(self, ctxt, completions): if ctxt.remainder: @@ -359,7 +351,7 @@ class ParsingRuleSet: (r'[@()|?*;]', lambda s, t: t), (r'\s+', None), (r'#[^\n]*', None), - ], re.I | re.S) + ], re.I | re.S | re.U) def __init__(self): self.ruleset = {} @@ -382,7 +374,7 @@ class ParsingRuleSet: tokeniter = iter(tokens) for t in tokeniter: if isinstance(t, tuple) and t[0] in ('reference', 'junk'): - assign = tokeniter.next() + assign = next(tokeniter) if assign != '::=': raise ValueError('Unexpected token %r; expected "::="' % (assign,)) name = t[1] @@ -405,7 +397,7 @@ class ParsingRuleSet: @classmethod def read_rule_tokens_until(cls, endtoks, tokeniter): - if isinstance(endtoks, basestring): + if isinstance(endtoks, str): endtoks = (endtoks,) counttarget = None if isinstance(endtoks, int): @@ -419,7 +411,7 @@ class ParsingRuleSet: if t in endtoks: if len(mybranches) == 1: return cls.mkrule(mybranches[0]) - return choice(map(cls.mkrule, mybranches)) + return choice(list(map(cls.mkrule, mybranches))) if isinstance(t, tuple): if t[0] == 'reference': t = rule_reference(t[1]) @@ -441,7 +433,7 @@ class ParsingRuleSet: elif t == '*': t = repeat(myrules.pop(-1)) elif t == '@': - x = tokeniter.next() + x = next(tokeniter) if not isinstance(x, tuple) or x[0] != 'litstring': raise ValueError("Unexpected token %r following '@'" % (x,)) t = case_match(x[1]) @@ -455,7 +447,7 @@ class ParsingRuleSet: if countsofar == counttarget: if len(mybranches) == 1: return cls.mkrule(mybranches[0]) - return choice(map(cls.mkrule, mybranches)) + return choice(list(map(cls.mkrule, mybranches))) raise ValueError('Unexpected end of rule tokens') def append_rules(self, rulestr): @@ -474,7 +466,7 @@ class ParsingRuleSet: return None return lambda s, t: (name, t, s.match.span()) regexes = [(p.pattern(), make_handler(name)) for (name, p) in self.terminals] - return SaferScanner(regexes, re.I | re.S).scan + return SaferScanner(regexes, re.I | re.S | re.U).scan def lex(self, text): if self.scanner is None: diff --git a/pylib/cqlshlib/saferscanner.py b/pylib/cqlshlib/saferscanner.py index 2a05608f7a..e559f1c067 100644 --- a/pylib/cqlshlib/saferscanner.py +++ b/pylib/cqlshlib/saferscanner.py @@ -19,24 +19,11 @@ # regex in-pattern flags. Any of those can break correct operation of Scanner. import re +import six from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS -class SaferScanner(re.Scanner): - - def __init__(self, lexicon, flags=0): - self.lexicon = lexicon - p = [] - s = re.sre_parse.Pattern() - s.flags = flags - for phrase, action in lexicon: - p.append(re.sre_parse.SubPattern(s, [ - (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))), - ])) - s.groups = len(p) + 1 - p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))]) - self.p = p - self.scanner = re.sre_compile.compile(p) +class SaferScannerBase(re.Scanner): @classmethod def subpat(cls, phrase, flags): @@ -60,3 +47,39 @@ class SaferScanner(re.Scanner): if sub.pattern.flags ^ flags: raise ValueError("RE flag setting not allowed in SaferScanner lexicon (%s)" % (bin(sub.pattern.flags),)) return re.sre_parse.SubPattern(sub.pattern, scrubbedsub) + + +class Py2SaferScanner(SaferScannerBase): + + def __init__(self, lexicon, flags=0): + self.lexicon = lexicon + p = [] + s = re.sre_parse.Pattern() + s.flags = flags + for phrase, action in lexicon: + p.append(re.sre_parse.SubPattern(s, [ + (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))), + ])) + s.groups = len(p) + 1 + p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.p = p + self.scanner = re.sre_compile.compile(p) + + +class Py36SaferScanner(SaferScannerBase): + + def __init__(self, lexicon, flags=0): + self.lexicon = lexicon + p = [] + s = re.sre_parse.Pattern() + s.flags = flags + for phrase, action in lexicon: + gid = s.opengroup() + p.append(re.sre_parse.SubPattern(s, [(SUBPATTERN, (gid, 0, 0, re.sre_parse.parse(phrase, flags))), ])) + s.closegroup(gid, p[-1]) + p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.p = p + self.scanner = re.sre_compile.compile(p) + + +SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner diff --git a/pylib/cqlshlib/sslhandling.py b/pylib/cqlshlib/sslhandling.py index 8765ffa31a..b6a136995f 100644 --- a/pylib/cqlshlib/sslhandling.py +++ b/pylib/cqlshlib/sslhandling.py @@ -16,9 +16,10 @@ import os import sys -import ConfigParser import ssl +from six.moves import configparser + def ssl_settings(host, config_file, env=os.environ): """ @@ -38,13 +39,13 @@ def ssl_settings(host, config_file, env=os.environ): either in the config file or as an environment variable. Environment variables override any options set in cqlsh config file. """ - configs = ConfigParser.SafeConfigParser() + configs = configparser.SafeConfigParser() configs.read(config_file) def get_option(section, option): try: return configs.get(section, option) - except ConfigParser.Error: + except configparser.Error: return None ssl_validate = env.get('SSL_VALIDATE') diff --git a/pylib/cqlshlib/test/__init__.py b/pylib/cqlshlib/test/__init__.py index ba8f373bfa..4bb037e118 100644 --- a/pylib/cqlshlib/test/__init__.py +++ b/pylib/cqlshlib/test/__init__.py @@ -15,6 +15,3 @@ # limitations under the License. from .cassconnect import create_db, remove_db - -setUp = create_db -tearDown = remove_db diff --git a/pylib/cqlshlib/test/ansi_colors.py b/pylib/cqlshlib/test/ansi_colors.py index b0bc7382c3..9fc341154c 100644 --- a/pylib/cqlshlib/test/ansi_colors.py +++ b/pylib/cqlshlib/test/ansi_colors.py @@ -14,31 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals + import re +import six -LIGHT = 010 +LIGHT = 0o10 -ansi_CSI = '\033[' + +ansi_CSI = '\x1b[' ansi_seq = re.compile(re.escape(ansi_CSI) + r'(?P[\x20-\x3f]*)(?P[\x40-\x7e])') ansi_cmd_SGR = 'm' # set graphics rendition color_defs = ( (000, 'k', 'black'), - (001, 'r', 'dark red'), - (002, 'g', 'dark green'), - (003, 'w', 'brown', 'dark yellow'), - (004, 'b', 'dark blue'), - (005, 'm', 'dark magenta', 'dark purple'), - (006, 'c', 'dark cyan'), - (007, 'n', 'light grey', 'light gray', 'neutral', 'dark white'), - (010, 'B', 'dark grey', 'dark gray', 'light black'), - (011, 'R', 'red', 'light red'), - (012, 'G', 'green', 'light green'), - (013, 'Y', 'yellow', 'light yellow'), - (014, 'B', 'blue', 'light blue'), - (015, 'M', 'magenta', 'purple', 'light magenta', 'light purple'), - (016, 'C', 'cyan', 'light cyan'), - (017, 'W', 'white', 'light white'), + (0o01, 'r', 'dark red'), + (0o02, 'g', 'dark green'), + (0o03, 'w', 'brown', 'dark yellow'), + (0o04, 'b', 'dark blue'), + (0o05, 'm', 'dark magenta', 'dark purple'), + (0o06, 'c', 'dark cyan'), + (0o07, 'n', 'light grey', 'light gray', 'neutral', 'dark white'), + (0o10, 'B', 'dark grey', 'dark gray', 'light black'), + (0o11, 'R', 'red', 'light red'), + (0o12, 'G', 'green', 'light green'), + (0o13, 'Y', 'yellow', 'light yellow'), + (0o14, 'B', 'blue', 'light blue'), + (0o15, 'M', 'magenta', 'purple', 'light magenta', 'light purple'), + (0o16, 'C', 'cyan', 'light cyan'), + (0o17, 'W', 'white', 'light white'), ) colors_by_num = {} @@ -61,7 +65,7 @@ for colordef in color_defs: for c in nameset: colors_by_name[c] = colorcode -class ColoredChar: +class ColoredChar(object): def __init__(self, c, colorcode): self.c = c self._colorcode = colorcode @@ -76,8 +80,8 @@ class ColoredChar: return getattr(self.c, name) def ansi_color(self): - clr = str(30 + (07 & self._colorcode)) - if self._colorcode & 010: + clr = str(30 + (0o7 & self._colorcode)) + if self._colorcode & 0o10: clr = '1;' + clr return clr @@ -100,11 +104,11 @@ class ColoredChar: def colortag(self): return lookup_letter_from_code(self._colorcode) -class ColoredText: +class ColoredText(object): def __init__(self, source=''): - if isinstance(source, basestring): + if isinstance(source, six.text_type): plain, colors = self.parse_ansi_colors(source) - self.chars = map(ColoredChar, plain, colors) + self.chars = list(map(ColoredChar, plain, colors)) else: # expected that source is an iterable of ColoredChars (or duck-typed as such) self.chars = tuple(source) @@ -149,7 +153,7 @@ class ColoredText: @staticmethod def parse_sgr_param(curclr, paramstr): oldclr = curclr - args = map(int, paramstr.split(';')) + args = list(map(int, paramstr.split(';'))) for a in args: if a == 0: curclr = lookup_colorcode('neutral') diff --git a/pylib/cqlshlib/test/basecase.py b/pylib/cqlshlib/test/basecase.py index d3937690ed..f3985113b3 100644 --- a/pylib/cqlshlib/test/basecase.py +++ b/pylib/cqlshlib/test/basecase.py @@ -17,7 +17,7 @@ import os import sys import logging -from itertools import izip + from os.path import dirname, join, normpath cqlshlog = logging.getLogger('test_cqlsh') @@ -46,13 +46,27 @@ TEST_PORT = int(os.environ.get('CQL_TEST_PORT', 9042)) class BaseTestCase(unittest.TestCase): def assertNicelyFormattedTableHeader(self, line, msg=None): - return self.assertRegexpMatches(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg) + return self.assertRegex(line, r'^ +\w+( +\| \w+)*\s*$', msg=msg) def assertNicelyFormattedTableRule(self, line, msg=None): - return self.assertRegexpMatches(line, r'^-+(\+-+)*\s*$', msg=msg) + return self.assertRegex(line, r'^-+(\+-+)*\s*$', msg=msg) def assertNicelyFormattedTableData(self, line, msg=None): - return self.assertRegexpMatches(line, r'^ .* \| ', msg=msg) + return self.assertRegex(line, r'^ .* \| ', msg=msg) + + def assertRegex(self, text, regex, msg=None): + """Call assertRegexpMatches() if in Python 2""" + if hasattr(unittest.TestCase, 'assertRegex'): + return super().assertRegex(text, regex, msg) + else: + return self.assertRegexpMatches(text, regex, msg) + + def assertNotRegex(self, text, regex, msg=None): + """Call assertNotRegexpMatches() if in Python 2""" + if hasattr(unittest.TestCase, 'assertNotRegex'): + return super().assertNotRegex(text, regex, msg) + else: + return self.assertNotRegexpMatches(text, regex, msg) def dedent(s): lines = [ln.rstrip() for ln in s.splitlines()] @@ -63,4 +77,4 @@ def dedent(s): return '\n'.join(line[minspace:] for line in lines) def at_a_time(i, num): - return izip(*([iter(i)] * num)) + return zip(*([iter(i)] * num)) diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py index 501850c787..c9571f2bcc 100644 --- a/pylib/cqlshlib/test/cassconnect.py +++ b/pylib/cqlshlib/test/cassconnect.py @@ -14,12 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import with_statement import contextlib -import tempfile +import io import os.path -from .basecase import cql, cqlsh, cqlshlog, TEST_HOST, TEST_PORT, rundir, policy, quote_name +import tempfile + +from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, rundir from .run_cqlsh import run_cqlsh, call_cqlsh test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql') @@ -43,12 +44,12 @@ def make_ks_name(): return os.path.basename(tempfile.mktemp(prefix='CqlshTests_')) def create_keyspace(cursor): - ksname = make_ks_name() + ksname = make_ks_name().lower() qksname = quote_name(ksname) cursor.execute(''' CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; - ''' % quote_name(ksname)) + ''' % qksname) cursor.execute('USE %s;' % qksname) TEST_KEYSPACES_CREATED.append(ksname) return ksname @@ -63,11 +64,11 @@ def split_cql_commands(source): def execute_cql_commands(cursor, source, logprefix='INIT: '): for cql in split_cql_commands(source): - cqlshlog.debug(logprefix + cql) + cqlshlog.debug((logprefix + cql).encode("utf-8")) cursor.execute(cql) def execute_cql_file(cursor, fname): - with open(fname) as f: + with io.open(fname, "r", encoding="utf-8") as f: return execute_cql_commands(cursor, f.read()) def create_db(): @@ -116,8 +117,6 @@ def cassandra_cursor(cql_version=None, ks=''): c = conn.connect(ks) # increase default timeout to fix flacky tests, see CASSANDRA-12481 c.default_timeout = 60.0 - # if ks is not None: - # c.execute('USE %s;' % quote_name(c, ks)) yield c finally: conn.shutdown() @@ -137,4 +136,6 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): def testcall_cqlsh(keyspace=None, **kwargs): if keyspace is None: keyspace = get_keyspace() + if ('input' in kwargs.keys() and isinstance(kwargs['input'], str)): + kwargs['input'] = kwargs['input'].encode('utf-8') return call_cqlsh(keyspace=keyspace, **kwargs) diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py index fa010fef3e..1c90d58f31 100644 --- a/pylib/cqlshlib/test/run_cqlsh.py +++ b/pylib/cqlshlib/test/run_cqlsh.py @@ -16,6 +16,8 @@ # NOTE: this testing tool is *nix specific +from __future__ import unicode_literals + import os import sys import re @@ -32,7 +34,7 @@ def is_win(): return sys.platform in ("cygwin", "win32") if is_win(): - from winpty import WinPty + from .winpty import WinPty DEFAULT_PREFIX = '' else: import pty @@ -41,6 +43,24 @@ else: DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> ' DEFAULT_CQLSH_TERM = 'xterm' +def get_smm_sequence(term='xterm'): + """ + Return the set meta mode (smm) sequence, if any. + On more recent Linux systems, xterm emits the smm sequence + before each prompt. + """ + result = '' + if not is_win(): + tput_proc = subprocess.Popen(['tput', '-T{}'.format(term), 'smm'], stdout=subprocess.PIPE) + tput_stdout = tput_proc.communicate()[0] + if (tput_stdout and (tput_stdout != b'')): + result = tput_stdout + if isinstance(result, bytes): + result = result.decode("utf-8") + return result + +DEFAULT_SMM_SEQUENCE = get_smm_sequence() + cqlshlog = basecase.cqlshlog def set_controlling_pty(master, slave): @@ -103,7 +123,7 @@ def timing_out_alarm(seconds): if is_win(): try: import eventlet - except ImportError, e: + except ImportError as e: sys.exit("evenlet library required to run cqlshlib tests on Windows") def timing_out(seconds): @@ -171,22 +191,33 @@ class ProcRunner: return self.proc.wait() def send_tty(self, data): + if not isinstance(data, bytes): + data = data.encode("utf-8") os.write(self.childpty, data) def send_pipe(self, data): self.proc.stdin.write(data) def read_tty(self, blksize, timeout=None): - return os.read(self.childpty, blksize) + buf = os.read(self.childpty, blksize) + if isinstance(buf, bytes): + buf = buf.decode("utf-8") + return buf def read_pipe(self, blksize, timeout=None): - return self.proc.stdout.read(blksize) + buf = self.proc.stdout.read(blksize) + if isinstance(buf, bytes): + buf = buf.decode("utf-8") + return buf def read_winpty(self, blksize, timeout=None): - return self.winpty.read(blksize, timeout) + buf = self.winpty.read(blksize, timeout) + if isinstance(buf, bytes): + buf = buf.decode("utf-8") + return buf def read_until(self, until, blksize=4096, timeout=None, - flags=0, ptty_timeout=None): + flags=0, ptty_timeout=None, replace=[]): if not isinstance(until, re._pattern_type): until = re.compile(until, flags) @@ -196,6 +227,9 @@ class ProcRunner: with timing_out(timeout): while True: val = self.read(blksize, ptty_timeout) + for replace_target in replace: + if (replace_target != ''): + val = val.replace(replace_target, '') cqlshlog.debug("read %r from subproc" % (val,)) if val == '': raise EOFError("'until' pattern %r not found" % (until.pattern,)) @@ -252,16 +286,21 @@ class CqlshRunner(ProcRunner): env.setdefault('TERM', 'xterm') env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', '')) env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', '')) + coverage = False + if ('CQLSH_COVERAGE' in env.keys()): + coverage = True args = tuple(args) + (host, str(port)) if cqlver is not None: args += ('--cqlversion', str(cqlver)) if keyspace is not None: - args += ('--keyspace', keyspace) + args += ('--keyspace', keyspace.lower()) if tty and is_win(): args += ('--tty',) args += ('--encoding', 'utf-8') if win_force_colors: args += ('--color',) + if coverage: + args += ('--coverage',) self.keyspace = keyspace ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs) self.prompt = prompt @@ -270,8 +309,8 @@ class CqlshRunner(ProcRunner): else: self.output_header = self.read_to_next_prompt() - def read_to_next_prompt(self): - return self.read_until(self.prompt, timeout=10.0, ptty_timeout=3) + def read_to_next_prompt(self, timeout=10.0): + return self.read_until(self.prompt, timeout=timeout, ptty_timeout=3, replace=[DEFAULT_SMM_SEQUENCE,]) def read_up_to_timeout(self, timeout, blksize=4096): output = ProcRunner.read_up_to_timeout(self, timeout, blksize=blksize) @@ -309,4 +348,6 @@ def call_cqlsh(**kwargs): c = CqlshRunner(**kwargs) output, _ = c.proc.communicate(proginput) result = c.close() + if isinstance(output, bytes): + output = output.decode("utf-8") return output, result diff --git a/pylib/cqlshlib/test/test_copyutil.py b/pylib/cqlshlib/test/test_copyutil.py new file mode 100644 index 0000000000..18b167adbf --- /dev/null +++ b/pylib/cqlshlib/test/test_copyutil.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# to configure behavior, define $CQL_TEST_HOST to the destination address +# and $CQL_TEST_PORT to the associated port. + + +import unittest + +from cassandra.metadata import MIN_LONG, Murmur3Token, TokenMap +from cassandra.policies import SimpleConvictionPolicy +from cassandra.pool import Host +from unittest.mock import Mock + +from cqlshlib.copyutil import ExportTask + + +class CopyTaskTest(unittest.TestCase): + + def setUp(self): + # set up default test data + self.ks = 'testks' + self.table = 'testtable' + self.columns = ['a', 'b'] + self.fname = 'test_fname' + self.opts = {} + self.protocol_version = 0 + self.config_file = 'test_config' + self.hosts = [ + Host('10.0.0.1', SimpleConvictionPolicy, 9000), + Host('10.0.0.2', SimpleConvictionPolicy, 9000), + Host('10.0.0.3', SimpleConvictionPolicy, 9000), + Host('10.0.0.4', SimpleConvictionPolicy, 9000) + ] + + def mock_shell(self): + """ + Set up a mock Shell so we can unit test ExportTask internals + """ + shell = Mock() + shell.conn = Mock() + shell.conn.get_control_connection_host.return_value = self.hosts[0] + shell.get_column_names.return_value = self.columns + shell.debug = False + return shell + + +class TestExportTask(CopyTaskTest): + + def _test_get_ranges_murmur3_base(self, opts, expected_ranges): + """ + Set up a mock shell with a simple token map to test the ExportTask get_ranges function. + """ + shell = self.mock_shell() + shell.conn.metadata.partitioner = 'Murmur3Partitioner' + # token range for a cluster of 4 nodes with replication factor 3 + shell.get_ring.return_value = { + Murmur3Token(-9223372036854775808): self.hosts[0:3], + Murmur3Token(-4611686018427387904): self.hosts[1:4], + Murmur3Token(0): [self.hosts[2], self.hosts[3], self.hosts[0]], + Murmur3Token(4611686018427387904): [self.hosts[3], self.hosts[0], self.hosts[1]] + } + # merge override options with standard options + overridden_opts = dict(self.opts) + for k,v in opts.items(): + overridden_opts[k] = v + export_task = ExportTask(shell, self.ks, self.table, self.columns, self.fname, overridden_opts, self.protocol_version, self.config_file) + assert export_task.get_ranges() == expected_ranges + + def test_get_ranges_murmur3(self): + """ + Test behavior of ExportTask internal get_ranges function + """ + + # return empty dict and print error if begin_token < min_token + self._test_get_ranges_murmur3_base({'begintoken': MIN_LONG - 1}, {}) + + # return empty dict and print error if begin_token < min_token + self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': -1}, {}) + + # simple case of a single range + expected_ranges = {(1,2): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1}} + self._test_get_ranges_murmur3_base({'begintoken': 1, 'endtoken': 2}, expected_ranges) + + # simple case of two contiguous ranges + expected_ranges = { + (-4611686018427387903,0): {'hosts': ('10.0.0.3', '10.0.0.4', '10.0.0.1'), 'attempts': 0, 'rows': 0, 'workerno': -1}, + (0,1): {'hosts': ('10.0.0.4', '10.0.0.1', '10.0.0.2'), 'attempts': 0, 'rows': 0, 'workerno': -1} + } + self._test_get_ranges_murmur3_base({'begintoken': -4611686018427387903, 'endtoken': 1}, expected_ranges) + + # specify a begintoken only (endtoken defaults to None) + expected_ranges = { + (4611686018427387905,None): {'hosts': ('10.0.0.1', '10.0.0.2', '10.0.0.3'), 'attempts': 0, 'rows': 0, 'workerno': -1} + } + self._test_get_ranges_murmur3_base({'begintoken': 4611686018427387905}, expected_ranges) + + # specify an endtoken only (begintoken defaults to None) + expected_ranges = { + (None, MIN_LONG + 1): {'hosts': ('10.0.0.2', '10.0.0.3', '10.0.0.4'), 'attempts': 0, 'rows': 0, 'workerno': -1} + } + self._test_get_ranges_murmur3_base({'endtoken': MIN_LONG + 1}, expected_ranges) + diff --git a/pylib/cqlshlib/test/test_cql_parsing.py b/pylib/cqlshlib/test/test_cql_parsing.py index 7376fbf38e..10be99fe03 100644 --- a/pylib/cqlshlib/test/test_cql_parsing.py +++ b/pylib/cqlshlib/test/test_cql_parsing.py @@ -20,7 +20,7 @@ from unittest import TestCase from operator import itemgetter -from ..cql3handling import CqlRuleSet +from cqlshlib.cql3handling import CqlRuleSet class TestCqlParsing(TestCase): diff --git a/pylib/cqlshlib/test/test_cqlsh_commands.py b/pylib/cqlshlib/test/test_cqlsh_commands.py deleted file mode 100644 index e4fe0bce96..0000000000 --- a/pylib/cqlshlib/test/test_cqlsh_commands.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# to configure behavior, define $CQL_TEST_HOST to the destination address -# and $CQL_TEST_PORT to the associated port. - -from .basecase import BaseTestCase, cqlsh - -class TestCqlshCommands(BaseTestCase): - def setUp(self): - pass - - def tearDown(self): - pass - - def test_show(self): - pass - - def test_describe(self): - pass - - def test_exit(self): - pass - - def test_help(self): - pass diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index db18e3f1c8..a8d6719815 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -17,11 +17,13 @@ # to configure behavior, define $CQL_TEST_HOST to the destination address # and $CQL_TEST_PORT to the associated port. -from __future__ import with_statement +import locale +import os import re -from .basecase import BaseTestCase, cqlsh -from .cassconnect import testrun_cqlsh +from .basecase import BaseTestCase, cqlsh, cqlshlog +from .cassconnect import create_db, remove_db, testrun_cqlsh +from .run_cqlsh import TimeoutError import unittest import sys @@ -41,8 +43,22 @@ completion_separation_re = re.compile(r'\s+') @unittest.skipIf(sys.platform == "win32", 'Tab completion tests not supported on Windows') class CqlshCompletionCase(BaseTestCase): + @classmethod + def setUpClass(cls): + create_db() + + @classmethod + def tearDownClass(cls): + remove_db() + def setUp(self): - self.cqlsh_runner = testrun_cqlsh(cqlver=None, env={'COLUMNS': '100000'}) + env = os.environ + env['COLUMNS'] = '100000' + if (locale.getpreferredencoding() != 'UTF-8'): + env['LC_CTYPE'] = 'en_US.utf8' + if ('PATH' in os.environ.keys()): + env['PATH'] = os.environ['PATH'] + self.cqlsh_runner = testrun_cqlsh(cqlver=None, env=env) self.cqlsh = self.cqlsh_runner.__enter__() def tearDown(self): @@ -81,14 +97,14 @@ class CqlshCompletionCase(BaseTestCase): prompt_regex = self.cqlsh.prompt.lstrip() + re.escape(inputstring) msg = ('Double-tab completion ' 'does not print prompt for input "{}"'.format(inputstring)) - self.assertRegexpMatches(choice_lines[-1], prompt_regex, msg=msg) + self.assertRegex(choice_lines[-1], prompt_regex, msg=msg) choice_lines = [line.strip() for line in choice_lines[:-1]] choice_lines = [line for line in choice_lines if line] if split_completed_lines: - completed_lines = map(set, (completion_separation_re.split(line.strip()) - for line in choice_lines)) + completed_lines = list(map(set, (completion_separation_re.split(line.strip()) + for line in choice_lines))) if not completed_lines: return set() @@ -132,8 +148,14 @@ class CqlshCompletionCase(BaseTestCase): other_choices_ok=other_choices_ok, split_completed_lines=split_completed_lines) finally: - self.cqlsh.send(CTRL_C) # cancel any current line - self.cqlsh.read_to_next_prompt() + try: + self.cqlsh.send(CTRL_C) # cancel any current line + self.cqlsh.read_to_next_prompt(timeout=1.0) + except TimeoutError: + # retry once + self.cqlsh.send(CTRL_C) + self.cqlsh.read_to_next_prompt(timeout=10.0) + def strategies(self): return self.module.CqlRuleSet.replication_strategies @@ -345,8 +367,6 @@ class TestCqlshCompletion(CqlshCompletionCase): choices=[',', 'WHERE']) self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ", choices=['TOKEN(', 'lonelykey']) - self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE ", - choices=['TOKEN(', 'lonelykey']) self.trycompletions("UPDATE empty_table SET lonelycol = 'eggs' WHERE lonel", immediate='ykey ') @@ -392,7 +412,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'twenty_rows_composite_table', 'utf8_with_special_chars', 'system_traces.', 'songs', - '"' + self.cqlsh.keyspace + '".'], + self.cqlsh.keyspace + '.'], other_choices_ok=True) self.trycompletions('DELETE FROM ', @@ -407,7 +427,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'system_traces.', 'songs', 'system_auth.', 'system_distributed.', 'system_schema.', 'system_traces.', - '"' + self.cqlsh.keyspace + '".'], + self.cqlsh.keyspace + '.'], other_choices_ok=True) self.trycompletions('DELETE FROM twenty_rows_composite_table ', choices=['USING', 'WHERE']) @@ -529,13 +549,13 @@ class TestCqlshCompletion(CqlshCompletionCase): self.trycompletions('DROP K', immediate='EYSPACE ') quoted_keyspace = '"' + self.cqlsh.keyspace + '"' self.trycompletions('DROP KEYSPACE ', - choices=['IF', quoted_keyspace]) + choices=['IF', self.cqlsh.keyspace]) self.trycompletions('DROP KEYSPACE ' + quoted_keyspace, choices=[';']) self.trycompletions('DROP KEYSPACE I', - immediate='F EXISTS ' + quoted_keyspace + ';') + immediate='F EXISTS ' + self.cqlsh.keyspace + ' ;') def create_columnfamily_table_template(self, name): """Parameterized test for CREATE COLUMNFAMILY and CREATE TABLE. Since @@ -544,11 +564,11 @@ class TestCqlshCompletion(CqlshCompletionCase): prefix = 'CREATE ' + name + ' ' quoted_keyspace = '"' + self.cqlsh.keyspace + '"' self.trycompletions(prefix + '', - choices=['IF', quoted_keyspace, '']) + choices=['IF', self.cqlsh.keyspace, '']) self.trycompletions(prefix + 'IF ', immediate='NOT EXISTS ') self.trycompletions(prefix + 'IF NOT EXISTS ', - choices=['', quoted_keyspace]) + choices=['', self.cqlsh.keyspace]) self.trycompletions(prefix + 'IF NOT EXISTS new_table ', immediate='( ') @@ -594,7 +614,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH ', choices=['bloom_filter_fp_chance', 'compaction', 'compression', @@ -603,7 +623,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance ', immediate='= ') self.trycompletions(prefix + ' new_table (col_a int PRIMARY KEY) WITH bloom_filter_fp_chance = ', @@ -650,7 +670,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'memtable_flush_period_in_ms', 'CLUSTERING', 'COMPACT', 'caching', 'comment', - 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc']) + 'min_index_interval', 'speculative_retry', 'additional_write_policy', 'cdc', 'read_repair']) self.trycompletions(prefix + " new_table (col_a int PRIMARY KEY) WITH compaction = " + "{'class': 'DateTieredCompactionStrategy', '", choices=['base_time_seconds', 'max_sstable_age_days', @@ -694,7 +714,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'utf8_with_special_chars', 'system_traces.', 'songs', 'system_distributed.', - '"' + self.cqlsh.keyspace + '".'], + self.cqlsh.keyspace + '.'], other_choices_ok=True) self.trycompletions('DESC TYPE ', @@ -717,7 +737,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'fbestsong', 'fmax', 'fmin', - '"' + self.cqlsh.keyspace + '".'], + self.cqlsh.keyspace + '.'], other_choices_ok=True) self.trycompletions('DESC AGGREGATE ', @@ -727,7 +747,7 @@ class TestCqlshCompletion(CqlshCompletionCase): 'system_distributed.', 'aggmin', 'aggmax', - '"' + self.cqlsh.keyspace + '".'], + self.cqlsh.keyspace + '.'], other_choices_ok=True) # Unfortunately these commented tests will not work. This is due to the keyspace name containing quotes; diff --git a/pylib/cqlshlib/test/test_cqlsh_invocation.py b/pylib/cqlshlib/test/test_cqlsh_invocation.py deleted file mode 100644 index 84319805ff..0000000000 --- a/pylib/cqlshlib/test/test_cqlsh_invocation.py +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# to configure behavior, define $CQL_TEST_HOST to the destination address -# and $CQL_TEST_PORT to the associated port. - -from .basecase import BaseTestCase - -class TestCqlshInvocation(BaseTestCase): - def setUp(self): - pass - - def tearDown(self): - pass - - def test_normal_run(self): - pass - - def test_python_interpreter_location(self): - pass - - def test_color_capability_detection(self): - pass - - def test_colored_output(self): - pass - - def test_color_cmdline_option(self): - pass - - def test_debug_option(self): - pass - - def test_connection_args(self): - pass - - def test_connection_config(self): - pass - - def test_connection_envvars(self): - pass - - def test_command_history(self): - pass - - def test_missing_dependencies(self): - pass - - def test_completekey_config(self): - pass - - def test_ctrl_c(self): - pass - - def test_eof(self): - pass - - def test_output_encoding_detection(self): - pass - - def test_output_encoding(self): - pass - - def test_retries(self): - pass diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index fb9fcd6145..d2ee800223 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -17,35 +17,55 @@ # to configure behavior, define $CQL_TEST_HOST to the destination address # and $CQL_TEST_PORT to the associated port. -from __future__ import with_statement +from __future__ import unicode_literals, with_statement +import locale +import os import re -from itertools import izip -from .basecase import (BaseTestCase, cqlshlog, dedent, at_a_time, cqlsh, - TEST_HOST, TEST_PORT) -from .cassconnect import (get_keyspace, testrun_cqlsh, testcall_cqlsh, - cassandra_cursor, split_cql_commands, quote_name) -from .ansi_colors import (ColoredText, lookup_colorcode, lookup_colorname, - lookup_colorletter, ansi_seq) -import unittest import sys +import unittest + +from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT, + at_a_time, cqlsh, cqlshlog, dedent) +from .cassconnect import (cassandra_cursor, create_db, get_keyspace, + quote_name, remove_db, split_cql_commands, + testcall_cqlsh, testrun_cqlsh) +from .ansi_colors import (ColoredText, ansi_seq, lookup_colorcode, + lookup_colorname, lookup_colorletter) CONTROL_C = '\x03' CONTROL_D = '\x04' class TestCqlshOutput(BaseTestCase): + @classmethod + def setUpClass(cls): + create_db() + + @classmethod + def tearDownClass(cls): + remove_db() + def setUp(self): - pass + env = os.environ + env['COLUMNS'] = '100000' + # carry forward or override locale LC_CTYPE for UTF-8 encoding + if (locale.getpreferredencoding() != 'UTF-8'): + env['LC_CTYPE'] = 'en_US.utf8' + else: + env['LC_CTYPE'] = os.environ.get('LC_CTYPE', 'en_US.utf8') + if ('PATH' in os.environ.keys()): + env['PATH'] = os.environ['PATH'] + self.default_env = env def tearDown(self): pass def assertNoHasColors(self, text, msg=None): - self.assertNotRegexpMatches(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text) + self.assertNotRegex(text, ansi_seq, msg='ANSI CSI sequence found in %r' % text) def assertHasColors(self, text, msg=None): - self.assertRegexpMatches(text, ansi_seq, msg=msg) + self.assertRegex(text, ansi_seq, msg=msg) def assertColored(self, coloredtext, colorname): wanted_colorcode = lookup_colorcode(colorname) @@ -57,7 +77,7 @@ class TestCqlshOutput(BaseTestCase): % (coloredtext, num, lookup_colorname(ccolor), colorname)) def assertColorFromTags(self, coloredtext, tags): - for (char, tag) in izip(coloredtext, tags): + for (char, tag) in zip(coloredtext, tags): if char.isspace(): continue if tag.isspace(): @@ -67,12 +87,14 @@ class TestCqlshOutput(BaseTestCase): 'Actually got: %s\ncolor code: %s' % (tags, coloredtext.colored_version(), coloredtext.colortags())) - def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, **kwargs): + def assertQueriesGiveColoredOutput(self, queries_and_expected_outputs, env=None, **kwargs): """ Allow queries and expected output to be specified in structured tuples, along with expected color information. """ - with testrun_cqlsh(tty=True, **kwargs) as c: + if env is None: + env = self.default_env + with testrun_cqlsh(tty=True, env=env, **kwargs) as c: for query, expected in queries_and_expected_outputs: cqlshlog.debug('Testing %r' % (query,)) output = c.cmd_and_response(query).lstrip("\r\n") @@ -83,10 +105,25 @@ class TestCqlshOutput(BaseTestCase): self.assertEqual(outputline.plain().rstrip(), plain) self.assertColorFromTags(outputline, colorcodes) + def strip_read_repair_chance(self, describe_statement): + """ + Remove read_repair_chance and dclocal_read_repair_chance options + from output of DESCRIBE statements. The resulting string may be + reused as a CREATE statement. + Useful after CASSANDRA-13910, which removed read_repair_chance + options from CREATE statements but did not remove them completely + from the system. + """ + describe_statement = re.sub(r"( AND)? (dclocal_)?read_repair_chance = [\d\.]+", "", describe_statement) + describe_statement = re.sub(r"WITH[\s]*;", "", describe_statement) + return describe_statement + def test_no_color_output(self): + env = self.default_env for termname in ('', 'dumb', 'vt100'): cqlshlog.debug('TERM=%r' % termname) - with testrun_cqlsh(tty=True, env={'TERM': termname}, + env['TERM'] = termname + with testrun_cqlsh(tty=True, env=env, win_force_colors=False) as c: c.send('select * from has_all_types;\n') self.assertNoHasColors(c.read_to_next_prompt()) @@ -96,15 +133,17 @@ class TestCqlshOutput(BaseTestCase): self.assertNoHasColors(c.read_to_next_prompt()) def test_no_prompt_or_colors_output(self): + env = self.default_env for termname in ('', 'dumb', 'vt100', 'xterm'): cqlshlog.debug('TERM=%r' % termname) + env['TERM'] = termname query = 'select * from has_all_types limit 1;' - output, result = testcall_cqlsh(prompt=None, env={'TERM': termname}, + output, result = testcall_cqlsh(prompt=None, env=env, tty=False, input=query + '\n') output = output.splitlines() for line in output: self.assertNoHasColors(line) - self.assertNotRegexpMatches(line, r'^cqlsh\S*>') + self.assertNotRegex(line, r'^cqlsh\S*>') self.assertEqual(len(output), 6, msg='output: %r' % '\n'.join(output)) self.assertEqual(output[0], '') @@ -115,9 +154,12 @@ class TestCqlshOutput(BaseTestCase): self.assertEqual(output[5].strip(), '(1 rows)') def test_color_output(self): + env = self.default_env for termname in ('xterm', 'unknown-garbage'): cqlshlog.debug('TERM=%r' % termname) - with testrun_cqlsh(tty=True, env={'TERM': termname}) as c: + env['TERMNAME'] = termname + env['TERM'] = termname + with testrun_cqlsh(tty=True, env=env) as c: c.send('select * from has_all_types;\n') self.assertHasColors(c.read_to_next_prompt()) c.send('select count(*) from has_all_types;\n') @@ -349,6 +391,8 @@ class TestCqlshOutput(BaseTestCase): )) def test_timestamp_output(self): + env = self.default_env + env['TZ'] = 'Etc/UTC' self.assertQueriesGiveColoredOutput(( ('''select timestampcol from has_all_types where num = 0;''', """ timestampcol @@ -362,9 +406,10 @@ class TestCqlshOutput(BaseTestCase): (1 rows) nnnnnnnn """), - ), env={'TZ': 'Etc/UTC'}) + ), env=env) try: import pytz # test only if pytz is available on PYTHONPATH + env['TZ'] = 'America/Sao_Paulo' self.assertQueriesGiveColoredOutput(( ('''select timestampcol from has_all_types where num = 0;''', """ timestampcol @@ -378,7 +423,7 @@ class TestCqlshOutput(BaseTestCase): (1 rows) nnnnnnnn """), - ), env={'TZ': 'America/Sao_Paulo'}) + ), env=env) except ImportError: pass @@ -470,8 +515,10 @@ class TestCqlshOutput(BaseTestCase): # terminals, but the color-checking machinery here will still treat # it as one character, so those won't seem to line up visually either. + env = self.default_env + env['LANG'] = 'en_US.UTF-8' self.assertQueriesGiveColoredOutput(( - ("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", u""" + ("select * from utf8_with_special_chars where k in (0, 1, 2, 3, 4, 5, 6);", """ k | val R MMM ---+------------------------------- @@ -494,8 +541,8 @@ class TestCqlshOutput(BaseTestCase): (7 rows) nnnnnnnn - """.encode('utf-8')), - ), env={'LANG': 'en_US.UTF-8'}) + """), + ), env=env) def test_blob_output(self): self.assertQueriesGiveColoredOutput(( @@ -516,11 +563,11 @@ class TestCqlshOutput(BaseTestCase): (4 rows) nnnnnnnn - """), + """, ), )) def test_prompt(self): - with testrun_cqlsh(tty=True, keyspace=None) as c: + with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c: self.assertTrue(c.output_header.splitlines()[-1].endswith('cqlsh> ')) c.send('\n') @@ -552,7 +599,7 @@ class TestCqlshOutput(BaseTestCase): "RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR") def test_describe_keyspace_output(self): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: ks = get_keyspace() qks = quote_name(ks) for cmd in ('describe keyspace', 'desc keyspace'): @@ -572,27 +619,27 @@ class TestCqlshOutput(BaseTestCase): with cassandra_cursor() as curs: try: for stmt in statements: + stmt = self.strip_read_repair_chance(stmt) cqlshlog.debug('TEST EXEC: %s' % stmt) curs.execute(stmt) finally: curs.execute('use system') if do_drop: - curs.execute('drop keyspace %s' % quote_name(new_ks_name)) + curs.execute('drop keyspace {}'.format(new_ks_name)) def check_describe_keyspace_output(self, output, qksname): expected_bits = [r'(?im)^CREATE KEYSPACE %s WITH\b' % re.escape(qksname), r';\s*$', r'\breplication = {\'class\':'] for expr in expected_bits: - self.assertRegexpMatches(output, expr) + self.assertRegex(output, expr) def test_describe_columnfamily_output(self): # we can change these to regular expressions if/when it makes sense # to do so; these will likely be subject to lots of adjustments. - + # note columns are now comparator-ordered instead of original-order. table_desc3 = dedent(""" - CREATE TABLE %s.has_all_types ( num int PRIMARY KEY, asciicol ascii, @@ -615,37 +662,35 @@ class TestCqlshOutput(BaseTestCase): AND cdc = false AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} - AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.0 AND default_time_to_live = 0 AND gc_grace_seconds = 864000 AND max_index_interval = 2048 AND memtable_flush_period_in_ms = 0 AND min_index_interval = 128 - AND speculative_retry = '99PERCENTILE' - AND additional_write_policy = '99PERCENTILE'; + AND read_repair_chance = 0.0 + AND speculative_retry = '99p'; + """ % quote_name(get_keyspace())) - """ % quote_name(get_keyspace())) - - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: for cmdword in ('describe table', 'desc columnfamily'): for semicolon in (';', ''): output = c.cmd_and_response('%s has_all_types%s' % (cmdword, semicolon)) self.assertNoHasColors(output) - self.assertSequenceEqual(output.split('\n'), table_desc3.split('\n')) + self.assertSequenceEqual(dedent(output).split('\n'), table_desc3.split('\n')) def test_describe_columnfamilies_output(self): output_re = r''' - \n - Keyspace [ ] (?P \S+ ) \n + \n? Keyspace [ ] (?P \S+ ) \n -----------* \n - (?P .*? ) - \n - ''' + (?P ( ( ["']?\w+["']? [^\w\n]* )+ \n )* ) + ''' ks = get_keyspace() - with testrun_cqlsh(tty=True, keyspace=None) as c: + with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c: # when not in a keyspace for cmdword in ('DESCRIBE COLUMNFAMILIES', 'desc tables'): @@ -653,7 +698,7 @@ class TestCqlshOutput(BaseTestCase): ksnames = [] output = c.cmd_and_response(cmdword + semicolon) self.assertNoHasColors(output) - self.assertRegexpMatches(output, '(?xs) ^ ( %s )+ $' % output_re) + self.assertRegex(output, '(?xs) ^ ( %s )+ $' % output_re) for section in re.finditer('(?xs)' + output_re, output): ksname = section.group('ksname') @@ -696,13 +741,13 @@ class TestCqlshOutput(BaseTestCase): \n ''' - with testrun_cqlsh(tty=True, keyspace=None) as c: + with testrun_cqlsh(tty=True, keyspace=None, env=self.default_env) as c: # not in a keyspace for semicolon in ('', ';'): output = c.cmd_and_response('describe cluster' + semicolon) self.assertNoHasColors(output) - self.assertRegexpMatches(output, output_re + '$') + self.assertRegex(output, output_re + '$') c.send('USE %s;\n' % quote_name(get_keyspace())) c.read_to_next_prompt() @@ -710,32 +755,32 @@ class TestCqlshOutput(BaseTestCase): for semicolon in ('', ';'): output = c.cmd_and_response('describe cluster' + semicolon) self.assertNoHasColors(output) - self.assertRegexpMatches(output, output_re + ringinfo_re + '$') + self.assertRegex(output, output_re + ringinfo_re + '$') def test_describe_schema_output(self): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: for semicolon in ('', ';'): output = c.cmd_and_response('desc full schema' + semicolon) self.assertNoHasColors(output) - self.assertRegexpMatches(output, '^\nCREATE KEYSPACE') + self.assertRegex(output, '^\nCREATE KEYSPACE') self.assertIn("\nCREATE KEYSPACE system WITH replication = {'class': 'LocalStrategy'} AND durable_writes = true;\n", output) - self.assertRegexpMatches(output, ';\s*$') + self.assertRegex(output, ';\s*$') def test_show_output(self): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: output = c.cmd_and_response('show version;') - self.assertRegexpMatches(output, + self.assertRegex(output, '^\[cqlsh \S+ \| Cassandra \S+ \| CQL spec \S+ \| Native protocol \S+\]$') output = c.cmd_and_response('show host;') self.assertHasColors(output) - self.assertRegexpMatches(output, '^Connected to .* at %s:%d\.$' + self.assertRegex(output, '^Connected to .* at %s:%d\.$' % (re.escape(TEST_HOST), TEST_PORT)) @unittest.skipIf(sys.platform == "win32", 'EOF signaling not supported on Windows') def test_eof_prints_newline(self): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: c.send(CONTROL_D) out = c.read_lines(1)[0].replace('\r', '') self.assertEqual(out, '\n') @@ -745,7 +790,7 @@ class TestCqlshOutput(BaseTestCase): def test_exit_prints_no_newline(self): for semicolon in ('', ';'): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: cmd = 'exit%s\n' % semicolon c.send(cmd) if c.realtty: @@ -756,7 +801,7 @@ class TestCqlshOutput(BaseTestCase): self.assertIn(type(cm.exception), (EOFError, OSError)) def test_help_types(self): - with testrun_cqlsh(tty=True) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: c.cmd_and_response('help types') def test_help(self): @@ -800,6 +845,7 @@ class TestCqlshOutput(BaseTestCase): nnnnnnnn """), )) + self.assertQueriesGiveColoredOutput(( ("select phone_numbers from users;", r""" phone_numbers @@ -832,6 +878,7 @@ class TestCqlshOutput(BaseTestCase): nnnnnnnn """), )) + self.assertQueriesGiveColoredOutput(( ("select tags from songs;", r""" tags diff --git a/pylib/cqlshlib/test/test_cqlsh_parsing.py b/pylib/cqlshlib/test/test_cqlsh_parsing.py deleted file mode 100644 index 9c37dd948c..0000000000 --- a/pylib/cqlshlib/test/test_cqlsh_parsing.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# to configure behavior, define $CQL_TEST_HOST to the destination address -# and $CQL_TEST_PORT to the associated port. - -from unittest import TestCase - - -class TestCqlshParsing(TestCase): - def test_describe(self): - pass - diff --git a/pylib/cqlshlib/test/winpty.py b/pylib/cqlshlib/test/winpty.py index 0db9ec31dd..f197aa5b45 100644 --- a/pylib/cqlshlib/test/winpty.py +++ b/pylib/cqlshlib/test/winpty.py @@ -15,11 +15,11 @@ # limitations under the License. from threading import Thread -from cStringIO import StringIO -from Queue import Queue, Empty +from six import StringIO +from six.moves.queue import Queue, Empty -class WinPty: +class WinPty(object): def __init__(self, stdin): self._s = stdin @@ -47,4 +47,4 @@ class WinPty: count = count + 1 except Empty: pass - return buf.getvalue() + return buf.getvalue() \ No newline at end of file diff --git a/pylib/cqlshlib/tracing.py b/pylib/cqlshlib/tracing.py index 26e228b222..14a1e951b5 100644 --- a/pylib/cqlshlib/tracing.py +++ b/pylib/cqlshlib/tracing.py @@ -14,11 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from cqlshlib.displaying import MAGENTA from datetime import datetime, timedelta -from formatting import CqlType import time + from cassandra.query import QueryTrace, TraceUnavailable +from cqlshlib.displaying import MAGENTA +from cqlshlib.formatting import CqlType def print_trace_session(shell, session, session_id, partial_session=False): @@ -45,8 +46,8 @@ def print_trace(shell, trace): return names = ['activity', 'timestamp', 'source', 'source_elapsed', 'client'] - formatted_names = map(shell.myformat_colname, names) - formatted_values = [map(shell.myformat_value, row) for row in rows] + formatted_names = list(map(shell.myformat_colname, names)) + formatted_values = [list(map(shell.myformat_value, row)) for row in rows] shell.writeresult('') shell.writeresult('Tracing session: ', color=MAGENTA, newline=False) diff --git a/pylib/cqlshlib/util.py b/pylib/cqlshlib/util.py index 3ee128dba6..82a332f85a 100644 --- a/pylib/cqlshlib/util.py +++ b/pylib/cqlshlib/util.py @@ -19,9 +19,9 @@ import cProfile import codecs import pstats -from itertools import izip + from datetime import timedelta, tzinfo -from StringIO import StringIO +from six import StringIO try: from line_profiler import LineProfiler @@ -77,7 +77,7 @@ def find_common_prefix(strs): """ common = [] - for cgroup in izip(*strs): + for cgroup in zip(*strs): if all(x == cgroup[0] for x in cgroup[1:]): common.append(cgroup[0]) else: @@ -161,6 +161,6 @@ def profile_off(pr, file_name): ret = s.getvalue() if file_name: with open(file_name, 'w') as f: - print "Writing to %s\n" % (f.name, ) + print("Writing to %s\n" % (f.name, )) f.write(ret) return ret diff --git a/pylib/cqlshlib/wcwidth.py b/pylib/cqlshlib/wcwidth.py index 985fd41acc..0be3af215b 100644 --- a/pylib/cqlshlib/wcwidth.py +++ b/pylib/cqlshlib/wcwidth.py @@ -85,7 +85,7 @@ def bisearch(ucs, table): if ucs < table[0][0] or ucs > table[max][1]: return 0 while max >= min: - mid = (min + max) / 2 + mid = int((min + max) / 2) if ucs > table[mid][1]: min = mid + 1 elif ucs < table[mid][0]: @@ -251,20 +251,21 @@ def mk_wcwidth(ucs): # if we arrive here, ucs is not a combining or C0/C1 control character - return 1 + \ - int(ucs >= 0x1100 and - (ucs <= 0x115f or # Hangul Jamo init. consonants - ucs == 0x2329 or ucs == 0x232a or - (ucs >= 0x2e80 and ucs <= 0xa4cf and - ucs != 0x303f) or # CJK ... Yi - (ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables - (ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs - (ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms - (ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms - (ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms - (ucs >= 0xffe0 and ucs <= 0xffe6) or - (ucs >= 0x20000 and ucs <= 0x2fffd) or - (ucs >= 0x30000 and ucs <= 0x3fffd))) + return 1 + int( + ucs >= 0x1100 + and (ucs <= 0x115f # Hangul Jamo init. consonants + or ucs == 0x2329 or ucs == 0x232a + or (ucs >= 0x2e80 and ucs <= 0xa4cf + and ucs != 0x303f) # CJK ... Yi + or (ucs >= 0xac00 and ucs <= 0xd7a3) # Hangul Syllables + or (ucs >= 0xf900 and ucs <= 0xfaff) # CJK Compatibility Ideographs + or (ucs >= 0xfe10 and ucs <= 0xfe19) # Vertical forms + or (ucs >= 0xfe30 and ucs <= 0xfe6f) # CJK Compatibility Forms + or (ucs >= 0xff00 and ucs <= 0xff60) # Fullwidth Forms + or (ucs >= 0xffe0 and ucs <= 0xffe6) + or (ucs >= 0x20000 and ucs <= 0x2fffd) + or (ucs >= 0x30000 and ucs <= 0x3fffd)) + ) def mk_wcswidth(pwcs): @@ -313,7 +314,7 @@ def wcwidth(c): def wcswidth(s): - return mk_wcswidth(map(ord, s)) + return mk_wcswidth(list(map(ord, s))) def wcwidth_cjk(c): @@ -321,7 +322,7 @@ def wcwidth_cjk(c): def wcswidth_cjk(s): - return mk_wcswidth_cjk(map(ord, s)) + return mk_wcswidth_cjk(list(map(ord, s))) if __name__ == "__main__": @@ -341,7 +342,7 @@ if __name__ == "__main__": ('COMBINING PALATALIZED HOOK BELOW', 0), ('COMBINING GRAVE ACCENT', 0), ) - nonprinting = u'\r\n\t\a\b\f\v\x7f' + nonprinting = '\r\n\t\a\b\f\v\x7f' import unicodedata @@ -366,13 +367,13 @@ if __name__ == "__main__": assert mk_wcwidth(0x10ffff) == 1 assert mk_wcwidth(0x3fffd) == 2 - teststr = u'B\0ig br\u00f8wn moose\ub143\u200b' + teststr = 'B\0ig br\u00f8wn moose\ub143\u200b' calculatedwidth = wcswidth(teststr) assert calculatedwidth == 17, 'expected 17, got %d' % calculatedwidth calculatedwidth = wcswidth_cjk(teststr) assert calculatedwidth == 18, 'expected 18, got %d' % calculatedwidth - assert wcswidth(u'foobar\u200b\a') < 0 + assert wcswidth('foobar\u200b\a') < 0 - print 'tests pass.' + print('tests pass.') diff --git a/pylib/requirements.txt b/pylib/requirements.txt index a9b62177f5..cdfa566a74 100644 --- a/pylib/requirements.txt +++ b/pylib/requirements.txt @@ -2,10 +2,11 @@ # cythonizing the driver, perhaps only on old pips. # http://datastax.github.io/python-driver/installation.html#cython-based-extensions futures -six +six>=0.12.0 -e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver # Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases. -e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm +coverage cql decorator docopt @@ -17,5 +18,4 @@ nose-test-select parse pycodestyle psutil -pycassa thrift==0.9.3