From db3583318258b14b27abe1529f8893da2ff22bf5 Mon Sep 17 00:00:00 2001 From: Brad Schoening <5796692+bschoening@users.noreply.github.com> Date: Fri, 11 Feb 2022 20:35:18 -0500 Subject: [PATCH] update Python test framework from nose to pytest patch by Brad Schoening; reviewed by Brandon Williams, Berenguer Blasi and Stefan Miklosovic for CASSANDRA-17293 --- CHANGES.txt | 1 + pylib/README.asc | 13 ++++++++++- pylib/cassandra-cqlsh-tests.sh | 16 ++------------ pylib/cqlshlib/saferscanner.py | 21 +----------------- pylib/cqlshlib/setup.cfg | 4 ---- pylib/cqlshlib/sslhandling.py | 2 +- pylib/cqlshlib/test/basecase.py | 8 ++----- pylib/cqlshlib/test/cassconnect.py | 23 ++++++++++++++------ pylib/cqlshlib/test/test_cqlsh_completion.py | 6 ++--- pylib/cqlshlib/test/test_cqlsh_output.py | 4 +--- pylib/cqlshlib/test/test_sslhandling.py | 10 ++++----- pylib/requirements.txt | 5 ++--- 12 files changed, 46 insertions(+), 67 deletions(-) delete mode 100644 pylib/cqlshlib/setup.cfg diff --git a/CHANGES.txt b/CHANGES.txt index ab0ff4d35a..9692f814eb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * update Python test framework from nose to pytest (CASSANDRA-17293) * Fix improper CDC commit log segments deletion in non-blocking mode (CASSANDRA-17233) * Add support for string concatenations through the + operator (CASSANDRA-17190) * Limit the maximum hints size per host (CASSANDRA-17142) diff --git a/pylib/README.asc b/pylib/README.asc index a192393533..53c40ea4b0 100644 --- a/pylib/README.asc +++ b/pylib/README.asc @@ -9,7 +9,18 @@ This directory contains code primarily for cqlsh. cqlsh uses cqlshlib in this di == Running tests -In order to run tests for cqlshlib, run cassandra-cqlsh-tests.sh in this directory. It will +The following environment variables can be configured for the database connection - + +. CQL_TEST_HOST, default 127.0.0.1 +. CQL_TEST_PORT, default 9042 +. CQL_TEST_USER, default 'cassandra' +. CQL_TEST_PWD + +You can run tests with a local Cassandra server simply by - + + $ pytest + +In order to run tests in a virtual environment for cqlshlib, run cassandra-cqlsh-tests.sh in this directory. It will automatically setup a virtualenv with the appropriate version of Python and run tests inside it. There are Dockerfiles that can be used to test whether cqlsh works with a default, barebones diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh index b147194231..591094fb68 100755 --- a/pylib/cassandra-cqlsh-tests.sh +++ b/pylib/cassandra-cqlsh-tests.sh @@ -24,21 +24,13 @@ ################################ WORKSPACE=$1 -PYTHON_VERSION=$2 if [ "${WORKSPACE}" = "" ]; then echo "Specify Cassandra source directory" exit fi -if [ "${PYTHON_VERSION}" = "" ]; then - PYTHON_VERSION=python3 -fi - -if [ "${PYTHON_VERSION}" != "python3" -a "${PYTHON_VERSION}" != "python2" ]; then - echo "Specify Python version python3 or python2" - exit -fi +PYTHON_VERSION=python3 export PYTHONIOENCODING="utf-8" export PYTHONUNBUFFERED=true @@ -126,14 +118,10 @@ ccm start --wait-for-binary-proto cd ${CASSANDRA_DIR}/pylib/cqlshlib/ set +e # disable immediate exit from this point -nosetests +pytest RETURN="$?" ccm remove -# hack around --xunit-prefix-with-testsuite-name not being available in nose 1.3.7 -sed -i "s/testsuite name=\"nosetests\"/testsuite name=\"${TESTSUITE_NAME}\"/g" nosetests.xml -sed -i "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" nosetests.xml -mv nosetests.xml ${WORKSPACE}/cqlshlib.xml ################################ # diff --git a/pylib/cqlshlib/saferscanner.py b/pylib/cqlshlib/saferscanner.py index 8949321dff..3cc343033a 100644 --- a/pylib/cqlshlib/saferscanner.py +++ b/pylib/cqlshlib/saferscanner.py @@ -19,7 +19,6 @@ # regex in-pattern flags. Any of those can break correct operation of Scanner. import re -import six from sre_constants import BRANCH, SUBPATTERN, GROUPREF, GROUPREF_IGNORE, GROUPREF_EXISTS from sys import version_info @@ -50,23 +49,6 @@ class SaferScannerBase(re.Scanner): return re.sre_parse.SubPattern(sub.pattern, scrubbedsub) -class Py2SaferScanner(SaferScannerBase): - - def __init__(self, lexicon, flags=0): - self.lexicon = lexicon - p = [] - s = re.sre_parse.Pattern() - s.flags = flags - for phrase, action in lexicon: - p.append(re.sre_parse.SubPattern(s, [ - (SUBPATTERN, (len(p) + 1, self.subpat(phrase, flags))), - ])) - s.groups = len(p) + 1 - p = re.sre_parse.SubPattern(s, [(BRANCH, (None, p))]) - self.p = p - self.scanner = re.sre_compile.compile(p) - - class Py36SaferScanner(SaferScannerBase): def __init__(self, lexicon, flags=0): @@ -99,5 +81,4 @@ class Py38SaferScanner(SaferScannerBase): self.scanner = re.sre_compile.compile(p) -SaferScanner = Py36SaferScanner if six.PY3 else Py2SaferScanner -SaferScanner = Py38SaferScanner if version_info >= (3, 8) else SaferScanner +SaferScanner = Py38SaferScanner if version_info >= (3, 8) else Py36SaferScanner diff --git a/pylib/cqlshlib/setup.cfg b/pylib/cqlshlib/setup.cfg deleted file mode 100644 index 6c523ee93e..0000000000 --- a/pylib/cqlshlib/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[nosetests] -verbosity=3 -detailed-errors=1 -with-xunit=1 diff --git a/pylib/cqlshlib/sslhandling.py b/pylib/cqlshlib/sslhandling.py index 8a7592c117..2981396253 100644 --- a/pylib/cqlshlib/sslhandling.py +++ b/pylib/cqlshlib/sslhandling.py @@ -58,7 +58,7 @@ def ssl_settings(host, config_file, env=os.environ): for protocol in ['PROTOCOL_TLS', 'PROTOCOL_TLSv1_2', 'PROTOCOL_TLSv1_1', 'PROTOCOL_TLSv1']: if hasattr(ssl, protocol): return getattr(ssl, protocol) - return None + return ssl.PROTOCOL_TLS ssl_validate = env.get('SSL_VALIDATE') if ssl_validate is None: diff --git a/pylib/cqlshlib/test/basecase.py b/pylib/cqlshlib/test/basecase.py index 096853fe5d..ce4fd5ca7a 100644 --- a/pylib/cqlshlib/test/basecase.py +++ b/pylib/cqlshlib/test/basecase.py @@ -29,14 +29,10 @@ path_to_cqlsh = join(cqlsh_dir, 'cqlsh.py') sys.path.append(cqlsh_dir) -import cqlsh - -cql = cqlsh.cassandra.cluster.Cluster -policy = cqlsh.cassandra.policies.RoundRobinPolicy() -quote_name = cqlsh.cassandra.metadata.maybe_escape_name - TEST_HOST = os.environ.get('CQL_TEST_HOST', '127.0.0.1') TEST_PORT = int(os.environ.get('CQL_TEST_PORT', 9042)) +TEST_USER = os.environ.get('CQL_TEST_USER', 'cassandra') +TEST_PWD = os.environ.get('CQL_TEST_PWD') class BaseTestCase(unittest.TestCase): diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py index c4eae0ec8a..21ca5b128e 100644 --- a/pylib/cqlshlib/test/cassconnect.py +++ b/pylib/cqlshlib/test/cassconnect.py @@ -20,16 +20,24 @@ import io import os.path import random import string -from nose.tools import nottest -from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, test_dir +from cassandra.cluster import Cluster +from cassandra.policies import RoundRobinPolicy +from cassandra.metadata import maybe_escape_name as quote_name +from cassandra.auth import PlainTextAuthProvider +from cqlshlib.cql3handling import CqlRuleSet + +from .basecase import TEST_HOST, TEST_PORT, TEST_USER, TEST_PWD, cqlshlog, test_dir from .run_cqlsh import run_cqlsh, call_cqlsh test_keyspace_init = os.path.join(test_dir, 'test_keyspace_init.cql') def get_cassandra_connection(cql_version=None): - conn = cql((TEST_HOST,), TEST_PORT, cql_version=cql_version, load_balancing_policy=policy) + + auth_provider = PlainTextAuthProvider(username=TEST_USER, password=TEST_PWD) + conn = Cluster((TEST_HOST,), TEST_PORT, auth_provider=auth_provider, cql_version=cql_version, load_balancing_policy=RoundRobinPolicy()) + # until the cql lib does this for us conn.cql_version = cql_version return conn @@ -118,7 +126,8 @@ def cassandra_connection(cql_version=None): try: yield conn finally: - conn.close() + conn.shutdown() + @contextlib.contextmanager def cassandra_cursor(cql_version=None, ks=''): @@ -146,13 +155,13 @@ def cassandra_cursor(cql_version=None, ks=''): def cql_rule_set(): - return cqlsh.cql3handling.CqlRuleSet + return CqlRuleSet class DEFAULTVAL: pass -@nottest +__TEST__ = False def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): # use a positive default sentinel so that keyspace=None can be used # to override the default behavior @@ -161,7 +170,7 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): return run_cqlsh(keyspace=keyspace, **kwargs) -@nottest +__TEST__ = False def testcall_cqlsh(keyspace=None, **kwargs): if keyspace is None: keyspace = get_keyspace() diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index bef39e3f87..8bba22cef8 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -21,9 +21,10 @@ import locale import os import re -from .basecase import BaseTestCase, cqlsh +from .basecase import BaseTestCase from .cassconnect import create_db, remove_db, testrun_cqlsh from .run_cqlsh import TimeoutError +from cqlshlib.cql3handling import CqlRuleSet BEL = '\x07' # the terminal-bell character CTRL_C = '\x03' @@ -152,12 +153,11 @@ class CqlshCompletionCase(BaseTestCase): def strategies(self): - return self.module.CqlRuleSet.replication_strategies + return CqlRuleSet.replication_strategies class TestCqlshCompletion(CqlshCompletionCase): cqlver = '3.1.6' - module = cqlsh.cql3handling def test_complete_on_empty_string(self): self.trycompletions('', choices=('?', 'ALTER', 'BEGIN', 'CAPTURE', 'CONSISTENCY', diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 002e37171a..d575b8a3ef 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -22,11 +22,9 @@ from __future__ import unicode_literals, with_statement import locale import os import re -import six -import unittest from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT, - at_a_time, cqlsh, cqlshlog, dedent) + at_a_time, cqlshlog, dedent) from .cassconnect import (cassandra_cursor, create_db, get_keyspace, quote_name, remove_db, split_cql_commands, testcall_cqlsh, testrun_cqlsh) diff --git a/pylib/cqlshlib/test/test_sslhandling.py b/pylib/cqlshlib/test/test_sslhandling.py index 347fe2a114..dffe43abf5 100644 --- a/pylib/cqlshlib/test/test_sslhandling.py +++ b/pylib/cqlshlib/test/test_sslhandling.py @@ -17,12 +17,13 @@ from cassandra.policies import SimpleConvictionPolicy from cassandra.pool import Host from cqlshlib.sslhandling import ssl_settings -from nose.tools import assert_raises +import pytest import unittest import os import ssl + class SslSettingsTest(unittest.TestCase): def setUp(self): @@ -53,9 +54,9 @@ class SslSettingsTest(unittest.TestCase): def test_invalid_ssl_versions_from_env(self): msg = "invalid_ssl is not a valid SSL protocol, please use one of TLSv1, TLSv1_1, or TLSv1_2" - with assert_raises(SystemExit) as error: + with pytest.raises(SystemExit) as error: self._test_ssl_version_from_env('invalid_ssl') - assert msg == error.exception.message + assert msg == error.args[0] def test_default_ssl_version(self): ssl_ret_val = ssl_settings(self.host, self.config_file) @@ -69,7 +70,6 @@ class SslSettingsTest(unittest.TestCase): def test_invalid_ssl_version_config(self): msg = "invalid_ssl is not a valid SSL protocol, please use one of TLSv1, TLSv1_1, or TLSv1_2" - with assert_raises(SystemExit) as error: + with pytest.raises(SystemExit) as error: ssl_settings(self.host, os.path.join('test', 'config', 'sslhandling_invalid.config')) assert msg in error.exception.message - \ No newline at end of file diff --git a/pylib/requirements.txt b/pylib/requirements.txt index b6ca329b07..a48c55cb93 100644 --- a/pylib/requirements.txt +++ b/pylib/requirements.txt @@ -11,8 +11,7 @@ decorator docopt flaky mock -nose -nose-test-select parse -pycodestyle psutil +pycodestyle +pytest