add plugin support for CQLSH

patch by Brian Houser; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-16456
This commit is contained in:
Bhouse99 2022-03-30 13:37:22 -07:00 committed by Stefan Miklosovic
parent f444c40286
commit 381c2a4fa8
21 changed files with 501 additions and 36 deletions

1
.gitignore vendored
View File

@ -14,6 +14,7 @@ lib/
pylib/src/
**/cqlshlib.xml
!lib/cassandra-driver-internal-only-*.zip
!lib/puresasl-*.zip
# C* debs
build-stamp

View File

@ -1,4 +1,5 @@
4.1
* Add plugin support for CQLSH (CASSANDRA-16456)
* Add guardrail to disallow querying with ALLOW FILTERING (CASSANDRA-17370)
* Enhance SnakeYAML properties to be reusable outside of YAML parsing, support camel case conversion to snake case, and add support to ignore properties (CASSANDRA-17166)
* nodetool compact should support using a key string to find the range to avoid operators having to manually do this (CASSANDRA-17537)

View File

@ -113,7 +113,7 @@ if cql_zip:
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
# the driver needs dependencies
third_parties = ('six-')
third_parties = ('six-', 'puresasl-')
for lib in third_parties:
lib_zip = find_zip(lib)
@ -145,7 +145,7 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib')
if os.path.isdir(cqlshlibdir):
sys.path.insert(0, cqlshlibdir)
from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling
from cqlshlib import cql3handling, pylexotron, sslhandling, cqlshhandling, authproviderhandling
from cqlshlib.copyutil import ExportTask, ImportTask
from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
RED, WHITE, FormattedValue, colorme)
@ -154,6 +154,8 @@ from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
format_by_type)
from cqlshlib.tracing import print_trace, print_trace_session
from cqlshlib.util import get_file_encoding_bomsize
from cqlshlib.util import is_file_secure
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 9042
@ -426,7 +428,7 @@ class Shell(cmd.Cmd):
default_page_size = 100
def __init__(self, hostname, port, color=False,
username=None, password=None, encoding=None, stdin=None, tty=True,
username=None, encoding=None, stdin=None, tty=True,
completekey=DEFAULT_COMPLETEKEY, browser=None, use_conn=None,
cqlver=None, keyspace=None,
tracing_enabled=False, expand_enabled=False,
@ -442,16 +444,21 @@ class Shell(cmd.Cmd):
request_timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,
protocol_version=None,
connect_timeout=DEFAULT_CONNECT_TIMEOUT_SECONDS,
is_subshell=False):
is_subshell=False,
auth_provider=None):
cmd.Cmd.__init__(self, completekey=completekey)
self.hostname = hostname
self.port = port
self.auth_provider = None
if username:
if not password:
password = getpass.getpass()
self.auth_provider = PlainTextAuthProvider(username=username, password=password)
self.auth_provider = auth_provider
self.username = username
if isinstance(auth_provider, PlainTextAuthProvider):
self.username = auth_provider.username
if not auth_provider.password:
# if no password is provided, we need to query the user to get one.
password = getpass.getpass()
self.auth_provider = PlainTextAuthProvider(username=auth_provider.username, password=password)
self.keyspace = keyspace
self.ssl = ssl
self.tracing_enabled = tracing_enabled
@ -1613,10 +1620,8 @@ class Shell(cmd.Cmd):
except IOError as e:
self.printerr('Could not open %r: %s' % (fname, e))
return
username = self.auth_provider.username if self.auth_provider else None
password = self.auth_provider.password if self.auth_provider else None
subshell = Shell(self.hostname, self.port, color=self.color,
username=username, password=password,
username=self.username,
encoding=self.encoding, stdin=f, tty=False, use_conn=self.conn,
cqlver=self.cql_version, keyspace=self.current_keyspace,
tracing_enabled=self.tracing_enabled,
@ -1629,7 +1634,8 @@ class Shell(cmd.Cmd):
max_trace_wait=self.max_trace_wait, ssl=self.ssl,
request_timeout=self.session.default_timeout,
connect_timeout=self.conn.connect_timeout,
is_subshell=True)
is_subshell=True,
auth_provider=self.auth_provider)
# duplicate coverage related settings in subshell
if self.coverage:
subshell.coverage = True
@ -2077,21 +2083,6 @@ def should_use_color():
return True
def is_file_secure(filename):
try:
st = os.stat(filename)
except OSError as e:
if e.errno != errno.ENOENT:
raise
return True # the file doesn't exists, the security of it is irrelevant
uid = os.getuid()
# Skip enforcing the file owner and UID matching for the root user (uid == 0).
# This is to allow "sudo cqlsh" to work with user owned credentials file.
return (uid == 0 or st.st_uid == uid) and stat.S_IMODE(st.st_mode) & (stat.S_IRGRP | stat.S_IROTH) == 0
def read_options(cmdlineargs, environment):
configs = configparser.ConfigParser()
configs.read(CONFIG_FILE)
@ -2102,13 +2093,14 @@ def read_options(cmdlineargs, environment):
username_from_cqlshrc = option_with_default(configs.get, 'authentication', 'username')
password_from_cqlshrc = option_with_default(rawconfigs.get, 'authentication', 'password')
if username_from_cqlshrc or password_from_cqlshrc:
if password_from_cqlshrc and not is_file_secure(CONFIG_FILE):
if password_from_cqlshrc and not is_file_secure(os.path.expanduser(CONFIG_FILE)):
print("\nWarning: Password is found in an insecure cqlshrc file. The file is owned or readable by other users on the system.",
end='', file=sys.stderr)
print("\nNotice: Credentials in the cqlshrc file is deprecated and will be ignored in the future."
"\nPlease use a credentials file to specify the username and password.\n", file=sys.stderr)
optvalues = optparse.Values()
optvalues.username = None
optvalues.password = None
optvalues.credentials = os.path.expanduser(option_with_default(configs.get, 'authentication', 'credentials',
@ -2153,6 +2145,13 @@ def read_options(cmdlineargs, environment):
(options, arguments) = parser.parse_args(cmdlineargs, values=optvalues)
# Credentials from cqlshrc will be expanded,
# credentials from the command line are also expanded if there is a space...
# we need the following so that these two scenarios will work
# cqlsh --credentials=~/.cassandra/creds
# cqlsh --credentials ~/.cassandra/creds
options.credentials = os.path.expanduser(options.credentials)
if not is_file_secure(options.credentials):
print("\nWarning: Credentials file '{0}' exists but is not used, because:"
"\n a. the file owner is not the current user; or"
@ -2169,7 +2168,7 @@ def read_options(cmdlineargs, environment):
credentials.read(options.credentials)
# use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters
options.username = option_with_default(credentials.get, 'plain_text_auth', 'username', username_from_cqlshrc)
options.username = username_from_cqlshrc
if not options.password:
rawcredentials = configparser.RawConfigParser()
@ -2177,6 +2176,7 @@ def read_options(cmdlineargs, environment):
# handling password in the same way as username, priority cli > credentials > cqlshrc
options.password = option_with_default(rawcredentials.get, 'plain_text_auth', 'password', password_from_cqlshrc)
options.password = password_from_cqlshrc
elif not options.insecure_password_without_warning:
print("\nWarning: Using a password on the command line interface can be insecure."
"\nRecommendation: use the credentials file to securely provide the password.\n", file=sys.stderr)
@ -2330,7 +2330,6 @@ def main(options, hostname, port):
port,
color=options.color,
username=options.username,
password=options.password,
stdin=stdin,
tty=options.tty,
completekey=options.completekey,
@ -2349,7 +2348,12 @@ def main(options, hostname, port):
single_statement=options.execute,
request_timeout=options.request_timeout,
connect_timeout=options.connect_timeout,
encoding=options.encoding)
encoding=options.encoding,
auth_provider=authproviderhandling.load_auth_provider(
config_file=CONFIG_FILE,
cred_file=options.credentials,
username=options.username,
password=options.password))
except KeyboardInterrupt:
sys.exit('Connection aborted.')
except CQL_ERRORS as e:

View File

@ -392,7 +392,7 @@
<target name="realclean" depends="clean" description="Remove the entire build directory and all downloaded artifacts">
<delete>
<fileset dir="${build.lib}" excludes="cassandra-driver-internal-only-*"/>
<fileset dir="${build.lib}" excludes="cassandra-driver-internal-only-*,puresasl-internal-only-*"/>
</delete>
<delete dir="${build.dir}" />
<delete dir="${doc.dir}/build" />
@ -1333,9 +1333,10 @@
<exclude name="ide/nbproject/private/**" />
</tarfileset>
<!-- python driver -->
<!-- python driver, puresasl for SASL / GSSAPI -->
<tarfileset dir="${basedir}" prefix="${final.name}-src">
<include name="lib/cassandra-driver-internal-only-**" />
<include name="lib/puresasl-internal-only-**" />
</tarfileset>
<!-- Shell includes in bin/ and tools/bin/ -->

View File

@ -24,6 +24,14 @@
; keyspace = ks1
[auth_provider]
;; you can specify any auth provider found in your python environment
;; module and class will be used to dynamically load the class
;; all other properties found here and in the credentials file under the class name
;; will be passed to the constructor
; module = cassandra.auth
; classname = PlainTextAuthProvider
; username = user1
[ui]
;; Whether or not to display query results with colors

View File

@ -19,7 +19,7 @@
;
; Please ensure this file is owned by the user and is not readable by group and other users
[plain_text_auth]
[PlainTextAuthProvider]
; username = fred
; password = !!bang!!$

Binary file not shown.

View File

@ -0,0 +1,176 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Handles loading of AuthProvider for CQLSH authentication.
"""
import configparser
import sys
from importlib import import_module
from cqlshlib.util import is_file_secure
def _warn_for_plain_text_security(config_file, provider_settings):
"""
Call when using PlainTextAuthProvider
check to see if password appears in the basic provider settings
as this is a security risk
Will write errors to stderr
"""
if 'password' in provider_settings:
if not is_file_secure(config_file):
print("""\nWarning: Password is found in an insecure cqlshrc file.
The file is owned or readable by other users on the system.""",
end='',
file=sys.stderr)
print("""\nNotice: Credentials in the cqlshrc file is deprecated and
will be ignored in the future.\n
Please use a credentials file to
specify the username and password.\n""",
file=sys.stderr)
def load_auth_provider(config_file=None, cred_file=None, username=None, password=None):
"""
Function which loads an auth provider from available config.
Params:
* config_file ..: path to cqlsh config file (usually ~/.cassandra/cqlshrc).
* cred_file ....: path to cqlsh credentials file (default is ~/.cassandra/credentials).
* username .....: override used to return PlainTextAuthProvider according to legacy case
* password .....: override used to return PlainTextAuthProvider according to legacy case
Will attempt to load an auth provider from available config file, using what's found in
credentials file as an override.
Config file is expected to list module name /class in the *auth_provider*
section for dynamic loading (which is to be of type auth_provider)
Additional params passed to the constructor of class should be specified
in the *auth_provider* section and can be freely named to match
auth provider's expectation.
If passed username and password these will be overridden and passed to auth provider
None is returned if no possible auth provider is found, and no username/password can be
returned. If a username is found, system will assume that PlainTextAuthProvider was
specified
EXAMPLE CQLSHRC:
# .. inside cqlshrc file
[auth_provider]
module = cassandra.auth
classname = PlainTextAuthProvider
username = user1
password = password1
if credentials file is specified put relevant properties under the class name
EXAMPLE
# ... inside credentials file for above example
[PlainTextAuthProvider]
password = password2
Credential attributes will override found in the cqlshrc.
in the above example, PlainTextAuthProvider would be used with a password of 'password2',
and username of 'user1'
"""
def get_settings_from_config(section_name,
conf_file,
interpolation=configparser.BasicInterpolation()):
"""
Returns dict from section_name, and ini based conf_file
* section_name ..: Section to read map of properties from (ex: [auth_provider])
* conf_file .....: Ini based config file to read. Will return empty dict if None.
* interpolation .: Interpolation to use.
If section is not found, or conf_file is None, function will return an empty dictionary.
"""
conf = configparser.ConfigParser(interpolation=interpolation)
if conf_file is None:
return {}
conf.read(conf_file)
if section_name in conf.sections():
return dict(conf.items(section_name))
return {}
def get_cred_file_settings(classname, creds_file):
# Since this is the credentials file we may be encountering raw strings
# as these are what passwords, or security tokens may inadvertently fall into
# we don't want interpolation to mess with them.
return get_settings_from_config(
section_name=classname,
conf_file=creds_file,
interpolation=None)
def get_auth_provider_settings(conf_file):
return get_settings_from_config(
section_name='auth_provider',
conf_file=conf_file)
def get_legacy_settings(legacy_username, legacy_password):
result = {}
if legacy_username is not None:
result['username'] = legacy_username
if legacy_password is not None:
result['password'] = legacy_password
return result
provider_settings = get_auth_provider_settings(config_file)
module_name = provider_settings.pop('module', None)
class_name = provider_settings.pop('classname', None)
if module_name is None and class_name is None:
# not specified, default to plaintext auth provider
module_name = 'cassandra.auth'
class_name = 'PlainTextAuthProvider'
elif module_name is None or class_name is None:
# then this was PARTIALLY specified.
return None
credential_settings = get_cred_file_settings(class_name, cred_file)
if module_name == 'cassandra.auth' and class_name == 'PlainTextAuthProvider':
# merge credential settings as overrides on top of provider settings.
# we need to ensure that password property gets "set" in all cases.
# this is to support the ability to give the user a prompt in other parts
# of the code.
_warn_for_plain_text_security(config_file, provider_settings)
ctor_args = {'password': None,
**provider_settings,
**credential_settings,
**get_legacy_settings(username, password)}
# if no username, we can't create PlainTextAuthProvider
if 'username' not in ctor_args:
return None
else:
# merge credential settings as overrides on top of provider settings.
ctor_args = {**provider_settings,
**credential_settings,
**get_legacy_settings(username, password)}
# Load class definitions
module = import_module(module_name)
auth_provider_klass = getattr(module, class_name)
# instantiate the class
return auth_provider_klass(**ctor_args)

View File

@ -0,0 +1,190 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import io
import os
import sys
import pytest
from cassandra.auth import PlainTextAuthProvider
from cqlshlib.authproviderhandling import load_auth_provider
def construct_config_path(config_file_name):
return os.path.join(os.path.dirname(__file__),
'test_authproviderhandling_config',
config_file_name)
# Simple class to help verify AuthProviders that don't need arguments.
class NoUserNamePlainTextAuthProvider(PlainTextAuthProvider):
def __init__(self):
super(NoUserNamePlainTextAuthProvider, self).__init__('', '')
class ComplexTextAuthProvider(PlainTextAuthProvider):
def __init__(self, username, password='default_pass', extra_flag=None):
super(ComplexTextAuthProvider, self).__init__(username, password)
self.extra_flag = extra_flag
def _assert_auth_provider_matches(actual, klass, expected_props):
"""
Assert that the provider matches class and properties
* actual ..........: Thing to compare with it
* klass ...........: Class to ensure this matches to (ie PlainTextAuthProvider)
* expected_props ..: Dict of var properties to match
"""
assert isinstance(actual, klass)
assert expected_props == vars(actual)
class CustomAuthProviderTest(unittest.TestCase):
def setUp(self):
self._captured_std_err = io.StringIO()
sys.stderr = self._captured_std_err
def tearDown(self):
self._captured_std_err.close()
sys.stdout = sys.__stderr__
def test_no_warning_insecure_if_no_pass(self):
load_auth_provider(construct_config_path('plain_text_partial_example'))
err_msg = self._captured_std_err.getvalue()
assert err_msg == ''
def test_insecure_creds(self):
load_auth_provider(construct_config_path('full_plain_text_example'))
err_msg = self._captured_std_err.getvalue()
assert "Notice:" in err_msg
assert "Warning:" in err_msg
def test_creds_not_checked_for_non_plaintext(self):
load_auth_provider(construct_config_path('complex_auth_provider_with_pass'))
err_msg = self._captured_std_err.getvalue()
assert err_msg == ''
def test_partial_property_example(self):
actual = load_auth_provider(construct_config_path('partial_example'))
_assert_auth_provider_matches(
actual,
NoUserNamePlainTextAuthProvider,
{"username": '',
"password": ''})
def test_full_property_example(self):
actual = load_auth_provider(construct_config_path('full_plain_text_example'))
_assert_auth_provider_matches(
actual,
PlainTextAuthProvider,
{"username": 'user1',
"password": 'pass1'})
def test_empty_example(self):
actual = load_auth_provider(construct_config_path('empty_example'))
assert actual is None
def test_plaintextauth_when_not_defined(self):
creds_file = construct_config_path('plain_text_full_creds')
actual = load_auth_provider(cred_file=creds_file)
_assert_auth_provider_matches(
actual,
PlainTextAuthProvider,
{"username": 'user2',
"password": 'pass2'})
def test_no_cqlshrc_file(self):
actual = load_auth_provider()
assert actual is None
def test_no_classname_example(self):
actual = load_auth_provider(construct_config_path('no_classname_example'))
assert actual is None
def test_improper_config_example(self):
with pytest.raises(ModuleNotFoundError) as error:
load_auth_provider(construct_config_path('illegal_example'))
assert error is not None
def test_username_password_passed_from_commandline(self):
creds_file = construct_config_path('complex_auth_provider_creds')
cqlshrc = construct_config_path('complex_auth_provider')
actual = load_auth_provider(cqlshrc, creds_file, 'user-from-legacy', 'pass-from-legacy')
_assert_auth_provider_matches(
actual,
ComplexTextAuthProvider,
{"username": 'user-from-legacy',
"password": 'pass-from-legacy',
"extra_flag": 'flag2'})
def test_creds_example(self):
creds_file = construct_config_path('complex_auth_provider_creds')
cqlshrc = construct_config_path('complex_auth_provider')
actual = load_auth_provider(cqlshrc, creds_file)
_assert_auth_provider_matches(
actual,
ComplexTextAuthProvider,
{"username": 'user1',
"password": 'pass2',
"extra_flag": 'flag2'})
def test_legacy_example_use_passed_username(self):
creds_file = construct_config_path('plain_text_partial_creds')
cqlshrc = construct_config_path('plain_text_partial_example')
actual = load_auth_provider(cqlshrc, creds_file, 'user3')
_assert_auth_provider_matches(
actual,
PlainTextAuthProvider,
{"username": 'user3',
"password": 'pass2'})
def test_legacy_example_no_auth_provider_given(self):
cqlshrc = construct_config_path('empty_example')
creds_file = construct_config_path('complex_auth_provider_creds')
actual = load_auth_provider(cqlshrc, creds_file, 'user3', 'pass3')
_assert_auth_provider_matches(
actual,
PlainTextAuthProvider,
{"username": 'user3',
"password": 'pass3'})
def test_shouldnt_pass_no_password_when_alt_auth_provider(self):
cqlshrc = construct_config_path('complex_auth_provider')
creds_file = None
actual = load_auth_provider(cqlshrc, creds_file, 'user3')
_assert_auth_provider_matches(
actual,
ComplexTextAuthProvider,
{"username": 'user3',
"password": 'default_pass',
"extra_flag": 'flag1'})
def test_legacy_example_no_password(self):
cqlshrc = construct_config_path('plain_text_partial_example')
creds_file = None
actual = load_auth_provider(cqlshrc, creds_file, 'user3')
_assert_auth_provider_matches(
actual,
PlainTextAuthProvider,
{"username": 'user3',
"password": None})

View File

@ -0,0 +1,10 @@
; Config for a custom auth provider that uses the auth_provider field
; ComplexTextAuthProvider is a PlainTextAuthProvider in the driver which
; takes an extra field (extra_flag).
; used by unit testing
[auth_provider]
module = cqlshlib.test.test_authproviderhandling
classname = ComplexTextAuthProvider
username = user1
extra_flag = flag1

View File

@ -0,0 +1,3 @@
[ComplexTextAuthProvider]
extra_flag = flag2
password = pass2

View File

@ -0,0 +1,11 @@
; Config for a custom auth provider that uses the auth_provider field
; ComplexTextAuthProvider is a PlainTextAuthProvider in the driver which
; takes an extra field (extra_flag).
; used by unit testing
[auth_provider]
module = cqlshlib.test.test_authproviderhandling
classname = ComplexTextAuthProvider
username = user1
password = pass1
extra_flag = flag1

View File

@ -0,0 +1,2 @@
; Config for a custom auth provider that uses only the auth_provider field

View File

@ -0,0 +1,10 @@
; Config for a custom auth provider that uses all possible fields
; This example loads the PlainTextAuthProvider and passes username and password to constructor
; dynamically.
; used by unit testing
[auth_provider]
module = cassandra.auth
classname = PlainTextAuthProvider
username = user1
password = pass1

View File

@ -0,0 +1,5 @@
; Example that shouldn't work
[auth_provider]
module = nowhere.illegal.wrong
classname = badclass

View File

@ -0,0 +1,5 @@
; Config for a custom auth provider that uses only the auth_provider field
; this version doesn't have a classname, but has a module name.
[auth_provider]
module = cqlshlib.test

View File

@ -0,0 +1,8 @@
; Config for a custom auth provider that uses only the auth_provider field
; NoUserNamePlainTextAuthProvider is a PlainTextAuthProvider in the driver which
; doesn't take a username or password.
; used by unit testing
[auth_provider]
module = cqlshlib.test.test_authproviderhandling
classname = NoUserNamePlainTextAuthProvider

View File

@ -0,0 +1,3 @@
[PlainTextAuthProvider]
password = pass2
username = user2

View File

@ -0,0 +1,2 @@
[PlainTextAuthProvider]
password = pass2

View File

@ -0,0 +1,8 @@
; Config for a custom auth provider that uses some possible fields
; validate that the partial breakdown works successfully
; used by unit testing
[auth_provider]
module = cassandra.auth
classname = PlainTextAuthProvider
username = user1

View File

@ -18,7 +18,9 @@
import cProfile
import codecs
import pstats
import os
import errno
import stat
from datetime import timedelta, tzinfo
from io import StringIO
@ -112,6 +114,21 @@ def trim_if_present(s, prefix):
return s
def is_file_secure(filename):
try:
st = os.stat(filename)
except OSError as e:
if e.errno != errno.ENOENT:
raise
# the file doesn't exist, the security of it is irrelevant
return True
uid = os.getuid()
# Skip enforcing the file owner and UID matching for the root user (uid == 0).
# This is to allow "sudo cqlsh" to work with user owned credentials file.
return (uid == 0 or st.st_uid == uid) and stat.S_IMODE(st.st_mode) & (stat.S_IRGRP | stat.S_IROTH) == 0
def get_file_encoding_bomsize(filename):
"""
Checks the beginning of a file for a Unicode BOM. Based on this check,