From 2e2a49fcdcf34ef2e8afcb3369f5a62cb168f18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20de=20la=20Pe=C3=B1a?= Date: Wed, 7 Dec 2022 17:28:24 +0000 Subject: [PATCH] CEP-20: Dynamic Data Masking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows to attach the native masking functions and UDFs to the definitions of table columns in the schema, as defined by CEP-20. The functions masking a column can be specified on CREATE TABLE queries, right at the end of the column definition. The mask of a column can also be changed or dropped with an ALTER TABLE query. Once a column is masked, SELECT queries will always return the masked value of the column. That masking is done on the coordinator, at the end of the query execution. Thus, masking won't affect any filters or ordering, which would be based on the clear values of the masked columns. Column masks are stored on the table system_schema.column_masks. A new UNMASK permission allows to see the clear data of columns with an attached mask. Also, a new SELECT_MASKED permission allows to run SELECT queries restricting the clear values of masked columns. Superusers have both permissions by default, whereas regular users don't have them. Dynamic data masking can be enabled/disabled with the config property dynamic_data_masking_enabled in cassandra.yaml. It is disabled by default. This is the combination of multiple tickets: * Add masking functions to column metadata (CASSANDRA-18068) * Add UNMASK permission (CASSANDRA-18069) * Add SELECT_MASKED permission (CASSANDRA-18070) * Add support for using UDFs as masking functions (CASSANDRA-18071) * Add feature flag for dynamic data masking (CASSANDRA-18316) patch by Andrés de la Peña; reviewed by Benjamin Lerer and Berenguer Blasi for CASSANDRA-17940 --- CHANGES.txt | 1 + NEWS.txt | 8 + conf/cassandra.yaml | 8 + doc/cql3/CQL.textile | 28 +- .../cassandra/examples/BNF/alter_table.bnf | 5 +- .../cassandra/examples/BNF/create_table.bnf | 5 +- .../BNF/grant_permission_statement.bnf | 2 +- .../cassandra/examples/CQL/ddm_alter_mask.cql | 1 + .../examples/CQL/ddm_create_table.cql | 5 + .../CQL/ddm_create_table_with_udf.cql | 11 + .../examples/CQL/ddm_create_users.cql | 6 + .../cassandra/examples/CQL/ddm_drop_mask.cql | 1 + .../examples/CQL/ddm_insert_data.cql | 2 + .../examples/CQL/ddm_revoke_unmask.cql | 1 + .../CQL/ddm_select_with_masked_columns.cql | 6 + .../CQL/ddm_select_with_select_masked.cql | 8 + .../CQL/ddm_select_with_unmask_permission.cql | 7 + .../CQL/ddm_select_without_select_masked.cql | 6 + .../ddm_select_without_unmask_permission.cql | 7 + .../CQL/select_with_mask_functions.cql | 15 + doc/modules/cassandra/nav.adoc | 1 + .../cassandra/pages/cql/appendices.adoc | 3 + doc/modules/cassandra/pages/cql/changes.adoc | 4 + .../cassandra/pages/cql/cql_singlefile.adoc | 21 +- .../pages/cql/dynamic_data_masking.adoc | 178 ++++++ .../cassandra/pages/cql/functions.adoc | 63 +- doc/modules/cassandra/pages/cql/index.adoc | 1 + doc/modules/cassandra/pages/cql/security.adoc | 14 + .../cassandra/partials/masking_functions.adoc | 61 ++ pylib/cqlshlib/cql3handling.py | 7 +- pylib/cqlshlib/test/test_cqlsh_completion.py | 31 +- src/antlr/Cql.g | 1 + src/antlr/Lexer.g | 4 + src/antlr/Parser.g | 31 +- .../cassandra/auth/CassandraRoleManager.java | 2 +- .../apache/cassandra/auth/DataResource.java | 12 +- .../org/apache/cassandra/auth/Permission.java | 8 +- .../org/apache/cassandra/config/Config.java | 2 + .../cassandra/config/DatabaseDescriptor.java | 14 + .../apache/cassandra/cql3/QueryProcessor.java | 6 +- .../cassandra/cql3/UntypedResultSet.java | 4 +- .../cql3/functions/FunctionResolver.java | 35 +- .../cql3/functions/masking/ColumnMask.java | 314 ++++++++++ .../masking/DefaultMaskingFunction.java | 25 +- .../masking/HashMaskingFunction.java | 35 +- .../functions/masking/MaskingFunction.java | 31 + .../masking/NullMaskingFunction.java | 15 +- .../masking/PartialMaskingFunction.java | 63 +- .../masking/ReplaceMaskingFunction.java | 21 +- .../cql3/selection/ResultSetBuilder.java | 15 +- .../cassandra/cql3/selection/Selectable.java | 13 +- .../cassandra/cql3/selection/Selection.java | 26 +- .../cassandra/cql3/selection/Selector.java | 12 +- .../cql3/selection/SelectorFactories.java | 2 +- .../cql3/selection/SimpleSelector.java | 31 +- .../statements/ModificationStatement.java | 2 +- .../cql3/statements/SelectStatement.java | 73 ++- .../schema/AlterTableStatement.java | 119 +++- .../schema/CreateTableStatement.java | 158 +++-- .../schema/CreateViewStatement.java | 25 +- .../schema/DropFunctionStatement.java | 7 + src/java/org/apache/cassandra/db/Columns.java | 6 +- .../cassandra/db/aggregation/GroupMaker.java | 4 +- .../cassandra/schema/ColumnMetadata.java | 78 ++- .../cassandra/schema/KeyspaceMetadata.java | 11 + .../cassandra/schema/SchemaKeyspace.java | 156 ++++- .../schema/SchemaKeyspaceTables.java | 4 +- .../cassandra/schema/TableMetadata.java | 127 +++- .../apache/cassandra/schema/ViewMetadata.java | 12 + .../apache/cassandra/service/ClientState.java | 21 + .../utils/NativeSSTableLoaderClient.java | 4 +- .../distributed/test/ColumnMaskTest.java | 200 ++++++ .../util/SingleHostLoadBalancingPolicy.java | 100 +++ .../AtomicBTreePartitionUpdateBench.java | 8 +- .../org/apache/cassandra/SchemaLoader.java | 6 +- .../auth/CreateAndAlterRoleTest.java | 46 +- .../org/apache/cassandra/cql3/CQLTester.java | 28 + .../ColumnMaskInAnyPositionTester.java | 267 ++++++++ ...olumnMaskInAnyPositionWithDefaultTest.java | 45 ++ .../ColumnMaskInAnyPositionWithNullTest.java | 42 ++ ...olumnMaskInAnyPositionWithPartialTest.java | 54 ++ ...olumnMaskInAnyPositionWithReplaceTest.java | 53 ++ .../ColumnMaskInAnyPositionWithUDFTest.java | 71 +++ .../masking/ColumnMaskNativeTypesTest.java | 89 +++ .../masking/ColumnMaskQueryTester.java | 128 ++++ .../ColumnMaskQueryWithDefaultTest.java | 46 ++ .../masking/ColumnMaskQueryWithNullTest.java | 44 ++ .../ColumnMaskQueryWithPartialTest.java | 48 ++ .../ColumnMaskQueryWithReplaceTest.java | 49 ++ .../masking/ColumnMaskQueryWithUDFTest.java | 71 +++ .../functions/masking/ColumnMaskTest.java | 580 ++++++++++++++++++ .../functions/masking/ColumnMaskTester.java | 158 +++++ ...olumnMaskWithTypeAlteringFunctionTest.java | 148 +++++ .../masking/ColumnMaskWithUDFTest.java | 284 +++++++++ .../masking/MaskingFunctionTester.java | 8 +- .../masking/SelectMaskedPermissionTest.java | 367 +++++++++++ .../masking/UnmaskPermissionTest.java | 223 +++++++ .../statements/DescribeStatementTest.java | 57 +- .../org/apache/cassandra/db/CellTest.java | 3 +- .../org/apache/cassandra/db/ColumnsTest.java | 2 +- .../apache/cassandra/db/NativeCellTest.java | 3 +- .../db/compaction/CompactionIteratorTest.java | 4 +- .../db/guardrails/GuardrailTester.java | 8 + .../db/transform/DuplicateRowCheckerTest.java | 10 +- .../io/sstable/SSTableHeaderFixTest.java | 2 +- .../cassandra/schema/SchemaKeyspaceTest.java | 2 +- .../cassandra/utils/CassandraGenerators.java | 2 +- 107 files changed, 4916 insertions(+), 386 deletions(-) create mode 100644 doc/modules/cassandra/examples/CQL/ddm_alter_mask.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_create_table.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_create_users.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_drop_mask.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_insert_data.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_revoke_unmask.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_select_with_masked_columns.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_select_with_select_masked.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_select_with_unmask_permission.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_select_without_select_masked.cql create mode 100644 doc/modules/cassandra/examples/CQL/ddm_select_without_unmask_permission.cql create mode 100644 doc/modules/cassandra/examples/CQL/select_with_mask_functions.cql create mode 100644 doc/modules/cassandra/pages/cql/dynamic_data_masking.adoc create mode 100644 doc/modules/cassandra/partials/masking_functions.adoc create mode 100644 src/java/org/apache/cassandra/cql3/functions/masking/ColumnMask.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/ColumnMaskTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/util/SingleHostLoadBalancingPolicy.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionTester.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithDefaultTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithNullTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithPartialTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithReplaceTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskInAnyPositionWithUDFTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskNativeTypesTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryTester.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithNullTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithPartialTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithUDFTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTester.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithTypeAlteringFunctionTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskWithUDFTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/SelectMaskedPermissionTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/functions/masking/UnmaskPermissionTest.java diff --git a/CHANGES.txt b/CHANGES.txt index be048ff248..062fa500b6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * CEP-20: Dynamic Data Masking (CASSANDRA-17940) * Add system_views.snapshots virtual table (CASSANDRA-18102) * Update OpenHFT dependencies (chronicle-queue, chronicle-core, chronicle-bytes, chronicle-wire, chronicle-threads) (CASSANDRA-18049) * Remove org.apache.cassandra.hadoop code (CASSANDRA-18323) diff --git a/NEWS.txt b/NEWS.txt index 48789e4bbe..3d7a25eea9 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -132,6 +132,14 @@ New features - Added local read/write ratio to tablestats. - Added system_views.max_sstable_size and system_views.max_sstable_duration tables. - Added virtual table system_views.snapshots to see all snapshots from CQL shell. + - Added support for attaching CQL dynamic data masking functions to table columns on the schema. These masking + functions can be attached to or dettached from columns with CREATE/ALTER TABLE statements. The functions obscure + the masked data during queries, but they don't change the stored data. + - Added new UNMASK permission. It allows to see the clear data of columns with an attached mask. Superusers have it + by default, whereas regular users don't have it by default. + - Added new SELECT_MASKED permission. It allows to run SELECT queries selecting the clear values of masked columns. + Superusers have it by default, whereas regular users don't have it by default. + - Added support for using UDFs as masking functions attached to table columns on the schema. Upgrading --------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 746fd5743e..ee616e1674 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1660,6 +1660,14 @@ report_unconfirmed_repaired_data_mismatches: false # warming of auth caches prior to node completing startup. See CASSANDRA-16958 # auth_cache_warming_enabled: false +# If enabled, dynamic data masking allows to attach CQL masking functions to the columns of a table. +# Users without the UNMASK permission will see an obscured version of the values of the columns with an attached mask. +# If dynamic data masking is disabled it won't be allowed to create new column masks, although it will still be possible +# to drop any previously existing masks. Also, any existing mask will be ignored at query time, so all users will see +# the clear values of the masked columns. +# Defaults to false to disable dynamic data masking. +# dynamic_data_masking_enabled: false + ######################### # EXPERIMENTAL FEATURES # ######################### diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index 0147caa322..f5bb83807c 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -252,9 +252,11 @@ bc(syntax).. '(' ( ',' )* ')' ( WITH