From 3bdcaa336a6e6a9727c333b433bb9f5d3afc0fb1 Mon Sep 17 00:00:00 2001 From: Carl Yeksigian Date: Wed, 29 Jul 2015 16:06:08 -0400 Subject: [PATCH] Materialized Views patch by carl; reviewed by jmckenzie for CASSANDRA-6477 --- CHANGES.txt | 1 + NEWS.txt | 10 + conf/cassandra.yaml | 5 + doc/cql3/CQL.textile | 55 + pylib/cqlshlib/cql3handling.py | 13 +- .../apache/cassandra/concurrent/Stage.java | 4 + .../cassandra/concurrent/StageManager.java | 2 + .../apache/cassandra/config/CFMetaData.java | 61 +- .../org/apache/cassandra/config/Config.java | 2 + .../cassandra/config/DatabaseDescriptor.java | 11 + .../config/MaterializedViewDefinition.java | 93 ++ src/java/org/apache/cassandra/cql3/Cql.g | 143 ++- .../AlterMaterializedViewStatement.java | 83 ++ .../cql3/statements/AlterTableStatement.java | 85 ++ .../cql3/statements/CFProperties.java | 58 ++ .../CreateMaterializedViewStatement.java | 268 +++++ .../cql3/statements/CreateTableStatement.java | 29 +- .../statements/CreateTriggerStatement.java | 5 +- .../DropMaterializedViewStatement.java | 105 ++ .../cql3/statements/DropTableStatement.java | 25 + .../statements/ModificationStatement.java | 23 + .../cql3/statements/SelectStatement.java | 9 +- .../cql3/statements/TruncateStatement.java | 6 + .../db}/AbstractReadCommandBuilder.java | 114 ++- .../cassandra/db/ColumnFamilyStore.java | 58 +- .../org/apache/cassandra/db/Keyspace.java | 77 +- .../org/apache/cassandra/db/Mutation.java | 2 + .../cassandra/db/MutationVerbHandler.java | 8 + .../apache/cassandra/db/SystemKeyspace.java | 98 ++ .../org/apache/cassandra/db/WriteType.java | 3 +- .../db/compaction/CompactionManager.java | 26 + .../compaction/CompactionStrategyManager.java | 2 +- .../db/compaction/OperationType.java | 3 +- .../cassandra/db/view/MaterializedView.java | 691 +++++++++++++ .../db/view/MaterializedViewBuilder.java | 222 ++++ .../db/view/MaterializedViewManager.java | 237 +++++ .../db/view/MaterializedViewUtils.java | 95 ++ .../apache/cassandra/db/view/TemporalRow.java | 414 ++++++++ .../cassandra/metrics/KeyspaceMetrics.java | 2 +- .../cassandra/metrics/MVWriteMetrics.java | 42 + .../cassandra/net/MessagingService.java | 15 +- .../schema/LegacySchemaMigrator.java | 4 +- .../cassandra/schema/MaterializedViews.java | 149 +++ .../cassandra/schema/SchemaKeyspace.java | 158 ++- .../service/AbstractReadExecutor.java | 2 +- .../service/AbstractWriteResponseHandler.java | 2 +- .../service/BatchlogResponseHandler.java | 121 +++ .../cassandra/service/CassandraDaemon.java | 18 + .../cassandra/service/StorageProxy.java | 244 ++++- .../cassandra/service/StorageService.java | 24 +- .../streaming/StreamReceiveTask.java | 85 +- .../cassandra/thrift/CassandraServer.java | 28 + .../cassandra/thrift/ThriftConversion.java | 5 +- .../utils/NativeSSTableLoaderClient.java | 3 +- .../cql3/MaterializedViewLongTest.java | 194 ++++ .../org/apache/cassandra/cql3/CQLTester.java | 4 +- .../cassandra/cql3/MaterializedViewTest.java | 955 ++++++++++++++++++ .../cassandra/db/RangeTombstoneTest.java | 1 - .../db/view/MaterializedViewUtilsTest.java | 115 +++ 59 files changed, 5104 insertions(+), 213 deletions(-) create mode 100644 src/java/org/apache/cassandra/config/MaterializedViewDefinition.java create mode 100644 src/java/org/apache/cassandra/cql3/statements/AlterMaterializedViewStatement.java create mode 100644 src/java/org/apache/cassandra/cql3/statements/CFProperties.java create mode 100644 src/java/org/apache/cassandra/cql3/statements/CreateMaterializedViewStatement.java create mode 100644 src/java/org/apache/cassandra/cql3/statements/DropMaterializedViewStatement.java rename {test/unit/org/apache/cassandra => src/java/org/apache/cassandra/db}/AbstractReadCommandBuilder.java (75%) create mode 100644 src/java/org/apache/cassandra/db/view/MaterializedView.java create mode 100644 src/java/org/apache/cassandra/db/view/MaterializedViewBuilder.java create mode 100644 src/java/org/apache/cassandra/db/view/MaterializedViewManager.java create mode 100644 src/java/org/apache/cassandra/db/view/MaterializedViewUtils.java create mode 100644 src/java/org/apache/cassandra/db/view/TemporalRow.java create mode 100644 src/java/org/apache/cassandra/metrics/MVWriteMetrics.java create mode 100644 src/java/org/apache/cassandra/schema/MaterializedViews.java create mode 100644 src/java/org/apache/cassandra/service/BatchlogResponseHandler.java create mode 100644 test/long/org/apache/cassandra/cql3/MaterializedViewLongTest.java create mode 100644 test/unit/org/apache/cassandra/cql3/MaterializedViewTest.java create mode 100644 test/unit/org/apache/cassandra/db/view/MaterializedViewUtilsTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 71d5a2db6c..d8b83bf4ae 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,7 @@ * Populate TokenMetadata early during startup (CASSANDRA-9317) * Undeprecate cache recentHitRate (CASSANDRA-6591) * Add support for selectively varint encoding fields (CASSANDRA-9499, 9865) + * Materialized Views (CASSANDRA-6477) 2.2.1 diff --git a/NEWS.txt b/NEWS.txt index ba81d6e15b..392a5815d8 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -16,8 +16,18 @@ using the provided 'sstableupgrade' tool. 3.0 === +New features +------------ + - Materialized Views, which allow for server-side denormalization, is now + available. Materialized views provide an alternative to secondary indexes + for non-primary key queries, and perform much better for indexing high + cardinality columns. + See http://www.datastax.com/dev/blog/new-in-cassandra-3-0-materialized-views + Upgrading --------- + - New write stages have been added for batchlog and materialized view mutations + you can set their size in cassandra.yaml - User defined functions are now executed in a sandbox. To use UDFs and UDAs, you have to enable them in cassandra.yaml. - New SSTable version 'la' with improved bloom-filter false-positive handling diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 7ce36af0ac..0e200955a4 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -342,6 +342,11 @@ seed_provider: concurrent_reads: 32 concurrent_writes: 32 concurrent_counter_writes: 32 +concurrent_batchlog_writes: 32 + +# For materialized view writes, as there is a read involved, so this should +# be limited by the less of concurrent reads or concurrent writes. +concurrent_materialized_view_writes: 32 # Maximum memory to use for pooling sstable buffers. Defaults to the smaller # of 1/4 of heap or 512MB. This pool is allocated off-heap, so is in addition diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index bc7c385feb..d5c5684358 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -479,6 +479,61 @@ The @DROP INDEX@ statement is used to drop an existing secondary index. The argu If the index does not exists, the statement will return an error, unless @IF EXISTS@ is used in which case the operation is a no-op. + +h3(#createMVStmt). CREATE MATERIALIZED VIEW + +__Syntax:__ + +bc(syntax).. + ::= CREATE MATERIALIZED VIEW ( IF NOT EXISTS )? AS + SELECT ( '(' ( ',' ) * ')' | '*' ) + FROM + WHERE ( IS NOT NULL ( AND IS NOT NULL )* )? + PRIMARY KEY '(' ( ',' )* ')' + ( WITH