natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Materialized Views" 

This reverts commit 3bdcaa336a.
This commit is contained in:
Jonathan Ellis 2015-07-29 17:03:52 -05:00
parent ce3d9af331
commit 24d185d72b
59 changed files with 213 additions and 5104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -21,7 +21,6 @@
* Populate TokenMetadata early during startup (CASSANDRA-9317)
* Undeprecate cache recentHitRate (CASSANDRA-6591)
* Add support for selectively varint encoding fields (CASSANDRA-9499, 9865)
* Materialized Views (CASSANDRA-6477)
2.2.1

10
NEWS.txt
View File

@ -16,18 +16,8 @@ using the provided 'sstableupgrade' tool.
3.0
===
New features
------------
- Materialized Views, which allow for server-side denormalization, is now
available. Materialized views provide an alternative to secondary indexes
for non-primary key queries, and perform much better for indexing high
cardinality columns.
See http://www.datastax.com/dev/blog/new-in-cassandra-3-0-materialized-views
Upgrading
---------
- New write stages have been added for batchlog and materialized view mutations
you can set their size in cassandra.yaml
- User defined functions are now executed in a sandbox.
To use UDFs and UDAs, you have to enable them in cassandra.yaml.
- New SSTable version 'la' with improved bloom-filter false-positive handling

5
conf/cassandra.yaml
View File

@ -342,11 +342,6 @@ seed_provider:
concurrent_reads: 32
concurrent_writes: 32
concurrent_counter_writes: 32
concurrent_batchlog_writes: 32
# For materialized view writes, as there is a read involved, so this should
# be limited by the less of concurrent reads or concurrent writes.
concurrent_materialized_view_writes: 32
# Maximum memory to use for pooling sstable buffers. Defaults to the smaller
# of 1/4 of heap or 512MB. This pool is allocated off-heap, so is in addition

55
doc/cql3/CQL.textile
View File

@ -479,61 +479,6 @@ The @DROP INDEX@ statement is used to drop an existing secondary index. The argu
If the index does not exists, the statement will return an error, unless @IF EXISTS@ is used in which case the operation is a no-op.
h3(#createMVStmt). CREATE MATERIALIZED VIEW
__Syntax:__
bc(syntax)..
<create-table-stmt> ::= CREATE MATERIALIZED VIEW ( IF NOT EXISTS )? <viewname> AS
SELECT ( '(' <identifier> ( ',' <identifier> ) * ')' | '*' )
FROM <tablename>
WHERE ( <identifier> IS NOT NULL ( AND <identifier> IS NOT NULL )* )?
PRIMARY KEY '(' <partition-key> ( ',' <identifier> )* ')'
( WITH <option> ( AND <option>)* )?
__Sample:__
bc(sample)..
CREATE MATERIALIZED VIEW monkeySpecies_by_population AS
SELECT *
FROM monkeySpecies
WHERE population IS NOT NULL AND species IS NOT NULL
PRIMARY KEY (population, species)
WITH comment='Allow query by population instead of species';
p.
The @CREATE MATERIALIZED VIEW@ statement creates a new materialized view. Each such view is a set of _rows_ which corresponds to rows which are present in the underlying, or base, table specified in the @SELECT@ statement. A materialized view cannot be directly updated, but updates to the base table will cause corresponding updates in the view.
Attempting to create an already existing materialized view will return an error unless the @IF NOT EXISTS@ option is used. If it is used, the statement will be a no-op if the materialized view already exists.
h4(#createMVWhere). @WHERE <identifier> IS NOT NULL@
The where clause is required to explicitly exclude all primary key columns' null values. Any row which contains null values in the primary key will not be present in the materialized view.
h3(#alterMVStmt). ALTER MATERIALIZED VIEW
__Syntax:__
bc(syntax). <alter-materialized-view-stmt> ::= ALTER MATERIALIZED VIEW <viewname>
WITH <option> ( AND <option> )*
p.
The @ALTER MATERIALIZED VIEW@ statement allows options to be update; these options are the same as <a href="#createTableOptions">@CREATE TABLE@'s options</a>.
h3(#dropMVStmt). DROP MATERIALIZED VIEW
__Syntax:__
bc(syntax). <drop-materialized-stmt> ::= DROP MATERIALIZED VIEW ( IF EXISTS )? <tablename>
__Sample:__
bc(sample). DROP MATERIALIZED VIEW monkeySpecies_by_population;
The @DROP MATERIALIZED VIEW@ statement is used to drop an existing materialized view.
If the materialized view does not exists, the statement will return an error, unless @IF EXISTS@ is used in which case the operation is a no-op.
h3(#createTypeStmt). CREATE TYPE
__Syntax:__

13
pylib/cqlshlib/cql3handling.py
View File

@ -44,8 +44,7 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet):
'function', 'aggregate', 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop',
'primary', 'into', 'values', 'date', 'time', 'timestamp', 'ttl', 'alter', 'add', 'type',
'compact', 'storage', 'order', 'by', 'asc', 'desc', 'clustering',
'token', 'writetime', 'map', 'list', 'to', 'custom', 'if', 'not',
'materialized', 'view'
'token', 'writetime', 'map', 'list', 'to', 'custom', 'if', 'not'
))
unreserved_keywords = set((
@ -245,7 +244,6 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
<schemaChangeStatement> ::= <createKeyspaceStatement>
| <createColumnFamilyStatement>
| <createIndexStatement>
| <createMaterializedViewStatement>
| <createUserTypeStatement>
| <createFunctionStatement>
| <createAggregateStatement>
@ -253,7 +251,6 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
| <dropKeyspaceStatement>
| <dropColumnFamilyStatement>
| <dropIndexStatement>
| <dropMaterializedViewStatement>
| <dropUserTypeStatement>
| <dropFunctionStatement>
| <dropAggregateStatement>
@ -1093,11 +1090,6 @@ syntax_rules += r'''
( "USING" <stringLiteral> ( "WITH" "OPTIONS" "=" <mapLiteral> )? )?
;
<createMaterializedViewStatement> ::= "CREATE" "MATERIALIZED" "VIEW" ("IF" "NOT" "EXISTS")? <columnFamilyName>?
"AS" <selectStatement>
"PRIMARY" "KEY" <pkDef>
;
<createUserTypeStatement> ::= "CREATE" "TYPE" ( ks=<nonSystemKeyspaceName> dot="." )? typename=<cfOrKsName> "(" newcol=<cident> <storageType>
( "," [newcolname]=<cident> <storageType> )*
")"
@ -1154,9 +1146,6 @@ syntax_rules += r'''
<dropIndexStatement> ::= "DROP" "INDEX" ("IF" "EXISTS")? idx=<indexName>
;
<dropMaterializedViewStatement> ::= "DROP" "MATERIALIZED" "VIEW" ("IF" "EXISTS")? mv=<columnFamilyName>
;
<dropUserTypeStatement> ::= "DROP" "TYPE" ut=<userTypeName>
;

4
src/java/org/apache/cassandra/concurrent/Stage.java
View File

@ -27,8 +27,6 @@ public enum Stage
READ,
MUTATION,
COUNTER_MUTATION,
BATCHLOG_MUTATION,
MATERIALIZED_VIEW_MUTATION,
GOSSIP,
REQUEST_RESPONSE,
ANTI_ENTROPY,
@ -62,8 +60,6 @@ public enum Stage
return "internal";
case MUTATION:
case COUNTER_MUTATION:
case BATCHLOG_MUTATION:
case MATERIALIZED_VIEW_MUTATION:
case READ:
case REQUEST_RESPONSE:
case READ_REPAIR:

2
src/java/org/apache/cassandra/concurrent/StageManager.java
View File

@ -47,8 +47,6 @@ public class StageManager
{
stages.put(Stage.MUTATION, multiThreadedLowSignalStage(Stage.MUTATION, getConcurrentWriters()));
stages.put(Stage.COUNTER_MUTATION, multiThreadedLowSignalStage(Stage.COUNTER_MUTATION, getConcurrentCounterWriters()));
stages.put(Stage.BATCHLOG_MUTATION, multiThreadedLowSignalStage(Stage.BATCHLOG_MUTATION, getConcurrentBatchlogWriters()));
stages.put(Stage.MATERIALIZED_VIEW_MUTATION, multiThreadedLowSignalStage(Stage.MATERIALIZED_VIEW_MUTATION, getConcurrentMaterializedViewWriters()));
stages.put(Stage.READ, multiThreadedLowSignalStage(Stage.READ, getConcurrentReaders()));
stages.put(Stage.REQUEST_RESPONSE, multiThreadedLowSignalStage(Stage.REQUEST_RESPONSE, FBUtilities.getAvailableProcessors()));
stages.put(Stage.INTERNAL_RESPONSE, multiThreadedStage(Stage.INTERNAL_RESPONSE, FBUtilities.getAvailableProcessors()));

61
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -50,7 +50,6 @@ import org.apache.cassandra.io.compress.CompressionParameters;
import org.apache.cassandra.io.compress.LZ4Compressor;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.MaterializedViews;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.schema.Triggers;
import org.apache.cassandra.utils.*;
@ -64,7 +63,7 @@ public final class CFMetaData
{
public enum Flag
{
SUPER, COUNTER, DENSE, COMPOUND, MATERIALIZEDVIEW
SUPER, COUNTER, DENSE, COMPOUND
}
private static final Logger logger = LoggerFactory.getLogger(CFMetaData.class);
@ -86,15 +85,6 @@ public final class CFMetaData
// Note that this is the default only for user created tables
public final static String DEFAULT_COMPRESSOR = LZ4Compressor.class.getCanonicalName();
// Note that this need to come *before* any CFMetaData is defined so before the compile below.
private static final Comparator<ColumnDefinition> regularColumnComparator = new Comparator<ColumnDefinition>()
{
public int compare(ColumnDefinition def1, ColumnDefinition def2)
{
return ByteBufferUtil.compareUnsigned(def1.name.bytes, def2.name.bytes);
}
};
public static class SpeculativeRetry
{
public enum RetryType
@ -181,7 +171,6 @@ public final class CFMetaData
private final boolean isCompound;
private final boolean isSuper;
private final boolean isCounter;
private final boolean isMaterializedView;
public volatile ClusteringComparator comparator; // bytes, long, timeuuid, utf8, etc. This is built directly from clusteringColumns
@ -204,7 +193,6 @@ public final class CFMetaData
private volatile SpeculativeRetry speculativeRetry = DEFAULT_SPECULATIVE_RETRY;
private volatile Map<ByteBuffer, DroppedColumn> droppedColumns = new HashMap<>();
private volatile Triggers triggers = Triggers.none();
private volatile MaterializedViews materializedViews = MaterializedViews.none();
/*
* All CQL3 columns definition are stored in the columnMetadata map.
@ -247,7 +235,6 @@ public final class CFMetaData
public CFMetaData speculativeRetry(SpeculativeRetry prop) {speculativeRetry = prop; return this;}
public CFMetaData droppedColumns(Map<ByteBuffer, DroppedColumn> cols) {droppedColumns = cols; return this;}
public CFMetaData triggers(Triggers prop) {triggers = prop; return this;}
public CFMetaData materializedViews(MaterializedViews prop) {materializedViews = prop; return this;}
private CFMetaData(String keyspace,
String name,
@ -256,7 +243,6 @@ public final class CFMetaData
boolean isCounter,
boolean isDense,
boolean isCompound,
boolean isMaterializedView,
List<ColumnDefinition> partitionKeyColumns,
List<ColumnDefinition> clusteringColumns,
PartitionColumns partitionColumns)
@ -269,7 +255,6 @@ public final class CFMetaData
this.isCompound = isCompound;
this.isSuper = isSuper;
this.isCounter = isCounter;
this.isMaterializedView = isMaterializedView;
EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
if (isSuper)
@ -280,8 +265,6 @@ public final class CFMetaData
flags.add(Flag.DENSE);
if (isCompound)
flags.add(Flag.COMPOUND);
if (isMaterializedView)
flags.add(Flag.MATERIALIZEDVIEW);
this.flags = Sets.immutableEnumSet(flags);
// A compact table should always have a clustering
@ -316,11 +299,6 @@ public final class CFMetaData
this.compactValueColumn = CompactTables.getCompactValueColumn(partitionColumns, isSuper());
}
public MaterializedViews getMaterializedViews()
{
return materializedViews;
}
public static CFMetaData create(String ksName,
String name,
UUID cfId,
@ -328,7 +306,6 @@ public final class CFMetaData
boolean isCompound,
boolean isSuper,
boolean isCounter,
boolean isMaterializedView,
List<ColumnDefinition> columns)
{
List<ColumnDefinition> partitions = new ArrayList<>();
@ -361,7 +338,6 @@ public final class CFMetaData
isCounter,
isDense,
isCompound,
isMaterializedView,
partitions,
clusterings,
builder.build());
@ -463,7 +439,6 @@ public final class CFMetaData
isCounter(),
isDense(),
isCompound(),
isMaterializedView(),
copy(partitionKeyColumns),
copy(clusteringColumns),
copy(partitionColumns)),
@ -506,8 +481,7 @@ public final class CFMetaData
.speculativeRetry(oldCFMD.speculativeRetry)
.memtableFlushPeriod(oldCFMD.memtableFlushPeriod)
.droppedColumns(new HashMap<>(oldCFMD.droppedColumns))
.triggers(oldCFMD.triggers)
.materializedViews(oldCFMD.materializedViews);
.triggers(oldCFMD.triggers);
}
/**
@ -779,8 +753,7 @@ public final class CFMetaData
&& Objects.equal(maxIndexInterval, other.maxIndexInterval)
&& Objects.equal(speculativeRetry, other.speculativeRetry)
&& Objects.equal(droppedColumns, other.droppedColumns)
&& Objects.equal(triggers, other.triggers)
&& Objects.equal(materializedViews, other.materializedViews);
&& Objects.equal(triggers, other.triggers);
}
@Override
@ -812,7 +785,6 @@ public final class CFMetaData
.append(speculativeRetry)
.append(droppedColumns)
.append(triggers)
.append(materializedViews)
.toHashCode();
}
@ -874,7 +846,6 @@ public final class CFMetaData
compressionParameters = cfm.compressionParameters;
triggers = cfm.triggers;
materializedViews = cfm.materializedViews;
logger.debug("application result is {}", this);
@ -1291,14 +1262,6 @@ public final class CFMetaData
return false;
}
public boolean hasComplexColumns()
{
for (ColumnDefinition def : partitionColumns())
if (def.isComplex())
return true;
return false;
}
public boolean hasDroppedCollectionColumns()
{
for (DroppedColumn def : getDroppedColumns().values())
@ -1330,11 +1293,6 @@ public final class CFMetaData
return isCompound;
}
public boolean isMaterializedView()
{
return isMaterializedView;
}
public Serializers serializers()
{
return serializers;
@ -1379,7 +1337,6 @@ public final class CFMetaData
.append("speculativeRetry", speculativeRetry)
.append("droppedColumns", droppedColumns)
.append("triggers", triggers)
.append("materializedViews", materializedViews)
.toString();
}
@ -1391,7 +1348,6 @@ public final class CFMetaData
private final boolean isCompound;
private final boolean isSuper;
private final boolean isCounter;
private final boolean isMaterializedView;
private UUID tableId;
@ -1400,7 +1356,7 @@ public final class CFMetaData
private final List<Pair<ColumnIdentifier, AbstractType>> staticColumns = new ArrayList<>();
private final List<Pair<ColumnIdentifier, AbstractType>> regularColumns = new ArrayList<>();
private Builder(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter, boolean isMaterializedView)
private Builder(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter)
{
this.keyspace = keyspace;
this.table = table;
@ -1408,7 +1364,6 @@ public final class CFMetaData
this.isCompound = isCompound;
this.isSuper = isSuper;
this.isCounter = isCounter;
this.isMaterializedView = isMaterializedView;
}
public static Builder create(String keyspace, String table)
@ -1423,12 +1378,7 @@ public final class CFMetaData
public static Builder create(String keyspace, String table, boolean isDense, boolean isCompound, boolean isSuper, boolean isCounter)
{
return new Builder(keyspace, table, isDense, isCompound, isSuper, isCounter, false);
}
public static Builder createView(String keyspace, String table)
{
return new Builder(keyspace, table, false, true, false, false, true);
return new Builder(keyspace, table, isDense, isCompound, isSuper, isCounter);
}
public static Builder createDense(String keyspace, String table, boolean isCompound, boolean isCounter)
@ -1551,7 +1501,6 @@ public final class CFMetaData
isCounter,
isDense,
isCompound,
isMaterializedView,
partitions,
clusterings,
builder.build());

2
src/java/org/apache/cassandra/config/Config.java
View File

@ -92,8 +92,6 @@ public class Config
public Integer concurrent_reads = 32;
public Integer concurrent_writes = 32;
public Integer concurrent_counter_writes = 32;
public Integer concurrent_batchlog_writes = 32;
public Integer concurrent_materialized_view_writes = 32;
@Deprecated
public Integer concurrent_replicates = null;

11
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -990,8 +990,6 @@ public class DatabaseDescriptor
case PAXOS_COMMIT:
case PAXOS_PREPARE:
case PAXOS_PROPOSE:
case BATCHLOG_MUTATION:
case MATERIALIZED_VIEW_MUTATION:
return getWriteRpcTimeout();
case COUNTER_MUTATION:
return getCounterWriteRpcTimeout();
@ -1038,15 +1036,6 @@ public class DatabaseDescriptor
return conf.concurrent_counter_writes;
}
public static int getConcurrentBatchlogWriters()
{
return conf.concurrent_batchlog_writes;
}
public static int getConcurrentMaterializedViewWriters()
{
return conf.concurrent_materialized_view_writes;
}
public static int getFlushWriters()
{
return conf.memtable_flush_writers;

93
src/java/org/apache/cassandra/config/MaterializedViewDefinition.java
View File

@ -1,93 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.config;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.cassandra.cql3.ColumnIdentifier;
public class MaterializedViewDefinition
{
public final String baseCfName;
public final String viewName;
// The order of partititon columns and clustering columns is important, so we cannot switch these two to sets
public final List<ColumnIdentifier> partitionColumns;
public final List<ColumnIdentifier> clusteringColumns;
public final Set<ColumnIdentifier> included;
public final boolean includeAll;
public MaterializedViewDefinition(MaterializedViewDefinition def)
{
this(def.baseCfName, def.viewName, new ArrayList<>(def.partitionColumns), new ArrayList<>(def.clusteringColumns), new HashSet<>(def.included));
}
/**
* @param baseCfName Name of the column family from which this view is based
* @param viewName Name of the view
* @param partitionColumns List of all of the partition columns, in the order they are defined
* @param clusteringColumns List of all of the clustering columns, in the order they are defined
* @param included
*/
public MaterializedViewDefinition(String baseCfName, String viewName, List<ColumnIdentifier> partitionColumns, List<ColumnIdentifier> clusteringColumns, Set<ColumnIdentifier> included)
{
assert partitionColumns != null && !partitionColumns.isEmpty();
assert included != null;
this.baseCfName = baseCfName;
this.viewName = viewName;
this.partitionColumns = partitionColumns;
this.clusteringColumns = clusteringColumns;
this.includeAll = included.isEmpty();
this.included = included;
}
/**
* @return true if the view specified by this definition will include the column, false otherwise
*/
public boolean includes(ColumnIdentifier column)
{
return includeAll
|| partitionColumns.contains(column)
|| clusteringColumns.contains(column)
|| included.contains(column);
}
/**
* Replace the column {@param from} with {@param to} in this materialized view definition's partition,
* clustering, or included columns.
*/
public void renameColumn(ColumnIdentifier from, ColumnIdentifier to)
{
if (!includeAll && included.contains(from))
{
included.remove(from);
included.add(to);
}
int partitionIndex = partitionColumns.indexOf(from);
if (partitionIndex >= 0)
partitionColumns.set(partitionIndex, to);
int clusteringIndex = clusteringColumns.indexOf(from);
if (clusteringIndex >= 0)
clusteringColumns.set(clusteringIndex, to);
}
}

143
src/java/org/apache/cassandra/cql3/Cql.g
View File

@ -233,46 +233,43 @@ query returns [ParsedStatement stmnt]
cqlStatement returns [ParsedStatement stmt]
@after{ if (stmt != null) stmt.setBoundVariables(bindVariables); }
: st1= selectStatement { $stmt = st1; }
| st2= insertStatement { $stmt = st2; }
| st3= updateStatement { $stmt = st3; }
| st4= batchStatement { $stmt = st4; }
| st5= deleteStatement { $stmt = st5; }
| st6= useStatement { $stmt = st6; }
| st7= truncateStatement { $stmt = st7; }
| st8= createKeyspaceStatement { $stmt = st8; }
| st9= createTableStatement { $stmt = st9; }
| st10=createIndexStatement { $stmt = st10; }
| st11=dropKeyspaceStatement { $stmt = st11; }
| st12=dropTableStatement { $stmt = st12; }
| st13=dropIndexStatement { $stmt = st13; }
| st14=alterTableStatement { $stmt = st14; }
| st15=alterKeyspaceStatement { $stmt = st15; }
| st16=grantPermissionsStatement { $stmt = st16; }
| st17=revokePermissionsStatement { $stmt = st17; }
| st18=listPermissionsStatement { $stmt = st18; }
| st19=createUserStatement { $stmt = st19; }
| st20=alterUserStatement { $stmt = st20; }
| st21=dropUserStatement { $stmt = st21; }
| st22=listUsersStatement { $stmt = st22; }
| st23=createTriggerStatement { $stmt = st23; }
| st24=dropTriggerStatement { $stmt = st24; }
| st25=createTypeStatement { $stmt = st25; }
| st26=alterTypeStatement { $stmt = st26; }
| st27=dropTypeStatement { $stmt = st27; }
| st28=createFunctionStatement { $stmt = st28; }
| st29=dropFunctionStatement { $stmt = st29; }
| st30=createAggregateStatement { $stmt = st30; }
| st31=dropAggregateStatement { $stmt = st31; }
| st32=createRoleStatement { $stmt = st32; }
| st33=alterRoleStatement { $stmt = st33; }
| st34=dropRoleStatement { $stmt = st34; }
| st35=listRolesStatement { $stmt = st35; }
| st36=grantRoleStatement { $stmt = st36; }
| st37=revokeRoleStatement { $stmt = st37; }
| st38=createMaterializedViewStatement { $stmt = st38; }
| st39=dropMaterializedViewStatement { $stmt = st39; }
| st40=alterMaterializedViewStatement { $stmt = st40; }
: st1= selectStatement { $stmt = st1; }
| st2= insertStatement { $stmt = st2; }
| st3= updateStatement { $stmt = st3; }
| st4= batchStatement { $stmt = st4; }
| st5= deleteStatement { $stmt = st5; }
| st6= useStatement { $stmt = st6; }
| st7= truncateStatement { $stmt = st7; }
| st8= createKeyspaceStatement { $stmt = st8; }
| st9= createTableStatement { $stmt = st9; }
| st10=createIndexStatement { $stmt = st10; }
| st11=dropKeyspaceStatement { $stmt = st11; }
| st12=dropTableStatement { $stmt = st12; }
| st13=dropIndexStatement { $stmt = st13; }
| st14=alterTableStatement { $stmt = st14; }
| st15=alterKeyspaceStatement { $stmt = st15; }
| st16=grantPermissionsStatement { $stmt = st16; }
| st17=revokePermissionsStatement { $stmt = st17; }
| st18=listPermissionsStatement { $stmt = st18; }
| st19=createUserStatement { $stmt = st19; }
| st20=alterUserStatement { $stmt = st20; }
| st21=dropUserStatement { $stmt = st21; }
| st22=listUsersStatement { $stmt = st22; }
| st23=createTriggerStatement { $stmt = st23; }
| st24=dropTriggerStatement { $stmt = st24; }
| st25=createTypeStatement { $stmt = st25; }
| st26=alterTypeStatement { $stmt = st26; }
| st27=dropTypeStatement { $stmt = st27; }
| st28=createFunctionStatement { $stmt = st28; }
| st29=dropFunctionStatement { $stmt = st29; }
| st30=createAggregateStatement { $stmt = st30; }
| st31=dropAggregateStatement { $stmt = st31; }
| st32=createRoleStatement { $stmt = st32; }
| st33=alterRoleStatement { $stmt = st33; }
| st34=dropRoleStatement { $stmt = st34; }
| st35=listRolesStatement { $stmt = st35; }
| st36=grantRoleStatement { $stmt = st36; }
| st37=revokeRoleStatement { $stmt = st37; }
;
/*
@ -668,7 +665,7 @@ createTableStatement returns [CreateTableStatement.RawStatement expr]
cfamDefinition[CreateTableStatement.RawStatement expr]
: '(' cfamColumns[expr] ( ',' cfamColumns[expr]? )* ')'
( K_WITH cfamProperty[expr.properties] ( K_AND cfamProperty[expr.properties] )*)?
( K_WITH cfamProperty[expr] ( K_AND cfamProperty[expr] )*)?
;
cfamColumns[CreateTableStatement.RawStatement expr]
@ -682,15 +679,15 @@ pkDef[CreateTableStatement.RawStatement expr]
| '(' { List<ColumnIdentifier> l = new ArrayList<ColumnIdentifier>(); } k1=ident { l.add(k1); } ( ',' kn=ident { l.add(kn); } )* ')' { $expr.addKeyAliases(l); }
;
cfamProperty[CFProperties props]
: property[props.properties]
| K_COMPACT K_STORAGE { $props.setCompactStorage(); }
| K_CLUSTERING K_ORDER K_BY '(' cfamOrdering[props] (',' cfamOrdering[props])* ')'
cfamProperty[CreateTableStatement.RawStatement expr]
: property[expr.properties]
| K_COMPACT K_STORAGE { $expr.setCompactStorage(); }
| K_CLUSTERING K_ORDER K_BY '(' cfamOrdering[expr] (',' cfamOrdering[expr])* ')'
;
cfamOrdering[CFProperties props]
cfamOrdering[CreateTableStatement.RawStatement expr]
@init{ boolean reversed=false; }
: k=ident (K_ASC | K_DESC { reversed=true;} ) { $props.setOrdering(k, reversed); }
: k=ident (K_ASC | K_DESC { reversed=true;} ) { $expr.setOrdering(k, reversed); }
;
@ -737,34 +734,6 @@ indexIdent returns [IndexTarget.Raw id]
| K_FULL '(' c=cident ')' { $id = IndexTarget.Raw.fullCollection(c); }
;
/**
* CREATE MATERIALIZED VIEW <viewName> AS
* SELECT <columns>
* FROM <CF>
* WHERE <pkColumns> IS NOT NULL
* PRIMARY KEY (<pkColumns>)
* WITH <property> = <value> AND ...;
*/
createMaterializedViewStatement returns [CreateMaterializedViewStatement expr]
@init {
boolean ifNotExists = false;
List<ColumnIdentifier.Raw> partitionKeys = new ArrayList<>();
List<ColumnIdentifier.Raw> compositeKeys = new ArrayList<>();
}
: K_CREATE K_MATERIALIZED K_VIEW (K_IF K_NOT K_EXISTS { ifNotExists = true; })? cf=columnFamilyName K_AS
K_SELECT sclause=selectClause K_FROM basecf=columnFamilyName
(K_WHERE wclause=mvWhereClause)?
K_PRIMARY K_KEY (
'(' '(' k1=cident { partitionKeys.add(k1); } ( ',' kn=cident { partitionKeys.add(kn); } )* ')' ( ',' c1=cident { compositeKeys.add(c1); } )* ')'
| '(' k1=cident { partitionKeys.add(k1); } ( ',' cn=cident { compositeKeys.add(cn); } )* ')'
)
{ $expr = new CreateMaterializedViewStatement(cf, basecf, sclause, wclause, partitionKeys, compositeKeys, ifNotExists); }
( K_WITH cfamProperty[expr.properties] ( K_AND cfamProperty[expr.properties] )*)?
;
mvWhereClause returns [List<ColumnIdentifier.Raw> expr]
: t1=cident { $expr = new ArrayList<ColumnIdentifier.Raw>(); $expr.add(t1); } K_IS K_NOT K_NULL (K_AND tN=cident { $expr.add(tN); } K_IS K_NOT K_NULL)*
;
/**
* CREATE TRIGGER triggerName ON columnFamily USING 'triggerClass';
@ -825,18 +794,6 @@ alterTableStatement returns [AlterTableStatement expr]
}
;
alterMaterializedViewStatement returns [AlterMaterializedViewStatement expr]
@init {
CFPropDefs props = new CFPropDefs();
}
: K_ALTER K_MATERIALIZED K_VIEW name=columnFamilyName
K_WITH properties[props]
{
$expr = new AlterMaterializedViewStatement(name, props);
}
;
/**
* ALTER TYPE <name> ALTER <field> TYPE <newtype>;
* ALTER TYPE <name> ADD <field> <newtype>;
@ -888,15 +845,6 @@ dropIndexStatement returns [DropIndexStatement expr]
{ $expr = new DropIndexStatement(index, ifExists); }
;
/**
* DROP MATERIALIZED VIEW [IF EXISTS] <view_name>
*/
dropMaterializedViewStatement returns [DropMaterializedViewStatement expr]
@init { boolean ifExists = false; }
: K_DROP K_MATERIALIZED K_VIEW (K_IF K_EXISTS { ifExists = true; } )? cf=columnFamilyName
{ $expr = new DropMaterializedViewStatement(cf, ifExists); }
;
/**
* TRUNCATE <CF>;
*/
@ -1651,8 +1599,6 @@ K_KEYSPACE: ( K E Y S P A C E
K_KEYSPACES: K E Y S P A C E S;
K_COLUMNFAMILY:( C O L U M N F A M I L Y
| T A B L E );
K_MATERIALIZED:M A T E R I A L I Z E D;
K_VIEW: V I E W;
K_INDEX: I N D E X;
K_CUSTOM: C U S T O M;
K_ON: O N;
@ -1676,7 +1622,6 @@ K_DESC: D E S C;
K_ALLOW: A L L O W;
K_FILTERING: F I L T E R I N G;
K_IF: I F;
K_IS: I S;
K_CONTAINS: C O N T A I N S;
K_GRANT: G R A N T;

83
src/java/org/apache/cassandra/cql3/statements/AlterMaterializedViewStatement.java
View File

@ -1,83 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3.statements;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CFName;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.transport.Event;
import static org.apache.cassandra.thrift.ThriftValidation.validateColumnFamily;
public class AlterMaterializedViewStatement extends SchemaAlteringStatement
{
private final CFPropDefs cfProps;
public AlterMaterializedViewStatement(CFName name,
CFPropDefs cfProps)
{
super(name);
this.cfProps = cfProps;
}
public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
{
state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.ALTER);
}
public void validate(ClientState state)
{
// validated in announceMigration()
}
public boolean announceMigration(boolean isLocalOnly) throws RequestValidationException
{
CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
if (!meta.isMaterializedView())
throw new InvalidRequestException("Cannot use ALTER MATERIALIZED VIEW on Table");
CFMetaData cfm = meta.copy();
if (cfProps == null)
throw new InvalidRequestException("ALTER MATERIALIZED VIEW WITH invoked, but no parameters found");
cfProps.validate();
cfProps.applyToCFMetadata(cfm);
MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
return true;
}
public String toString()
{
return String.format("AlterMaterializedViewStatement(name=%s)", cfName);
}
public Event.SchemaChange changeEvent()
{
return new Event.SchemaChange(Event.SchemaChange.Change.UPDATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
}
}

85
src/java/org/apache/cassandra/cql3/statements/AlterTableStatement.java
View File

@ -17,15 +17,11 @@
*/
package org.apache.cassandra.cql3.statements;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.exceptions.*;
@ -79,9 +75,6 @@ public class AlterTableStatement extends SchemaAlteringStatement
public boolean announceMigration(boolean isLocalOnly) throws RequestValidationException
{
CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
if (meta.isMaterializedView())
throw new InvalidRequestException("Cannot use ALTER TABLE on Materialized View");
CFMetaData cfm = meta.copy();
CQL3Type validator = this.validator == null ? null : this.validator.prepare(keyspace());
@ -93,8 +86,6 @@ public class AlterTableStatement extends SchemaAlteringStatement
def = cfm.getColumnDefinition(columnName);
}
List<CFMetaData> materializedViewUpdates = null;
switch (oType)
{
case ADD:
@ -153,22 +144,6 @@ public class AlterTableStatement extends SchemaAlteringStatement
cfm.addColumnDefinition(isStatic
? ColumnDefinition.staticDef(cfm, columnName.bytes, type)
: ColumnDefinition.regularDef(cfm, columnName.bytes, type));
// Adding a column to a table which has an include all materialized view requires the column to be added
// to the materialized view as well
for (MaterializedViewDefinition mv : cfm.getMaterializedViews())
{
if (mv.includeAll)
{
CFMetaData indexCfm = Schema.instance.getCFMetaData(keyspace(), mv.viewName).copy();
indexCfm.addColumnDefinition(isStatic
? ColumnDefinition.staticDef(indexCfm, columnName.bytes, type)
: ColumnDefinition.regularDef(indexCfm, columnName.bytes, type));
if (materializedViewUpdates == null)
materializedViewUpdates = new ArrayList<>();
materializedViewUpdates.add(indexCfm);
}
}
break;
case ALTER:
@ -218,19 +193,6 @@ public class AlterTableStatement extends SchemaAlteringStatement
}
// In any case, we update the column definition
cfm.addOrReplaceColumnDefinition(def.withNewType(validatorType));
// We have to alter the schema of the materialized view table as well; it doesn't affect the definition however
for (MaterializedViewDefinition mv : cfm.getMaterializedViews())
{
if (!mv.includes(columnName)) continue;
// We have to use the pre-adjusted CFM, otherwise we can't resolve the Index
CFMetaData indexCfm = Schema.instance.getCFMetaData(keyspace(), mv.viewName).copy();
indexCfm.addOrReplaceColumnDefinition(def.withNewType(validatorType));
if (materializedViewUpdates == null)
materializedViewUpdates = new ArrayList<>();
materializedViewUpdates.add(indexCfm);
}
break;
case DROP:
@ -261,27 +223,6 @@ public class AlterTableStatement extends SchemaAlteringStatement
cfm.recordColumnDrop(toDelete);
break;
}
// If a column is dropped which is the target of a materialized view,
// then we need to drop the view.
// If a column is dropped which was selected into a materialized view,
// we need to drop that column from the included materialzied view table
// and definition.
boolean rejectAlter = false;
StringBuilder builder = new StringBuilder();
for (MaterializedViewDefinition mv : cfm.getMaterializedViews())
{
if (!mv.includes(columnName)) continue;
if (rejectAlter)
builder.append(',');
rejectAlter = true;
builder.append(mv.viewName);
}
if (rejectAlter)
throw new InvalidRequestException(String.format("Cannot drop column %s, depended on by materialized views (%s.{%s})",
columnName.toString(),
keyspace(),
builder.toString()));
break;
case OPTS:
if (cfProps == null)
@ -300,36 +241,10 @@ public class AlterTableStatement extends SchemaAlteringStatement
ColumnIdentifier from = entry.getKey().prepare(cfm);
ColumnIdentifier to = entry.getValue().prepare(cfm);
cfm.renameColumn(from, to);
// If the materialized view includes a renamed column, it must be renamed in the index table and the definition.
for (MaterializedViewDefinition mv : cfm.getMaterializedViews())
{
if (!mv.includes(from)) continue;
CFMetaData indexCfm = Schema.instance.getCFMetaData(keyspace(), mv.viewName).copy();
ColumnIdentifier indexFrom = entry.getKey().prepare(indexCfm);
ColumnIdentifier indexTo = entry.getValue().prepare(indexCfm);
indexCfm.renameColumn(indexFrom, indexTo);
MaterializedViewDefinition mvCopy = new MaterializedViewDefinition(mv);
mvCopy.renameColumn(from, to);
cfm.materializedViews(cfm.getMaterializedViews().replace(mvCopy));
if (materializedViewUpdates == null)
materializedViewUpdates = new ArrayList<>();
materializedViewUpdates.add(indexCfm);
}
}
break;
}
if (materializedViewUpdates != null)
{
for (CFMetaData mvUpdates : materializedViewUpdates)
MigrationManager.announceColumnFamilyUpdate(mvUpdates, false, isLocalOnly);
}
MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
return true;
}

58
src/java/org/apache/cassandra/cql3/statements/CFProperties.java
View File

@ -1,58 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3.statements;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ReversedType;
public class CFProperties
{
public final CFPropDefs properties = new CFPropDefs();
final Map<ColumnIdentifier, Boolean> definedOrdering = new LinkedHashMap<>(); // Insertion ordering is important
boolean useCompactStorage = false;
public void validate()
{
properties.validate();
}
public void setOrdering(ColumnIdentifier alias, boolean reversed)
{
definedOrdering.put(alias, reversed);
}
public void setCompactStorage()
{
useCompactStorage = true;
}
public AbstractType getReversableType(ColumnIdentifier targetIdentifier, AbstractType<?> type)
{
if (!definedOrdering.containsKey(targetIdentifier))
{
return type;
}
return definedOrdering.get(targetIdentifier) ? ReversedType.getInstance(type) : type;
}
}

268
src/java/org/apache/cassandra/cql3/statements/CreateMaterializedViewStatement.java
View File

@ -1,268 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3.statements;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.google.common.collect.Iterables;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.cql3.CFName;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.selection.RawSelector;
import org.apache.cassandra.cql3.selection.Selectable;
import org.apache.cassandra.db.view.MaterializedView;
import org.apache.cassandra.exceptions.AlreadyExistsException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.RequestValidationException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.ClientWarn;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.thrift.ThriftValidation;
import org.apache.cassandra.transport.Event;
public class CreateMaterializedViewStatement extends SchemaAlteringStatement
{
private final CFName baseName;
private final List<RawSelector> selectClause;
private final List<ColumnIdentifier.Raw> notNullWhereClause;
private final List<ColumnIdentifier.Raw> partitionKeys;
private final List<ColumnIdentifier.Raw> clusteringKeys;
public final CFProperties properties = new CFProperties();
private final boolean ifNotExists;
public CreateMaterializedViewStatement(CFName viewName,
CFName baseName,
List<RawSelector> selectClause,
List<ColumnIdentifier.Raw> notNullWhereClause,
List<ColumnIdentifier.Raw> partitionKeys,
List<ColumnIdentifier.Raw> clusteringKeys,
boolean ifNotExists)
{
super(viewName);
this.baseName = baseName;
this.selectClause = selectClause;
this.notNullWhereClause = notNullWhereClause;
this.partitionKeys = partitionKeys;
this.clusteringKeys = clusteringKeys;
this.ifNotExists = ifNotExists;
}
public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
{
if (!baseName.hasKeyspace())
baseName.setKeyspace(keyspace(), true);
state.hasKeyspaceAccess(keyspace(), Permission.CREATE);
}
public void validate(ClientState state) throws RequestValidationException
{
// We do validation in announceMigration to reduce doubling up of work
}
public boolean announceMigration(boolean isLocalOnly) throws RequestValidationException
{
// We need to make sure that:
// - primary key includes all columns in base table's primary key
// - make sure that the select statement does not have anything other than columns
// and their names match the base table's names
// - make sure that primary key does not include any collections
// - make sure there is no where clause in the select statement
// - make sure there is not currently a table or view
properties.validate();
if (properties.useCompactStorage)
throw new InvalidRequestException("Cannot use 'COMPACT STORAGE' when defining a materialized view");
// We enforce the keyspace because if the RF is different, the logic to wait for a
// specific replica would break
if (!baseName.getKeyspace().equals(keyspace()))
throw new InvalidRequestException("Cannot create a materialized view on a table in a separate keyspace");
CFMetaData cfm = ThriftValidation.validateColumnFamily(baseName.getKeyspace(), baseName.getColumnFamily());
if (cfm.isCounter())
throw new InvalidRequestException("Materialized views are not supported on counter tables");
if (cfm.isMaterializedView())
throw new InvalidRequestException("Materialized views cannot be created against other materialized views");
Set<ColumnIdentifier> included = new HashSet<>();
for (RawSelector selector : selectClause)
{
Selectable.Raw selectable = selector.selectable;
if (selectable instanceof Selectable.WithFieldSelection.Raw)
throw new InvalidRequestException("Cannot select out a part of type when defining a materialized view");
if (selectable instanceof Selectable.WithFunction.Raw)
throw new InvalidRequestException("Cannot use function when defining a materialized view");
if (selectable instanceof Selectable.WritetimeOrTTL.Raw)
throw new InvalidRequestException("Cannot use function when defining a materialized view");
ColumnIdentifier identifier = (ColumnIdentifier) selectable.prepare(cfm);
if (selector.alias != null)
throw new InvalidRequestException(String.format("Cannot alias column '%s' as '%s' when defining a materialized view", identifier.toString(), selector.alias.toString()));
ColumnDefinition cdef = cfm.getColumnDefinition(identifier);
if (cdef == null)
throw new InvalidRequestException("Unknown column name detected in CREATE MATERIALIZED VIEW statement : "+identifier);
if (cdef.isStatic())
ClientWarn.warn(String.format("Unable to include static column '%s' in Materialized View SELECT statement", identifier));
else
included.add(identifier);
}
Set<ColumnIdentifier.Raw> targetPrimaryKeys = new HashSet<>();
for (ColumnIdentifier.Raw identifier : Iterables.concat(partitionKeys, clusteringKeys))
{
if (!targetPrimaryKeys.add(identifier))
throw new InvalidRequestException("Duplicate entry found in PRIMARY KEY: "+identifier);
ColumnDefinition cdef = cfm.getColumnDefinition(identifier.prepare(cfm));
if (cdef == null)
throw new InvalidRequestException("Unknown column name detected in CREATE MATERIALIZED VIEW statement : "+identifier);
if (cfm.getColumnDefinition(identifier.prepare(cfm)).type.isMultiCell())
throw new InvalidRequestException(String.format("Cannot use MultiCell column '%s' in PRIMARY KEY of materialized view", identifier));
if (cdef.isStatic())
throw new InvalidRequestException(String.format("Cannot use Static column '%s' in PRIMARY KEY of materialized view", identifier));
}
Set<ColumnIdentifier> basePrimaryKeyCols = new HashSet<>();
for (ColumnDefinition definition : Iterables.concat(cfm.partitionKeyColumns(), cfm.clusteringColumns()))
basePrimaryKeyCols.add(definition.name);
List<ColumnIdentifier> targetClusteringColumns = new ArrayList<>();
List<ColumnIdentifier> targetPartitionKeys = new ArrayList<>();
Set<ColumnIdentifier> notNullColumns = new HashSet<>();
if (notNullWhereClause != null)
{
for (ColumnIdentifier.Raw raw : notNullWhereClause)
{
notNullColumns.add(raw.prepare(cfm));
}
}
// This is only used as an intermediate state; this is to catch whether multiple non-PK columns are used
boolean hasNonPKColumn = false;
for (ColumnIdentifier.Raw raw : partitionKeys)
{
hasNonPKColumn = getColumnIdentifier(cfm, basePrimaryKeyCols, hasNonPKColumn, raw, targetPartitionKeys, notNullColumns);
}
for (ColumnIdentifier.Raw raw : clusteringKeys)
{
hasNonPKColumn = getColumnIdentifier(cfm, basePrimaryKeyCols, hasNonPKColumn, raw, targetClusteringColumns, notNullColumns);
}
// We need to include all of the primary key colums from the base table in order to make sure that we do not
// overwrite values in the materialized view. We cannot support "collapsing" the base table into a smaller
// number of rows in the view because if we need to generate a tombstone, we have no way of knowing which value
// is currently being used in the view and whether or not to generate a tombstone.
// In order to not surprise our users, we require that they include all of the columns. We provide them with
// a list of all of the columns left to include.
boolean missingClusteringColumns = false;
StringBuilder columnNames = new StringBuilder();
for (ColumnDefinition def : cfm.allColumns())
{
if (!def.isPrimaryKeyColumn()) continue;
ColumnIdentifier identifier = def.name;
if (!targetClusteringColumns.contains(identifier) && !targetPartitionKeys.contains(identifier))
{
if (missingClusteringColumns)
columnNames.append(',');
else
missingClusteringColumns = true;
columnNames.append(identifier);
}
}
if (missingClusteringColumns)
throw new InvalidRequestException(String.format("Cannot create Materialized View %s without primary key columns from base %s (%s)",
columnFamily(), baseName.getColumnFamily(), columnNames.toString()));
if (targetPartitionKeys.isEmpty())
throw new InvalidRequestException("Must select at least a column for a Materialized View");
if (targetClusteringColumns.isEmpty())
throw new InvalidRequestException("No columns are defined for Materialized View other than primary key");
MaterializedViewDefinition definition = new MaterializedViewDefinition(baseName.getColumnFamily(),
columnFamily(),
targetPartitionKeys,
targetClusteringColumns,
included);
CFMetaData indexCf = MaterializedView.getCFMetaData(definition, cfm, properties);
try
{
MigrationManager.announceNewColumnFamily(indexCf, isLocalOnly);
}
catch (AlreadyExistsException e)
{
if (ifNotExists)
return false;
throw e;
}
CFMetaData newCfm = cfm.copy();
newCfm.materializedViews(newCfm.getMaterializedViews().with(definition));
MigrationManager.announceColumnFamilyUpdate(newCfm, false, isLocalOnly);
return true;
}
private static boolean getColumnIdentifier(CFMetaData cfm,
Set<ColumnIdentifier> basePK,
boolean hasNonPKColumn,
ColumnIdentifier.Raw raw,
List<ColumnIdentifier> columns,
Set<ColumnIdentifier> allowedPKColumns)
{
ColumnIdentifier identifier = raw.prepare(cfm);
boolean isPk = basePK.contains(identifier);
if (!isPk && hasNonPKColumn)
{
throw new InvalidRequestException(String.format("Cannot include more than one non-primary key column '%s' in materialized view partition key", identifier));
}
if (!allowedPKColumns.contains(identifier))
{
throw new InvalidRequestException(String.format("Primary key column '%s' is required to be filtered by 'IS NOT NULL'", identifier));
}
columns.add(identifier);
return !isPk;
}
public Event.SchemaChange changeEvent()
{
return new Event.SchemaChange(Event.SchemaChange.Change.CREATED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
}
}

29
src/java/org/apache/cassandra/cql3/statements/CreateTableStatement.java
View File

@ -188,12 +188,14 @@ public class CreateTableStatement extends SchemaAlteringStatement
public static class RawStatement extends CFStatement
{
private final Map<ColumnIdentifier, CQL3Type.Raw> definitions = new HashMap<>();
public final CFProperties properties = new CFProperties();
public final CFPropDefs properties = new CFPropDefs();
private final List<List<ColumnIdentifier>> keyAliases = new ArrayList<List<ColumnIdentifier>>();
private final List<ColumnIdentifier> columnAliases = new ArrayList<ColumnIdentifier>();
private final Map<ColumnIdentifier, Boolean> definedOrdering = new LinkedHashMap<ColumnIdentifier, Boolean>(); // Insertion ordering is important
private final Set<ColumnIdentifier> staticColumns = new HashSet<ColumnIdentifier>();
private boolean useCompactStorage;
private final Multiset<ColumnIdentifier> definedNames = HashMultiset.create(1);
private final boolean ifNotExists;
@ -221,7 +223,7 @@ public class CreateTableStatement extends SchemaAlteringStatement
properties.validate();
CreateTableStatement stmt = new CreateTableStatement(cfName, properties.properties, ifNotExists, staticColumns);
CreateTableStatement stmt = new CreateTableStatement(cfName, properties, ifNotExists, staticColumns);
for (Map.Entry<ColumnIdentifier, CQL3Type.Raw> entry : definitions.entrySet())
{
@ -238,7 +240,7 @@ public class CreateTableStatement extends SchemaAlteringStatement
throw new InvalidRequestException("No PRIMARY KEY specifed (exactly one required)");
if (keyAliases.size() > 1)
throw new InvalidRequestException("Multiple PRIMARY KEYs specifed (exactly one required)");
if (stmt.hasCounters && properties.properties.getDefaultTimeToLive() > 0)
if (stmt.hasCounters && properties.getDefaultTimeToLive() > 0)
throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters");
List<ColumnIdentifier> kAliases = keyAliases.get(0);
@ -277,7 +279,6 @@ public class CreateTableStatement extends SchemaAlteringStatement
throw new InvalidRequestException("Cannot mix counter and non counter columns in the same table");
}
boolean useCompactStorage = properties.useCompactStorage;
// Dense means that on the thrift side, no part of the "thrift column name" stores a "CQL/metadata column name".
// This means COMPACT STORAGE with at least one clustering type (otherwise it's a thrift "static" CF).
stmt.isDense = useCompactStorage && !stmt.clusteringTypes.isEmpty();
@ -326,18 +327,18 @@ public class CreateTableStatement extends SchemaAlteringStatement
}
// If we give a clustering order, we must explicitly do so for all aliases and in the order of the PK
if (!properties.definedOrdering.isEmpty())
if (!definedOrdering.isEmpty())
{
if (properties.definedOrdering.size() > columnAliases.size())
if (definedOrdering.size() > columnAliases.size())
throw new InvalidRequestException("Only clustering key columns can be defined in CLUSTERING ORDER directive");
int i = 0;
for (ColumnIdentifier id : properties.definedOrdering.keySet())
for (ColumnIdentifier id : definedOrdering.keySet())
{
ColumnIdentifier c = columnAliases.get(i);
if (!id.equals(c))
{
if (properties.definedOrdering.containsKey(c))
if (definedOrdering.containsKey(c))
throw new InvalidRequestException(String.format("The order of columns in the CLUSTERING ORDER directive must be the one of the clustering key (%s must appear before %s)", c, id));
else
throw new InvalidRequestException(String.format("Missing CLUSTERING ORDER for column %s", c));
@ -358,7 +359,7 @@ public class CreateTableStatement extends SchemaAlteringStatement
throw new InvalidRequestException(String.format("Invalid collection type for PRIMARY KEY component %s", t));
columns.remove(t);
Boolean isReversed = properties.definedOrdering.get(t);
Boolean isReversed = definedOrdering.get(t);
return isReversed != null && isReversed ? ReversedType.getInstance(type) : type;
}
@ -379,5 +380,15 @@ public class CreateTableStatement extends SchemaAlteringStatement
{
columnAliases.add(alias);
}
public void setOrdering(ColumnIdentifier alias, boolean reversed)
{
definedOrdering.put(alias, reversed);
}
public void setCompactStorage()
{
useCompactStorage = true;
}
}
}

5
src/java/org/apache/cassandra/cql3/statements/CreateTriggerStatement.java
View File

@ -58,10 +58,7 @@ public class CreateTriggerStatement extends SchemaAlteringStatement
public void validate(ClientState state) throws RequestValidationException
{
CFMetaData cfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
if (cfm.isMaterializedView())
throw new InvalidRequestException("Cannot CREATE TRIGGER against a materialized view");
ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
try
{
TriggerExecutor.instance.loadTriggerInstance(triggerClass);

105
src/java/org/apache/cassandra/cql3/statements/DropMaterializedViewStatement.java
View File

@ -1,105 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3.statements;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CFName;
import org.apache.cassandra.db.KeyspaceNotDefinedException;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.cassandra.exceptions.UnauthorizedException;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.transport.Event;
public class DropMaterializedViewStatement extends SchemaAlteringStatement
{
public final boolean ifExists;
public DropMaterializedViewStatement(CFName cf, boolean ifExists)
{
super(cf);
this.ifExists = ifExists;
}
public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException
{
state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.DROP);
}
public void validate(ClientState state)
{
// validated in findIndexedCf()
}
public Event.SchemaChange changeEvent()
{
return new Event.SchemaChange(Event.SchemaChange.Change.DROPPED, Event.SchemaChange.Target.TABLE, keyspace(), columnFamily());
}
public boolean announceMigration(boolean isLocalOnly) throws InvalidRequestException, ConfigurationException
{
try
{
CFMetaData viewCfm = Schema.instance.getCFMetaData(keyspace(), columnFamily());
if (viewCfm == null)
throw new ConfigurationException(String.format("Cannot drop non existing materialized view '%s' in keyspace '%s'.", columnFamily(), keyspace()));
if (!viewCfm.isMaterializedView())
throw new ConfigurationException(String.format("Cannot drop non materialized view '%s' in keyspace '%s'", columnFamily(), keyspace()));
CFMetaData baseCfm = findBaseCf();
if (baseCfm == null)
throw new ConfigurationException(String.format("Cannot drop materialized view '%s' in keyspace '%s' without base CF.", columnFamily(), keyspace()));
CFMetaData updatedCfm = baseCfm.copy();
updatedCfm.materializedViews(updatedCfm.getMaterializedViews().without(columnFamily()));
MigrationManager.announceColumnFamilyUpdate(updatedCfm, false, isLocalOnly);
MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily(), isLocalOnly);
return true;
}
catch (ConfigurationException e)
{
if (ifExists)
return false;
throw e;
}
}
private CFMetaData findBaseCf() throws InvalidRequestException
{
KeyspaceMetadata ksm = Schema.instance.getKSMetaData(keyspace());
if (ksm == null)
throw new KeyspaceNotDefinedException("Keyspace " + keyspace() + " does not exist");
for (CFMetaData cfm : ksm.tables)
{
if (cfm.getMaterializedViews().get(columnFamily()).isPresent())
return cfm;
}
if (ifExists)
return null;
else
throw new InvalidRequestException("View '" + cfName + "' could not be found in any of the tables of keyspace '" + keyspace() + '\'');
}
}

25
src/java/org/apache/cassandra/cql3/statements/DropTableStatement.java
View File

@ -18,9 +18,6 @@
package org.apache.cassandra.cql3.statements;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.CFName;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
@ -61,28 +58,6 @@ public class DropTableStatement extends SchemaAlteringStatement
{
try
{
CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily());
if (cfm != null)
{
if (cfm.isMaterializedView())
throw new InvalidRequestException("Cannot use DROP TABLE on Materialized View");
boolean rejectDrop = false;
StringBuilder messageBuilder = new StringBuilder();
for (MaterializedViewDefinition def : cfm.getMaterializedViews())
{
if (rejectDrop)
messageBuilder.append(',');
rejectDrop = true;
messageBuilder.append(def.viewName);
}
if (rejectDrop)
{
throw new InvalidRequestException(String.format("Cannot drop table when materialized views still depend on it (%s.{%s})",
keyspace(),
messageBuilder.toString()));
}
}
MigrationManager.announceColumnFamilyDrop(keyspace(), columnFamily(), isLocalOnly);
return true;
}

23
src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
View File

@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
@ -156,16 +155,6 @@ public abstract class ModificationStatement implements CQLStatement
return cfm.isCounter();
}
public boolean isMaterializedView()
{
return cfm.isMaterializedView();
}
public boolean hasMaterializedViews()
{
return !cfm.getMaterializedViews().isEmpty();
}
public long getTimestamp(long now, QueryOptions options) throws InvalidRequestException
{
return attrs.getTimestamp(now, options);
@ -189,15 +178,6 @@ public abstract class ModificationStatement implements CQLStatement
if (hasConditions())
state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.SELECT);
// MV updates need to get the current state from the table, and might update the materialized views
// Require Permission.SELECT on the base table, and Permission.MODIFY on the views
if (hasMaterializedViews())
{
state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.SELECT);
for (MaterializedViewDefinition view : cfm.getMaterializedViews())
state.hasColumnFamilyAccess(keyspace(), view.viewName, Permission.MODIFY);
}
for (Function function : getFunctions())
state.ensureHasPermission(Permission.EXECUTE, function);
}
@ -212,9 +192,6 @@ public abstract class ModificationStatement implements CQLStatement
if (isCounter() && attrs.isTimeToLiveSet())
throw new InvalidRequestException("Cannot provide custom TTL for counter updates");
if (isMaterializedView())
throw new InvalidRequestException("Cannot directly modify a materialized view");
}
public void addOperation(Operation op)

9
src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
View File

@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
@ -685,10 +684,10 @@ public class SelectStatement implements CQLStatement
public static class RawStatement extends CFStatement
{
public final Parameters parameters;
public final List<RawSelector> selectClause;
public final List<Relation> whereClause;
public final Term.Raw limit;
private final Parameters parameters;
private final List<RawSelector> selectClause;
private final List<Relation> whereClause;
private final Term.Raw limit;
public RawStatement(CFName cfName, Parameters parameters, List<RawSelector> selectClause, List<Relation> whereClause, Term.Raw limit)
{

6
src/java/org/apache/cassandra/cql3/statements/TruncateStatement.java
View File

@ -21,8 +21,6 @@ import java.io.IOException;
import java.util.concurrent.TimeoutException;
import org.apache.cassandra.auth.Permission;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
@ -64,10 +62,6 @@ public class TruncateStatement extends CFStatement implements CQLStatement
{
try
{
CFMetaData metaData = Schema.instance.getCFMetaData(keyspace(), columnFamily());
if (metaData.isMaterializedView())
throw new InvalidRequestException("Cannot TRUNCATE materialized view directly; must truncate base table instead");
StorageProxy.truncateBlocking(keyspace(), columnFamily());
}
catch (UnavailableException | TimeoutException | IOException e)

58
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -53,7 +53,6 @@ import org.apache.cassandra.db.compaction.*;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.view.MaterializedViewManager;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.dht.Range;
@ -160,7 +159,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
private final AtomicInteger fileIndexGenerator = new AtomicInteger(0);
public final SecondaryIndexManager indexManager;
public final MaterializedViewManager materializedViewManager;
/* These are locally held copies to be changed from the config during runtime */
private volatile DefaultInteger minCompactionThreshold;
@ -197,7 +195,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
indexManager.reload();
materializedViewManager.reload();
// If the CF comparator has changed, we need to change the memtable,
// because the old one still aliases the previous comparator.
if (data.getView().getCurrentMemtable().initialComparator != metadata.comparator)
@ -334,7 +331,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
this.partitioner = partitioner;
this.directories = directories;
this.indexManager = new SecondaryIndexManager(this);
this.materializedViewManager = new MaterializedViewManager(this);
this.metric = new TableMetrics(this);
fileIndexGenerator.set(generation);
sampleLatencyNanos = DatabaseDescriptor.getReadRpcTimeout() / 2;
@ -455,7 +451,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
SystemKeyspace.removeTruncationRecord(metadata.cfId);
data.dropSSTables();
indexManager.invalidate();
materializedViewManager.invalidate();
invalidateCaches();
}
@ -577,10 +572,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
// must be called after all sstables are loaded since row cache merges all row versions
public void init()
public void initRowCache()
{
materializedViewManager.init();
if (!isRowCacheEnabled())
return;
@ -1813,7 +1806,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
cfs.data.reset();
return null;
}
}, true, false);
}, true);
}
}
@ -1841,16 +1834,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
// flush the CF being truncated before forcing the new segment
forceBlockingFlush();
materializedViewManager.forceBlockingFlush();
// sleep a little to make sure that our truncatedAt comes after any sstable
// that was part of the flushed we forced; otherwise on a tie, it won't get deleted.
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
}
else
{
dumpMemtable();
materializedViewManager.dumpMemtables();
// just nuke the memtable data w/o writing to disk first
synchronized (data)
{
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
postFlushExecutor.submit(flush.postFlush);
}
}
Runnable truncateRunnable = new Runnable()
@ -1870,32 +1866,17 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
for (SecondaryIndex index : indexManager.getIndexes())
index.truncateBlocking(truncatedAt);
materializedViewManager.truncateBlocking(truncatedAt);
SystemKeyspace.saveTruncationRecord(ColumnFamilyStore.this, truncatedAt, replayAfter);
logger.debug("cleaning out row cache");
invalidateCaches();
}
};
runWithCompactionsDisabled(Executors.callable(truncateRunnable), true, true);
runWithCompactionsDisabled(Executors.callable(truncateRunnable), true);
logger.debug("truncate complete");
}
/**
* Drops current memtable without flushing to disk. This should only be called when truncating a column family which is not durable.
*/
public void dumpMemtable()
{
synchronized (data)
{
final Flush flush = new Flush(true);
flushExecutor.execute(flush);
postFlushExecutor.submit(flush.postFlush);
}
}
public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation, boolean interruptViews)
public <V> V runWithCompactionsDisabled(Callable<V> callable, boolean interruptValidation)
{
// synchronize so that concurrent invocations don't re-enable compactions partway through unexpectedly,
// and so we only run one major compaction at a time
@ -1903,20 +1884,17 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
logger.debug("Cancelling in-progress compactions for {}", metadata.cfName);
Iterable<ColumnFamilyStore> selfWithAuxiliaryCfs = interruptViews
? Iterables.concat(concatWithIndexes(), materializedViewManager.allViewsCfs())
: concatWithIndexes();
for (ColumnFamilyStore cfs : selfWithAuxiliaryCfs)
Iterable<ColumnFamilyStore> selfWithIndexes = concatWithIndexes();
for (ColumnFamilyStore cfs : selfWithIndexes)
cfs.getCompactionStrategyManager().pause();
try
{
// interrupt in-progress compactions
CompactionManager.instance.interruptCompactionForCFs(selfWithAuxiliaryCfs, interruptValidation);
CompactionManager.instance.waitForCessation(selfWithAuxiliaryCfs);
CompactionManager.instance.interruptCompactionForCFs(selfWithIndexes, interruptValidation);
CompactionManager.instance.waitForCessation(selfWithIndexes);
// doublecheck that we finished, instead of timing out
for (ColumnFamilyStore cfs : selfWithAuxiliaryCfs)
for (ColumnFamilyStore cfs : selfWithIndexes)
{
if (!cfs.getTracker().getCompacting().isEmpty())
{
@ -1938,7 +1916,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
finally
{
for (ColumnFamilyStore cfs : selfWithAuxiliaryCfs)
for (ColumnFamilyStore cfs : selfWithIndexes)
cfs.getCompactionStrategyManager().resume();
}
}
@ -1958,7 +1936,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
};
return runWithCompactionsDisabled(callable, false, false);
return runWithCompactionsDisabled(callable, false);
}

77
src/java/org/apache/cassandra/db/Keyspace.java
View File

@ -23,15 +23,13 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.locks.Lock;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.commitlog.CommitLog;
@ -39,17 +37,14 @@ import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.index.SecondaryIndex;
import org.apache.cassandra.db.index.SecondaryIndexManager;
import org.apache.cassandra.db.view.MaterializedViewManager;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.SchemaKeyspace;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.metrics.KeyspaceMetrics;
@ -75,10 +70,7 @@ public class Keyspace
}
private volatile KeyspaceMetadata metadata;
//OpOrder is defined globally since we need to order writes across
//Keyspaces in the case of MaterializedViews (batchlog of MV mutations)
public static final OpOrder writeOrder = new OpOrder();
public final OpOrder writeOrder = new OpOrder();
/* ColumnFamilyStore per column family */
private final ConcurrentMap<UUID, ColumnFamilyStore> columnFamilyStores = new ConcurrentHashMap<>();
@ -130,7 +122,7 @@ public class Keyspace
// keyspace has to be constructed and in the cache before cacheRow can be called
for (ColumnFamilyStore cfs : keyspaceInstance.getColumnFamilyStores())
cfs.init();
cfs.initRowCache();
}
}
}
@ -360,14 +352,10 @@ public class Keyspace
// CFS being created for the first time, either on server startup or new CF being added.
// We don't worry about races here; startup is safe, and adding multiple idential CFs
// simultaneously is a "don't do that" scenario.
ColumnFamilyStore newCfs = ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables);
ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, newCfs);
ColumnFamilyStore oldCfs = columnFamilyStores.putIfAbsent(cfId, ColumnFamilyStore.createColumnFamilyStore(this, cfName, loadSSTables));
// CFS mbean instantiation will error out before we hit this, but in case that changes...
if (oldCfs != null)
throw new IllegalStateException("added multiple mappings for cf id " + cfId);
newCfs.init();
}
else
{
@ -392,41 +380,11 @@ public class Keyspace
* @param writeCommitLog false to disable commitlog append entirely
* @param updateIndexes false to disable index updates (used by CollationController "defragmenting")
*/
public void apply(final Mutation mutation, final boolean writeCommitLog, boolean updateIndexes)
public void apply(Mutation mutation, boolean writeCommitLog, boolean updateIndexes)
{
if (TEST_FAIL_WRITES && metadata.name.equals(TEST_FAIL_WRITES_KS))
throw new RuntimeException("Testing write failures");
Lock lock = null;
boolean requiresViewUpdate = updateIndexes && MaterializedViewManager.updatesAffectView(Collections.singleton(mutation), false);
if (requiresViewUpdate)
{
lock = MaterializedViewManager.acquireLockFor(mutation.key().getKey());
if (lock == null)
{
if ((System.currentTimeMillis() - mutation.createdAt) > DatabaseDescriptor.getWriteRpcTimeout())
{
logger.debug("Could not acquire lock for {}", ByteBufferUtil.bytesToHex(mutation.key().getKey()));
Tracing.trace("Could not acquire MV lock");
throw new WriteTimeoutException(WriteType.MATERIALIZED_VIEW, ConsistencyLevel.LOCAL_ONE, 0, 1);
}
else
{
//This MV update can't happen right now. so rather than keep this thread busy
// we will re-apply ourself to the queue and try again later
StageManager.getStage(Stage.MUTATION).execute(() -> {
if (writeCommitLog)
mutation.apply();
else
mutation.applyUnsafe();
});
return;
}
}
}
int nowInSec = FBUtilities.nowInSeconds();
try (OpOrder.Group opGroup = writeOrder.start())
{
@ -443,26 +401,10 @@ public class Keyspace
ColumnFamilyStore cfs = columnFamilyStores.get(upd.metadata().cfId);
if (cfs == null)
{
logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().cfId, upd.metadata().ksName, upd.metadata().cfName);
logger.error("Attempting to mutate non-existant table {}", upd.metadata().cfId);
continue;
}
if (requiresViewUpdate)
{
try
{
Tracing.trace("Create materialized view mutations from replica");
cfs.materializedViewManager.pushViewReplicaUpdates(upd.partitionKey().getKey(), upd);
}
catch (Exception e)
{
if (!(e instanceof WriteTimeoutException))
logger.warn("Encountered exception when creating materialized view mutations", e);
JVMStabilityInspector.inspectThrowable(e);
}
}
Tracing.trace("Adding to {} memtable", upd.metadata().cfName);
SecondaryIndexManager.Updater updater = updateIndexes
? cfs.indexManager.updaterFor(upd, opGroup, nowInSec)
@ -470,11 +412,6 @@ public class Keyspace
cfs.apply(upd, updater, opGroup, replayPosition);
}
}
finally
{
if (lock != null)
lock.unlock();
}
}
public AbstractReplicationStrategy getReplicationStrategy()

2
src/java/org/apache/cassandra/db/Mutation.java
View File

@ -54,8 +54,6 @@ public class Mutation implements IMutation
// map of column family id to mutations for that column family.
private final Map<UUID, PartitionUpdate> modifications;
// Time at which this mutation was instantiated
public final long createdAt = System.currentTimeMillis();
public Mutation(String keyspaceName, DecoratedKey key)
{
this(keyspaceName, key, new HashMap<UUID, PartitionUpdate>());

8
src/java/org/apache/cassandra/db/MutationVerbHandler.java
View File

@ -22,7 +22,6 @@ import java.io.IOError;
import java.io.IOException;
import java.net.InetAddress;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.util.FastByteArrayInputStream;
import org.apache.cassandra.net.*;
import org.apache.cassandra.tracing.Tracing;
@ -48,17 +47,10 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
replyTo = InetAddress.getByAddress(from);
}
try
{
message.payload.apply();
WriteResponse response = new WriteResponse();
Tracing.trace("Enqueuing response to {}", replyTo);
MessagingService.instance().sendReply(response.createMessage(), id, replyTo);
}
catch (WriteTimeoutException wto)
{
Tracing.trace("Payload application resulted in WriteTimeout, not replying");
}
}
/**

98
src/java/org/apache/cassandra/db/SystemKeyspace.java
View File

@ -101,8 +101,6 @@ public final class SystemKeyspace
public static final String SSTABLE_ACTIVITY = "sstable_activity";
public static final String SIZE_ESTIMATES = "size_estimates";
public static final String AVAILABLE_RANGES = "available_ranges";
public static final String MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS = "materializedviews_builds_in_progress";
public static final String BUILT_MATERIALIZEDVIEWS = "built_materializedviews";
@Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces";
@Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies";
@ -263,24 +261,6 @@ public final class SystemKeyspace
+ "ranges set<blob>,"
+ "PRIMARY KEY ((keyspace_name)))");
public static final CFMetaData MaterializedViewsBuildsInProgress =
compile(MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS,
"materialized views builds current progress",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "view_name text,"
+ "last_token varchar,"
+ "generation_number int,"
+ "PRIMARY KEY ((keyspace_name), view_name))");
public static final CFMetaData BuiltMaterializedViews =
compile(BUILT_MATERIALIZEDVIEWS,
"built materialized views",
"CREATE TABLE \"%s\" ("
+ "keyspace_name text,"
+ "view_name text,"
+ "PRIMARY KEY ((keyspace_name), view_name))");
@Deprecated
public static final CFMetaData LegacyKeyspaces =
compile(LEGACY_KEYSPACES,
@ -421,8 +401,6 @@ public final class SystemKeyspace
SSTableActivity,
SizeEstimates,
AvailableRanges,
MaterializedViewsBuildsInProgress,
BuiltMaterializedViews,
LegacyKeyspaces,
LegacyColumnfamilies,
LegacyColumns,
@ -515,82 +493,6 @@ public final class SystemKeyspace
return CompactionHistoryTabularData.from(queryResultSet);
}
public static boolean isViewBuilt(String keyspaceName, String viewName)
{
String req = "SELECT view_name FROM %s.\"%s\" WHERE keyspace_name=? AND view_name=?";
UntypedResultSet result = executeInternal(String.format(req, NAME, BUILT_MATERIALIZEDVIEWS), keyspaceName, viewName);
return !result.isEmpty();
}
public static void setMaterializedViewBuilt(String keyspaceName, String viewName)
{
String req = "INSERT INTO %s.\"%s\" (keyspace_name, view_name) VALUES (?, ?)";
executeInternal(String.format(req, NAME, BUILT_MATERIALIZEDVIEWS), keyspaceName, viewName);
forceBlockingFlush(BUILT_MATERIALIZEDVIEWS);
}
public static void setMaterializedViewRemoved(String keyspaceName, String viewName)
{
String buildReq = "DELETE FROM %S.%s WHERE keyspace_name = ? AND view_name = ?";
executeInternal(String.format(buildReq, NAME, MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS), keyspaceName, viewName);
forceBlockingFlush(MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS);
String builtReq = "DELETE FROM %s.\"%s\" WHERE keyspace_name = ? AND view_name = ?";
executeInternal(String.format(builtReq, NAME, BUILT_MATERIALIZEDVIEWS), keyspaceName, viewName);
forceBlockingFlush(BUILT_MATERIALIZEDVIEWS);
}
public static void beginMaterializedViewBuild(String ksname, String viewName, int generationNumber)
{
executeInternal(String.format("INSERT INTO system.%s (keyspace_name, view_name, generation_number) VALUES (?, ?, ?)", MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS),
ksname,
viewName,
generationNumber);
}
public static void finishMaterializedViewBuildStatus(String ksname, String viewName)
{
// We flush the view built first, because if we fail now, we'll restart at the last place we checkpointed
// materialized view build.
// If we flush the delete first, we'll have to restart from the beginning.
// Also, if the build succeeded, but the materialized view build failed, we will be able to skip the
// materialized view build check next boot.
setMaterializedViewBuilt(ksname, viewName);
forceBlockingFlush(BUILT_MATERIALIZEDVIEWS);
executeInternal(String.format("DELETE FROM system.%s WHERE keyspace_name = ? AND view_name = ?", MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS), ksname, viewName);
forceBlockingFlush(MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS);
}
public static void updateMaterializedViewBuildStatus(String ksname, String viewName, Token token)
{
String req = "INSERT INTO system.%s (keyspace_name, view_name, last_token) VALUES (?, ?, ?)";
Token.TokenFactory factory = StorageService.getPartitioner().getTokenFactory();
executeInternal(String.format(req, MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS), ksname, viewName, factory.toString(token));
}
public static Pair<Integer, Token> getMaterializedViewBuildStatus(String ksname, String viewName)
{
String req = "SELECT generation_number, last_token FROM system.%s WHERE keyspace_name = ? AND view_name = ?";
UntypedResultSet queryResultSet = executeInternal(String.format(req, MATERIALIZEDVIEWS_BUILDS_IN_PROGRESS), ksname, viewName);
if (queryResultSet == null || queryResultSet.isEmpty())
return null;
UntypedResultSet.Row row = queryResultSet.one();
Integer generation = null;
Token lastKey = null;
if (row.has("generation_number"))
generation = row.getInt("generation_number");
if (row.has("last_key"))
{
Token.TokenFactory factory = StorageService.getPartitioner().getTokenFactory();
lastKey = factory.fromString(row.getString("last_key"));
}
return Pair.create(generation, lastKey);
}
public static synchronized void saveTruncationRecord(ColumnFamilyStore cfs, long truncatedAt, ReplayPosition position)
{
String req = "UPDATE system.%s SET truncated_at = truncated_at + ? WHERE key = '%s'";

3
src/java/org/apache/cassandra/db/WriteType.java
View File

@ -24,6 +24,5 @@ public enum WriteType
UNLOGGED_BATCH,
COUNTER,
BATCH_LOG,
CAS,
MATERIALIZED_VIEW;
CAS;
}

26
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -58,7 +58,6 @@ import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionInfo.Holder;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.index.SecondaryIndexBuilder;
import org.apache.cassandra.db.view.MaterializedViewBuilder;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
@ -1366,31 +1365,6 @@ public class CompactionManager implements CompactionManagerMBean
}
}
public Future<?> submitMaterializedViewBuilder(final MaterializedViewBuilder builder)
{
Runnable runnable = new Runnable()
{
public void run()
{
metrics.beginCompaction(builder);
try
{
builder.run();
}
finally
{
metrics.finishCompaction(builder);
}
}
};
if (executor.isShutdown())
{
logger.info("Compaction executor has shut down, not submitting index build");
return null;
}
return executor.submit(runnable);
}
public int getActiveCompactions()
{
return CompactionMetrics.getCompactions().size();

2
src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
View File

@ -426,7 +426,7 @@ public class CompactionStrategyManager implements INotificationConsumer
return tasks;
}
}
}, false, false);
}, false);
}
public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)

3
src/java/org/apache/cassandra/db/compaction/OperationType.java
View File

@ -35,8 +35,7 @@ public enum OperationType
VERIFY("Verify"),
FLUSH("Flush"),
STREAM("Stream"),
WRITE("Write"),
VIEW_BUILD("Materialized view build");
WRITE("Write");
public final String type;
public final String fileName;

691
src/java/org/apache/cassandra/db/view/MaterializedView.java
View File

@ -1,691 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import com.google.common.collect.Iterables;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.statements.CFProperties;
import org.apache.cassandra.db.AbstractReadCommandBuilder.SinglePartitionSliceBuilder;
import org.apache.cassandra.db.CBuilder;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionInfo;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.LivenessInfo;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.RangeTombstone;
import org.apache.cassandra.db.ReadCommand;
import org.apache.cassandra.db.ReadOrderGroup;
import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.partitions.AbstractThreadUnsafePartition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.BTreeBackedRow;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.ColumnData;
import org.apache.cassandra.db.rows.ComplexColumnData;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.service.pager.QueryPager;
/**
* A Materialized View copies data from a base table into a view table which can be queried independently from the
* base. Every update which targets the base table must be fed through the {@link MaterializedViewManager} to ensure
* that if a view needs to be updated, the updates are properly created and fed into the view.
*
* This class does the job of translating the base row to the view row.
*
* It handles reading existing state and figuring out what tombstones need to be generated.
*
* createMutations below is the "main method"
*
*/
public class MaterializedView
{
/**
* The columns should all be updated together, so we use this object as group.
*/
private static class MVColumns
{
//These are the base column definitions in terms of the *views* partitioning.
//Meaning we can see (for example) the partition key of the view contains a clustering key
//from the base table.
public final List<ColumnDefinition> partitionDefs;
public final List<ColumnDefinition> primaryKeyDefs;
public final List<ColumnDefinition> baseComplexColumns;
private MVColumns(List<ColumnDefinition> partitionDefs, List<ColumnDefinition> primaryKeyDefs, List<ColumnDefinition> baseComplexColumns)
{
this.partitionDefs = partitionDefs;
this.primaryKeyDefs = primaryKeyDefs;
this.baseComplexColumns = baseComplexColumns;
}
}
public final String name;
private final ColumnFamilyStore baseCfs;
private ColumnFamilyStore _viewCfs = null;
private MVColumns columns;
private final boolean viewHasAllPrimaryKeys;
private final boolean includeAll;
private MaterializedViewBuilder builder;
public MaterializedView(MaterializedViewDefinition definition,
ColumnFamilyStore baseCfs)
{
this.baseCfs = baseCfs;
name = definition.viewName;
includeAll = definition.includeAll;
viewHasAllPrimaryKeys = updateDefinition(definition);
}
/**
* Lazily fetch the CFS instance for the view.
* We do this lazily to avoid initilization issues.
*
* @return The views CFS instance
*/
public ColumnFamilyStore getViewCfs()
{
if (_viewCfs == null)
_viewCfs = Keyspace.openAndGetStore(Schema.instance.getCFMetaData(baseCfs.keyspace.getName(), name));
return _viewCfs;
}
/**
* Lookup column definitions in the base table that correspond to the view columns (should be 1:1)
*
* Notify caller if all primary keys in the view are ALL primary keys in the base. We do this to simplify
* tombstone checks.
*
* @param columns a list of columns to lookup in the base table
* @param definitions lists to populate for the base table definitions
* @return true if all view PKs are also Base PKs
*/
private boolean resolveAndAddColumns(Iterable<ColumnIdentifier> columns, List<ColumnDefinition>... definitions)
{
boolean allArePrimaryKeys = true;
for (ColumnIdentifier identifier : columns)
{
ColumnDefinition cdef = baseCfs.metadata.getColumnDefinition(identifier);
assert cdef != null : "Could not resolve column " + identifier.toString();
for (List<ColumnDefinition> list : definitions)
{
list.add(cdef);
}
allArePrimaryKeys = allArePrimaryKeys && cdef.isPrimaryKeyColumn();
}
return allArePrimaryKeys;
}
/**
* This updates the columns stored which are dependent on the base CFMetaData.
*
* @return true if the view contains only columns which are part of the base's primary key; false if there is at
* least one column which is not.
*/
public boolean updateDefinition(MaterializedViewDefinition definition)
{
List<ColumnDefinition> partitionDefs = new ArrayList<>(definition.partitionColumns.size());
List<ColumnDefinition> primaryKeyDefs = new ArrayList<>(definition.partitionColumns.size()
+ definition.clusteringColumns.size());
List<ColumnDefinition> baseComplexColumns = new ArrayList<>();
// We only add the partition columns to the partitions list, but both partition columns and clustering
// columns are added to the primary keys list
boolean partitionAllPrimaryKeyColumns = resolveAndAddColumns(definition.partitionColumns, primaryKeyDefs, partitionDefs);
boolean clusteringAllPrimaryKeyColumns = resolveAndAddColumns(definition.clusteringColumns, primaryKeyDefs);
for (ColumnDefinition cdef : baseCfs.metadata.allColumns())
{
if (cdef.isComplex())
{
baseComplexColumns.add(cdef);
}
}
this.columns = new MVColumns(partitionDefs, primaryKeyDefs, baseComplexColumns);
return partitionAllPrimaryKeyColumns && clusteringAllPrimaryKeyColumns;
}
/**
* Check to see if the update could possibly modify a view. Cases where the view may be updated are:
* <ul>
* <li>View selects all columns</li>
* <li>Update contains any range tombstones</li>
* <li>Update touches one of the columns included in the view</li>
* </ul>
*
* If the update contains any range tombstones, there is a possibility that it will not touch a range that is
* currently included in the view.
*
* @return true if {@param partition} modifies a column included in the view
*/
public boolean updateAffectsView(AbstractThreadUnsafePartition partition)
{
// If we are including all of the columns, then any update will be included
if (includeAll)
return true;
// If there are range tombstones, tombstones will also need to be generated for the materialized view
// This requires a query of the base rows and generating tombstones for all of those values
if (!partition.deletionInfo().isLive())
return true;
// Check whether the update touches any of the columns included in the view
for (Row row : partition)
{
for (ColumnData data : row)
{
if (getViewCfs().metadata.getColumnDefinition(data.column().name) != null)
return true;
}
}
return false;
}
/**
* Creates the clustering columns for the view based on the specified row and resolver policy
*
* @param temporalRow The current row
* @param resolver The policy to use when selecting versions of cells use
* @return The clustering object to use for the view
*/
private Clustering viewClustering(TemporalRow temporalRow, TemporalRow.Resolver resolver)
{
CFMetaData viewCfm = getViewCfs().metadata;
int numViewClustering = viewCfm.clusteringColumns().size();
CBuilder clustering = CBuilder.create(getViewCfs().getComparator());
for (int i = 0; i < numViewClustering; i++)
{
ColumnDefinition definition = viewCfm.clusteringColumns().get(i);
clustering.add(temporalRow.clusteringValue(definition, resolver));
}
return clustering.build();
}
/**
* @return Mutation containing a range tombstone for a base partition key and TemporalRow.
*/
private PartitionUpdate createTombstone(TemporalRow temporalRow,
DecoratedKey partitionKey,
DeletionTime deletionTime,
TemporalRow.Resolver resolver,
int nowInSec)
{
CFMetaData viewCfm = getViewCfs().metadata;
Row.Builder builder = BTreeBackedRow.unsortedBuilder(viewCfm.partitionColumns().regulars, nowInSec);
builder.newRow(viewClustering(temporalRow, resolver));
builder.addRowDeletion(deletionTime);
return PartitionUpdate.singleRowUpdate(viewCfm, partitionKey, builder.build());
}
/**
* @return PartitionUpdate containing a complex tombstone for a TemporalRow, and the collection's column identifier.
*/
private PartitionUpdate createComplexTombstone(TemporalRow temporalRow,
DecoratedKey partitionKey,
ColumnDefinition deletedColumn,
DeletionTime deletionTime,
TemporalRow.Resolver resolver,
int nowInSec)
{
CFMetaData viewCfm = getViewCfs().metadata;
Row.Builder builder = BTreeBackedRow.unsortedBuilder(viewCfm.partitionColumns().regulars, nowInSec);
builder.newRow(viewClustering(temporalRow, resolver));
builder.addComplexDeletion(deletedColumn, deletionTime);
return PartitionUpdate.singleRowUpdate(viewCfm, partitionKey, builder.build());
}
/**
* @return View's DecoratedKey or null, if one of the view's primary key components has an invalid resolution from
* the TemporalRow and its Resolver
*/
private DecoratedKey viewPartitionKey(TemporalRow temporalRow, TemporalRow.Resolver resolver)
{
List<ColumnDefinition> partitionDefs = this.columns.partitionDefs;
Object[] partitionKey = new Object[partitionDefs.size()];
for (int i = 0; i < partitionKey.length; i++)
{
ByteBuffer value = temporalRow.clusteringValue(partitionDefs.get(i), resolver);
if (value == null)
return null;
partitionKey[i] = value;
}
return getViewCfs().partitioner.decorateKey(CFMetaData.serializePartitionKey(getViewCfs().metadata
.getKeyValidatorAsClusteringComparator()
.make(partitionKey)));
}
/**
* @return mutation which contains the tombstone for the referenced TemporalRow, or null if not necessary.
* TemporalRow's can reference at most one view row; there will be at most one row to be tombstoned, so only one
* mutation is necessary
*/
private PartitionUpdate createRangeTombstoneForRow(TemporalRow temporalRow)
{
// Primary Key and Clustering columns do not generate tombstones
if (viewHasAllPrimaryKeys)
return null;
boolean hasUpdate = false;
List<ColumnDefinition> primaryKeyDefs = this.columns.primaryKeyDefs;
for (ColumnDefinition viewPartitionKeys : primaryKeyDefs)
{
if (!viewPartitionKeys.isPrimaryKeyColumn() && temporalRow.clusteringValue(viewPartitionKeys, TemporalRow.oldValueIfUpdated) != null)
hasUpdate = true;
}
if (!hasUpdate)
return null;
TemporalRow.Resolver resolver = TemporalRow.earliest;
return createTombstone(temporalRow,
viewPartitionKey(temporalRow, resolver),
new DeletionTime(temporalRow.viewClusteringTimestamp(), temporalRow.nowInSec),
resolver,
temporalRow.nowInSec);
}
/**
* @return Mutation which is the transformed base table mutation for the materialized view.
*/
private PartitionUpdate createUpdatesForInserts(TemporalRow temporalRow)
{
TemporalRow.Resolver resolver = TemporalRow.latest;
DecoratedKey partitionKey = viewPartitionKey(temporalRow, resolver);
ColumnFamilyStore viewCfs = getViewCfs();
if (partitionKey == null)
{
// Not having a partition key means we aren't updating anything
return null;
}
Row.Builder regularBuilder = BTreeBackedRow.unsortedBuilder(viewCfs.metadata.partitionColumns().regulars, temporalRow.nowInSec);
CBuilder clustering = CBuilder.create(viewCfs.getComparator());
for (int i = 0; i < viewCfs.metadata.clusteringColumns().size(); i++)
{
clustering.add(temporalRow.clusteringValue(viewCfs.metadata.clusteringColumns().get(i), resolver));
}
regularBuilder.newRow(clustering.build());
regularBuilder.addPrimaryKeyLivenessInfo(LivenessInfo.create(viewCfs.metadata,
temporalRow.viewClusteringTimestamp(),
temporalRow.viewClusteringTtl(),
temporalRow.viewClusteringLocalDeletionTime()));
for (ColumnDefinition columnDefinition : viewCfs.metadata.allColumns())
{
if (columnDefinition.isPrimaryKeyColumn())
continue;
for (Cell cell : temporalRow.values(columnDefinition, resolver))
{
regularBuilder.addCell(cell);
}
}
return PartitionUpdate.singleRowUpdate(viewCfs.metadata, partitionKey, regularBuilder.build());
}
/**
* @param partition Update which possibly contains deletion info for which to generate view tombstones.
* @return View Tombstones which delete all of the rows which have been removed from the base table with
* {@param partition}
*/
private Collection<Mutation> createForDeletionInfo(TemporalRow.Set rowSet, AbstractThreadUnsafePartition partition)
{
final TemporalRow.Resolver resolver = TemporalRow.earliest;
DeletionInfo deletionInfo = partition.deletionInfo();
List<Mutation> mutations = new ArrayList<>();
// Check the complex columns to see if there are any which may have tombstones we need to create for the view
if (!columns.baseComplexColumns.isEmpty())
{
for (Row row : partition)
{
if (!row.hasComplexDeletion())
continue;
TemporalRow temporalRow = rowSet.getClustering(row.clustering());
assert temporalRow != null;
for (ColumnDefinition definition : columns.baseComplexColumns)
{
ComplexColumnData columnData = row.getComplexColumnData(definition);
if (columnData != null)
{
DeletionTime time = columnData.complexDeletion();
if (!time.isLive())
{
DecoratedKey targetKey = viewPartitionKey(temporalRow, resolver);
if (targetKey != null)
mutations.add(new Mutation(createComplexTombstone(temporalRow, targetKey, definition, time, resolver, temporalRow.nowInSec)));
}
}
}
}
}
ReadCommand command = null;
if (!deletionInfo.isLive())
{
// We have to generate tombstones for all of the affected rows, but we don't have the information in order
// to create them. This requires that we perform a read for the entire range that is being tombstoned, and
// generate a tombstone for each. This may be slow, because a single range tombstone can cover up to an
// entire partition of data which is not distributed on a single partition node.
DecoratedKey dk = rowSet.dk;
if (deletionInfo.hasRanges())
{
SinglePartitionSliceBuilder builder = new SinglePartitionSliceBuilder(baseCfs, dk);
Iterator<RangeTombstone> tombstones = deletionInfo.rangeIterator(false);
while (tombstones.hasNext())
{
RangeTombstone tombstone = tombstones.next();
builder.addSlice(tombstone.deletedSlice());
}
command = builder.build();
}
else
{
command = SinglePartitionReadCommand.fullPartitionRead(baseCfs.metadata, rowSet.nowInSec, dk);
}
}
if (command == null)
{
SinglePartitionSliceBuilder builder = null;
for (Row row : partition)
{
if (!row.deletion().isLive())
{
if (builder == null)
builder = new SinglePartitionSliceBuilder(baseCfs, rowSet.dk);
builder.addSlice(Slice.make(row.clustering()));
}
}
if (builder != null)
command = builder.build();
}
if (command != null)
{
QueryPager pager = command.getPager(null);
// Add all of the rows which were recovered from the query to the row set
while (!pager.isExhausted())
{
try (ReadOrderGroup orderGroup = pager.startOrderGroup();
PartitionIterator iter = pager.fetchPageInternal(128, orderGroup))
{
if (!iter.hasNext())
break;
try (RowIterator rowIterator = iter.next())
{
while (rowIterator.hasNext())
{
Row row = rowIterator.next();
rowSet.addRow(row, false);
}
}
}
}
// If the temporal row has been deleted by the deletion info, we generate the corresponding range tombstone
// for the view.
for (TemporalRow temporalRow : rowSet)
{
DeletionTime deletionTime = temporalRow.deletionTime(partition);
if (!deletionTime.isLive())
{
DecoratedKey value = viewPartitionKey(temporalRow, resolver);
if (value != null)
{
PartitionUpdate update = createTombstone(temporalRow, value, deletionTime, resolver, temporalRow.nowInSec);
if (update != null)
mutations.add(new Mutation(update));
}
}
}
}
return !mutations.isEmpty() ? mutations : null;
}
/**
* Read and update temporal rows in the set which have corresponding values stored on the local node
*/
private void readLocalRows(TemporalRow.Set rowSet)
{
SinglePartitionSliceBuilder builder = new SinglePartitionSliceBuilder(baseCfs, rowSet.dk);
for (TemporalRow temporalRow : rowSet)
builder.addSlice(temporalRow.baseSlice());
QueryPager pager = builder.build().getPager(null);
while (!pager.isExhausted())
{
try (ReadOrderGroup orderGroup = pager.startOrderGroup();
PartitionIterator iter = pager.fetchPageInternal(128, orderGroup))
{
while (iter.hasNext())
{
try (RowIterator rows = iter.next())
{
while (rows.hasNext())
{
rowSet.addRow(rows.next(), false);
}
}
}
}
}
}
/**
* @return Set of rows which are contained in the partition update {@param partition}
*/
private TemporalRow.Set separateRows(ByteBuffer key, AbstractThreadUnsafePartition partition)
{
Set<ColumnIdentifier> columns = new HashSet<>();
for (ColumnDefinition def : this.columns.primaryKeyDefs)
columns.add(def.name);
TemporalRow.Set rowSet = new TemporalRow.Set(baseCfs, columns, key);
for (Row row : partition)
rowSet.addRow(row, true);
return rowSet;
}
/**
* @param isBuilding If the view is currently being built, we do not query the values which are already stored,
* since all of the update will already be present in the base table.
* @return View mutations which represent the changes necessary as long as previously created mutations for the view
* have been applied successfully. This is based solely on the changes that are necessary given the current
* state of the base table and the newly applying partition data.
*/
public Collection<Mutation> createMutations(ByteBuffer key, AbstractThreadUnsafePartition partition, boolean isBuilding)
{
if (!updateAffectsView(partition))
return null;
TemporalRow.Set rowSet = separateRows(key, partition);
// If we are building the view, we do not want to add old values; they will always be the same
if (!isBuilding)
readLocalRows(rowSet);
Collection<Mutation> mutations = null;
for (TemporalRow temporalRow : rowSet)
{
// If we are building, there is no need to check for partition tombstones; those values will not be present
// in the partition data
if (!isBuilding)
{
PartitionUpdate partitionTombstone = createRangeTombstoneForRow(temporalRow);
if (partitionTombstone != null)
{
if (mutations == null) mutations = new LinkedList<>();
mutations.add(new Mutation(partitionTombstone));
}
}
PartitionUpdate insert = createUpdatesForInserts(temporalRow);
if (insert != null)
{
if (mutations == null) mutations = new LinkedList<>();
mutations.add(new Mutation(insert));
}
}
if (!isBuilding)
{
Collection<Mutation> deletion = createForDeletionInfo(rowSet, partition);
if (deletion != null && !deletion.isEmpty())
{
if (mutations == null) mutations = new LinkedList<>();
mutations.addAll(deletion);
}
}
return mutations;
}
public synchronized void build()
{
if (this.builder != null)
{
this.builder.stop();
this.builder = null;
}
this.builder = new MaterializedViewBuilder(baseCfs, this);
CompactionManager.instance.submitMaterializedViewBuilder(builder);
}
/**
* @return CFMetaData which represents the definition given
*/
public static CFMetaData getCFMetaData(MaterializedViewDefinition definition,
CFMetaData baseCf,
CFProperties properties)
{
CFMetaData.Builder viewBuilder = CFMetaData.Builder
.createView(baseCf.ksName, definition.viewName);
ColumnDefinition nonPkTarget = null;
for (ColumnIdentifier targetIdentifier : definition.partitionColumns)
{
ColumnDefinition target = baseCf.getColumnDefinition(targetIdentifier);
if (!target.isPartitionKey())
nonPkTarget = target;
viewBuilder.addPartitionKey(target.name, properties.getReversableType(targetIdentifier, target.type));
}
Collection<ColumnDefinition> included = new ArrayList<>();
for(ColumnIdentifier identifier : definition.included)
{
ColumnDefinition cfDef = baseCf.getColumnDefinition(identifier);
assert cfDef != null;
included.add(cfDef);
}
boolean includeAll = included.isEmpty();
for (ColumnIdentifier ident : definition.clusteringColumns)
{
ColumnDefinition column = baseCf.getColumnDefinition(ident);
viewBuilder.addClusteringColumn(ident, properties.getReversableType(ident, column.type));
}
for (ColumnDefinition column : baseCf.partitionColumns().regulars.columns)
{
if (column != nonPkTarget && (includeAll || included.contains(column)))
{
viewBuilder.addRegularColumn(column.name, column.type);
}
}
//Add any extra clustering columns
for (ColumnDefinition column : Iterables.concat(baseCf.partitionKeyColumns(), baseCf.clusteringColumns()))
{
if ( (!definition.partitionColumns.contains(column.name) && !definition.clusteringColumns.contains(column.name)) &&
(includeAll || included.contains(column)) )
{
viewBuilder.addRegularColumn(column.name, column.type);
}
}
CFMetaData cfm = viewBuilder.build();
properties.properties.applyToCFMetadata(cfm);
return cfm;
}
}

222
src/java/org/apache/cassandra/db/view/MaterializedViewBuilder.java
View File

@ -1,222 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.ReadOrderGroup;
import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.compaction.CompactionInfo;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.SSTableSet;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.partitions.FilteredPartition;
import org.apache.cassandra.db.partitions.PartitionIterator;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.sstable.ReducingKeyIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.pager.QueryPager;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.concurrent.Refs;
public class MaterializedViewBuilder extends CompactionInfo.Holder
{
private final ColumnFamilyStore baseCfs;
private final MaterializedView view;
private final UUID compactionId;
private volatile Token prevToken = null;
private static final Logger logger = LoggerFactory.getLogger(MaterializedViewBuilder.class);
private volatile boolean isStopped = false;
public MaterializedViewBuilder(ColumnFamilyStore baseCfs, MaterializedView view)
{
this.baseCfs = baseCfs;
this.view = view;
compactionId = UUIDGen.getTimeUUID();
}
private void buildKey(DecoratedKey key)
{
QueryPager pager = SinglePartitionReadCommand.fullPartitionRead(baseCfs.metadata, FBUtilities.nowInSeconds(), key).getPager(null);
while (!pager.isExhausted())
{
try (ReadOrderGroup orderGroup = pager.startOrderGroup();
PartitionIterator partitionIterator = pager.fetchPageInternal(128, orderGroup))
{
if (!partitionIterator.hasNext())
return;
try (RowIterator rowIterator = partitionIterator.next())
{
Collection<Mutation> mutations = view.createMutations(key.getKey(), FilteredPartition.create(rowIterator), true);
if (mutations != null)
{
try
{
StorageProxy.mutateMV(key.getKey(), mutations);
break;
}
catch (WriteTimeoutException ex)
{
NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES)
.warn("Encountered write timeout when building materialized view {}, the entries were stored in the batchlog and will be replayed at another time", view.name);
}
}
}
}
}
}
public void run()
{
String ksname = baseCfs.metadata.ksName, viewName = view.name;
if (SystemKeyspace.isViewBuilt(ksname, viewName))
return;
Iterable<Range<Token>> ranges = StorageService.instance.getLocalRanges(baseCfs.metadata.ksName);
final Pair<Integer, Token> buildStatus = SystemKeyspace.getMaterializedViewBuildStatus(ksname, viewName);
Token lastToken;
Function<View, Iterable<SSTableReader>> function;
if (buildStatus == null)
{
baseCfs.forceBlockingFlush();
function = View.select(SSTableSet.CANONICAL);
int generation = Integer.MIN_VALUE;
try (Refs<SSTableReader> temp = baseCfs.selectAndReference(function).refs)
{
for (SSTableReader reader : temp)
{
generation = Math.max(reader.descriptor.generation, generation);
}
}
SystemKeyspace.beginMaterializedViewBuild(ksname, viewName, generation);
lastToken = null;
}
else
{
function = new Function<View, Iterable<SSTableReader>>()
{
@Nullable
public Iterable<SSTableReader> apply(View view)
{
Iterable<SSTableReader> readers = View.select(SSTableSet.CANONICAL).apply(view);
if (readers != null)
return Iterables.filter(readers, ssTableReader -> ssTableReader.descriptor.generation <= buildStatus.left);
return null;
}
};
lastToken = buildStatus.right;
}
prevToken = lastToken;
try (Refs<SSTableReader> sstables = baseCfs.selectAndReference(function).refs;
ReducingKeyIterator iter = new ReducingKeyIterator(sstables))
{
while (!isStopped && iter.hasNext())
{
DecoratedKey key = iter.next();
Token token = key.getToken();
if (lastToken == null || lastToken.compareTo(token) < 0)
{
for (Range<Token> range : ranges)
{
if (range.contains(token))
{
buildKey(key);
if (prevToken == null || prevToken.compareTo(token) != 0)
{
SystemKeyspace.updateMaterializedViewBuildStatus(ksname, viewName, key.getToken());
prevToken = token;
}
}
}
lastToken = null;
}
}
SystemKeyspace.finishMaterializedViewBuildStatus(ksname, viewName);
}
catch (Exception e)
{
final MaterializedViewBuilder builder = new MaterializedViewBuilder(baseCfs, view);
ScheduledExecutors.nonPeriodicTasks.schedule(() -> CompactionManager.instance.submitMaterializedViewBuilder(builder),
5,
TimeUnit.MINUTES);
logger.warn("Materialized View failed to complete, sleeping 5 minutes before restarting", e);
}
}
public CompactionInfo getCompactionInfo()
{
long rangesLeft = 0, rangesTotal = 0;
Token lastToken = prevToken;
// This approximation is not very accurate, but since we do not have a method which allows us to calculate the
// percentage of a range covered by a second range, this is the best approximation that we can calculate.
// Instead, we just count the total number of ranges that haven't been seen by the node (we use the order of
// the tokens to determine whether they have been seen yet or not), and the total number of ranges that a node
// has.
for (Range<Token> range : StorageService.instance.getLocalRanges(baseCfs.keyspace.getName()))
{
rangesLeft++;
rangesTotal++;
// This will reset rangesLeft, so that the number of ranges left will be less than the total ranges at the
// end of the method.
if (lastToken == null || range.contains(lastToken))
rangesLeft = 0;
}
return new CompactionInfo(baseCfs.metadata, OperationType.VIEW_BUILD, rangesLeft, rangesTotal, "ranges", compactionId);
}
public void stop()
{
isStopped = true;
}
}

237
src/java/org/apache/cassandra/db/view/MaterializedViewManager.java
View File

@ -1,237 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.locks.Lock;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Striped;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.MaterializedViewDefinition;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.IMutation;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.commitlog.ReplayPosition;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.OverloadedException;
import org.apache.cassandra.exceptions.UnavailableException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
/**
* Manages {@link MaterializedView}'s for a single {@link ColumnFamilyStore}. All of the materialized views for that
* table are created when this manager is initialized.
*
* The main purposes of the manager are to provide a single location for updates to be vetted to see whether they update
* any views {@link MaterializedViewManager#updateAffectsView(PartitionUpdate)}, provide locks to prevent multiple
* updates from creating incoherent updates in the view {@link MaterializedViewManager#acquireLockFor(ByteBuffer)}, and
* to affect change on the view.
*/
public class MaterializedViewManager
{
private static final Striped<Lock> LOCKS = Striped.lazyWeakLock(DatabaseDescriptor.getConcurrentWriters() * 1024);
private final ConcurrentNavigableMap<String, MaterializedView> viewsByName;
private final ColumnFamilyStore baseCfs;
public MaterializedViewManager(ColumnFamilyStore baseCfs)
{
this.viewsByName = new ConcurrentSkipListMap<>();
this.baseCfs = baseCfs;
}
public Iterable<MaterializedView> allViews()
{
return viewsByName.values();
}
public Iterable<ColumnFamilyStore> allViewsCfs()
{
List<ColumnFamilyStore> viewColumnFamilies = new ArrayList<>();
for (MaterializedView view : allViews())
viewColumnFamilies.add(view.getViewCfs());
return viewColumnFamilies;
}
public void init()
{
reload();
}
public void invalidate()
{
for (MaterializedView view : allViews())
removeMaterializedView(view.name);
}
public void reload()
{
Map<String, MaterializedViewDefinition> newViewsByName = new HashMap<>();
for (MaterializedViewDefinition definition : baseCfs.metadata.getMaterializedViews())
{
newViewsByName.put(definition.viewName, definition);
}
for (String viewName : viewsByName.keySet())
{
if (!newViewsByName.containsKey(viewName))
removeMaterializedView(viewName);
}
for (Map.Entry<String, MaterializedViewDefinition> entry : newViewsByName.entrySet())
{
if (!viewsByName.containsKey(entry.getKey()))
addMaterializedView(entry.getValue());
}
for (MaterializedView view : allViews())
{
view.build();
// We provide the new definition from the base metadata
view.updateDefinition(newViewsByName.get(view.name));
}
}
public void buildAllViews()
{
for (MaterializedView view : allViews())
view.build();
}
public void removeMaterializedView(String name)
{
MaterializedView view = viewsByName.remove(name);
if (view == null)
return;
SystemKeyspace.setMaterializedViewRemoved(baseCfs.metadata.ksName, view.name);
}
public void addMaterializedView(MaterializedViewDefinition definition)
{
MaterializedView view = new MaterializedView(definition, baseCfs);
viewsByName.put(definition.viewName, view);
}
/**
* Calculates and pushes updates to the views replicas. The replicas are determined by
* {@link MaterializedViewUtils#getViewNaturalEndpoint(String, Token, Token)}.
*/
public void pushViewReplicaUpdates(ByteBuffer key, PartitionUpdate update) throws UnavailableException, OverloadedException, WriteTimeoutException
{
// This happens when we are replaying from commitlog. In that case, we have already sent this commit off to the
// view node.
if (!StorageService.instance.isJoined()) return;
List<Mutation> mutations = null;
for (Map.Entry<String, MaterializedView> view : viewsByName.entrySet())
{
Collection<Mutation> viewMutations = view.getValue().createMutations(key, update, false);
if (viewMutations != null && !viewMutations.isEmpty())
{
if (mutations == null)
mutations = Lists.newLinkedList();
mutations.addAll(viewMutations);
}
}
if (mutations != null)
{
StorageProxy.mutateMV(key, mutations);
}
}
public boolean updateAffectsView(PartitionUpdate upd)
{
for (MaterializedView view : allViews())
{
if (view.updateAffectsView(upd))
return true;
}
return false;
}
public static Lock acquireLockFor(ByteBuffer key)
{
Lock lock = LOCKS.get(key);
if (lock.tryLock())
return lock;
return null;
}
public static boolean updatesAffectView(Collection<? extends IMutation> mutations, boolean ignoreRf1)
{
for (IMutation mutation : mutations)
{
for (PartitionUpdate cf : mutation.getPartitionUpdates())
{
Keyspace keyspace = Keyspace.open(cf.metadata().ksName);
if (ignoreRf1 && keyspace.getReplicationStrategy().getReplicationFactor() == 1)
continue;
MaterializedViewManager viewManager = keyspace.getColumnFamilyStore(cf.metadata().cfId).materializedViewManager;
if (viewManager.updateAffectsView(cf))
return true;
}
}
return false;
}
public void forceBlockingFlush()
{
for (ColumnFamilyStore viewCfs : allViewsCfs())
viewCfs.forceBlockingFlush();
}
public void dumpMemtables()
{
for (ColumnFamilyStore viewCfs : allViewsCfs())
viewCfs.dumpMemtable();
}
public void truncateBlocking(long truncatedAt)
{
for (ColumnFamilyStore viewCfs : allViewsCfs())
{
ReplayPosition replayAfter = viewCfs.discardSSTables(truncatedAt);
SystemKeyspace.saveTruncationRecord(viewCfs, truncatedAt, replayAfter);
}
}
}

95
src/java/org/apache/cassandra/db/view/MaterializedViewUtils.java
View File

@ -1,95 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.List;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.utils.FBUtilities;
public final class MaterializedViewUtils
{
private MaterializedViewUtils()
{
}
/**
* Calculate the natural endpoint for the view.
*
* The view natural endpoint is the endpint which has the same cardinality as this node in the replication factor.
* The cardinality is the number at which this node would store a piece of data, given the change in replication
* factor.
*
* For example, if we have the following ring:
* A, T1 -> B, T2 -> C, T3 -> A
*
* For the token T1, at RF=1, A would be included, so A's cardinality for T1 is 1. For the token T1, at RF=2, B would
* be included, so B's cardinality for token T1 is 2. For token T3, at RF = 2, A would be included, so A's cardinality
* for T3 is 2.
*
* For a view whose base token is T1 and whose view token is T3, the pairings between the nodes would be:
* A writes to C (A's cardinality is 1 for T1, and C's cardinality is 1 for T3)
* B writes to A (B's cardinality is 2 for T1, and A's cardinality is 2 for T3)
* C writes to B (C's cardinality is 3 for T1, and B's cardinality is 3 for T3)
*
* @throws RuntimeException if this method is called using a base token which does not belong to this replica
*/
public static InetAddress getViewNaturalEndpoint(String keyspaceName, Token baseToken, Token viewToken)
{
AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspaceName).getReplicationStrategy();
String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
List<InetAddress> localBaseEndpoints = new ArrayList<>();
List<InetAddress> localViewEndpoints = new ArrayList<>();
for (InetAddress baseEndpoint : replicationStrategy.getNaturalEndpoints(baseToken))
{
if (DatabaseDescriptor.getEndpointSnitch().getDatacenter(baseEndpoint).equals(localDataCenter))
localBaseEndpoints.add(baseEndpoint);
}
for (InetAddress viewEndpoint : replicationStrategy.getNaturalEndpoints(viewToken))
{
// If we are a base endpoint which is also a view replica, we use ourselves as our view replica
if (viewEndpoint.equals(FBUtilities.getBroadcastAddress()))
return viewEndpoint;
// We have to remove any endpoint which is shared between the base and the view, as it will select itself
// and throw off the counts otherwise.
if (localBaseEndpoints.contains(viewEndpoint))
localBaseEndpoints.remove(viewEndpoint);
else if (DatabaseDescriptor.getEndpointSnitch().getDatacenter(viewEndpoint).equals(localDataCenter))
localViewEndpoints.add(viewEndpoint);
}
// The replication strategy will be the same for the base and the view, as they must belong to the same keyspace.
// Since the same replication strategy is used, the same placement should be used and we should get the same
// number of replicas for all of the tokens in the ring.
assert localBaseEndpoints.size() == localViewEndpoints.size() : "Replication strategy should have the same number of endpoints for the base and the view";
int baseIdx = localBaseEndpoints.indexOf(FBUtilities.getBroadcastAddress());
if (baseIdx < 0)
throw new RuntimeException("Trying to get the view natural endpoint on a non-data replica");
return localViewEndpoints.get(baseIdx);
}
}

414
src/java/org/apache/cassandra/db/view/TemporalRow.java
View File

@ -1,414 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import com.google.common.collect.Iterables;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.CBuilder;
import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Conflicts;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionInfo;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.LivenessInfo;
import org.apache.cassandra.db.RangeTombstone;
import org.apache.cassandra.db.Slice;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.partitions.AbstractThreadUnsafePartition;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.CellPath;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
/**
* Represents a single CQL Row in a base table, with both the currently persisted value and the update's value. The
* values are stored in timestamp order, but also indicate whether they are from the currently persisted, allowing a
* {@link TemporalRow.Resolver} to resolve if the value is an old value that has been updated; if it sorts after the
* update's value, then it does not qualify.
*/
public class TemporalRow
{
private static final int NO_TTL = LivenessInfo.NO_TTL;
private static final long NO_TIMESTAMP = LivenessInfo.NO_TIMESTAMP;
private static final int NO_DELETION_TIME = DeletionTime.LIVE.localDeletionTime();
public interface Resolver
{
/**
* @param cells Iterable of all cells for a certain TemporalRow's Cell, in timestamp sorted order
* @return A single TemporalCell from the iterable which satisfies the resolution criteria, or null if
* there is no cell which qualifies
*/
TemporalCell resolve(Iterable<TemporalCell> cells);
}
/**
* Returns the first value in the iterable if it is from the set of persisted cells, and the cell which results from
* reconciliation of the remaining cells does not have the same value.
*/
public static final Resolver oldValueIfUpdated = cells -> {
Iterator<TemporalCell> iterator = cells.iterator();
if (!iterator.hasNext())
return null;
TemporalCell initial = iterator.next();
if (initial.isNew || !iterator.hasNext())
return null;
TemporalCell value = initial;
while (iterator.hasNext())
value = value.reconcile(iterator.next());
return ByteBufferUtil.compareUnsigned(initial.value, value.value) != 0 ? initial : null;
};
public static final Resolver earliest = cells -> {
Iterator<TemporalCell> iterator = cells.iterator();
if (!iterator.hasNext())
return null;
return iterator.next();
};
public static final Resolver latest = cells -> {
Iterator<TemporalCell> iterator = cells.iterator();
if (!iterator.hasNext())
return null;
TemporalCell value = iterator.next();
while (iterator.hasNext())
value = value.reconcile(iterator.next());
return value;
};
private static class TemporalCell
{
public final ByteBuffer value;
public final long timestamp;
public final int ttl;
public final int localDeletionTime;
public final boolean isNew;
private TemporalCell(ByteBuffer value, long timestamp, int ttl, int localDeletionTime, boolean isNew)
{
this.value = value;
this.timestamp = timestamp;
this.ttl = ttl;
this.localDeletionTime = localDeletionTime;
this.isNew = isNew;
}
public TemporalCell reconcile(TemporalCell that)
{
int now = FBUtilities.nowInSeconds();
Conflicts.Resolution resolution = Conflicts.resolveRegular(that.timestamp,
that.isLive(now),
that.localDeletionTime,
that.value,
this.timestamp,
this.isLive(now),
this.localDeletionTime,
this.value);
assert resolution != Conflicts.Resolution.MERGE;
if (resolution == Conflicts.Resolution.LEFT_WINS)
return that;
return this;
}
private boolean isLive(int now)
{
return localDeletionTime == NO_DELETION_TIME || (ttl != NO_TTL && now < localDeletionTime);
}
public Cell cell(ColumnDefinition definition, CellPath cellPath)
{
return new BufferCell(definition, timestamp, ttl, localDeletionTime, value, cellPath);
}
}
private final ColumnFamilyStore baseCfs;
private final java.util.Set<ColumnIdentifier> viewPrimaryKey;
private final ByteBuffer basePartitionKey;
public final Map<ColumnIdentifier, ByteBuffer> clusteringColumns;
public final int nowInSec;
private final Map<ColumnIdentifier, Map<CellPath, SortedMap<Long, TemporalCell>>> columnValues = new HashMap<>();
private int viewClusteringTtl = NO_TTL;
private long viewClusteringTimestamp = NO_TIMESTAMP;
private int viewClusteringLocalDeletionTime = NO_DELETION_TIME;
TemporalRow(ColumnFamilyStore baseCfs, java.util.Set<ColumnIdentifier> viewPrimaryKey, ByteBuffer key, Row row, int nowInSec, boolean isNew)
{
this.baseCfs = baseCfs;
this.viewPrimaryKey = viewPrimaryKey;
this.basePartitionKey = key;
this.nowInSec = nowInSec;
clusteringColumns = new HashMap<>();
LivenessInfo liveness = row.primaryKeyLivenessInfo();
this.viewClusteringLocalDeletionTime = minValueIfSet(viewClusteringLocalDeletionTime, row.deletion().localDeletionTime(), NO_DELETION_TIME);
this.viewClusteringTimestamp = minValueIfSet(viewClusteringTimestamp, liveness.timestamp(), NO_TIMESTAMP);
this.viewClusteringTtl = minValueIfSet(viewClusteringTtl, liveness.ttl(), NO_TTL);
List<ColumnDefinition> clusteringDefs = baseCfs.metadata.clusteringColumns();
for (int i = 0; i < clusteringDefs.size(); i++)
{
ColumnDefinition cdef = clusteringDefs.get(i);
clusteringColumns.put(cdef.name, row.clustering().get(i));
addColumnValue(cdef.name, null, NO_TIMESTAMP, NO_TTL, NO_DELETION_TIME, row.clustering().get(i), isNew);
}
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TemporalRow that = (TemporalRow) o;
if (!clusteringColumns.equals(that.clusteringColumns)) return false;
if (!basePartitionKey.equals(that.basePartitionKey)) return false;
return true;
}
@Override
public int hashCode()
{
int result = basePartitionKey.hashCode();
result = 31 * result + clusteringColumns.hashCode();
return result;
}
public void addColumnValue(ColumnIdentifier identifier,
CellPath cellPath,
long timestamp,
int ttl,
int localDeletionTime,
ByteBuffer value, boolean isNew)
{
if (!columnValues.containsKey(identifier))
columnValues.put(identifier, new HashMap<>());
Map<CellPath, SortedMap<Long, TemporalCell>> innerMap = columnValues.get(identifier);
if (!innerMap.containsKey(cellPath))
innerMap.put(cellPath, new TreeMap<>());
// If this column is part of the view's primary keys
if (viewPrimaryKey.contains(identifier))
{
this.viewClusteringTtl = minValueIfSet(this.viewClusteringTtl, ttl, NO_TTL);
this.viewClusteringTimestamp = minValueIfSet(this.viewClusteringTimestamp, timestamp, NO_TIMESTAMP);
this.viewClusteringLocalDeletionTime = minValueIfSet(this.viewClusteringLocalDeletionTime, localDeletionTime, NO_DELETION_TIME);
}
innerMap.get(cellPath).put(timestamp, new TemporalCell(value, timestamp, ttl, localDeletionTime, isNew));
}
private static int minValueIfSet(int existing, int update, int defaultValue)
{
if (existing == defaultValue)
return update;
if (update == defaultValue)
return existing;
return Math.min(existing, update);
}
private static long minValueIfSet(long existing, long update, long defaultValue)
{
if (existing == defaultValue)
return update;
if (update == defaultValue)
return existing;
return Math.min(existing, update);
}
public int viewClusteringTtl()
{
return viewClusteringTtl;
}
public long viewClusteringTimestamp()
{
return viewClusteringTimestamp;
}
public int viewClusteringLocalDeletionTime()
{
return viewClusteringLocalDeletionTime;
}
public void addCell(Cell cell, boolean isNew)
{
addColumnValue(cell.column().name, cell.path(), cell.timestamp(), cell.ttl(), cell.localDeletionTime(), cell.value(), isNew);
}
// The Definition here is actually the *base table* definition
public ByteBuffer clusteringValue(ColumnDefinition definition, Resolver resolver)
{
ColumnDefinition baseDefinition = definition.cfName.equals(baseCfs.name)
? definition
: baseCfs.metadata.getColumnDefinition(definition.name);
if (baseDefinition.isPartitionKey())
{
if (baseDefinition.isOnAllComponents())
return basePartitionKey;
else
{
CompositeType keyComparator = (CompositeType) baseCfs.metadata.getKeyValidator();
ByteBuffer[] components = keyComparator.split(basePartitionKey);
return components[baseDefinition.position()];
}
}
else
{
ColumnIdentifier columnIdentifier = baseDefinition.name;
if (clusteringColumns.containsKey(columnIdentifier))
return clusteringColumns.get(columnIdentifier);
Collection<org.apache.cassandra.db.rows.Cell> val = values(definition, resolver);
if (val != null && val.size() == 1)
return Iterables.getOnlyElement(val).value();
}
return null;
}
public DeletionTime deletionTime(AbstractThreadUnsafePartition partition)
{
DeletionInfo deletionInfo = partition.deletionInfo();
if (!deletionInfo.getPartitionDeletion().isLive())
return deletionInfo.getPartitionDeletion();
Clustering baseClustering = baseClusteringBuilder().build();
RangeTombstone clusterTombstone = deletionInfo.rangeCovering(baseClustering);
if (clusterTombstone != null)
return clusterTombstone.deletionTime();
Row row = partition.getRow(baseClustering);
return row == null || row.deletion().isLive() ? DeletionTime.LIVE : row.deletion();
}
public Collection<org.apache.cassandra.db.rows.Cell> values(ColumnDefinition definition, Resolver resolver)
{
Map<CellPath, SortedMap<Long, TemporalCell>> innerMap = columnValues.get(definition.name);
if (innerMap == null)
{
return Collections.emptyList();
}
Collection<org.apache.cassandra.db.rows.Cell> value = new ArrayList<>();
for (Map.Entry<CellPath, SortedMap<Long, TemporalCell>> pathAndCells : innerMap.entrySet())
{
TemporalCell cell = resolver.resolve(pathAndCells.getValue().values());
if (cell != null)
value.add(cell.cell(definition, pathAndCells.getKey()));
}
return value;
}
public Slice baseSlice()
{
return baseClusteringBuilder().buildSlice();
}
private CBuilder baseClusteringBuilder()
{
CFMetaData metadata = baseCfs.metadata;
CBuilder builder = CBuilder.create(metadata.comparator);
ByteBuffer[] buffers = new ByteBuffer[clusteringColumns.size()];
for (Map.Entry<ColumnIdentifier, ByteBuffer> buffer : clusteringColumns.entrySet())
buffers[metadata.getColumnDefinition(buffer.getKey()).position()] = buffer.getValue();
for (ByteBuffer byteBuffer : buffers)
builder = builder.add(byteBuffer);
return builder;
}
static class Set implements Iterable<TemporalRow>
{
private final ColumnFamilyStore baseCfs;
private final java.util.Set<ColumnIdentifier> viewPrimaryKey;
private final ByteBuffer key;
public final DecoratedKey dk;
private final Map<Clustering, TemporalRow> clusteringToRow;
final int nowInSec = FBUtilities.nowInSeconds();
Set(ColumnFamilyStore baseCfs, java.util.Set<ColumnIdentifier> viewPrimaryKey, ByteBuffer key)
{
this.baseCfs = baseCfs;
this.viewPrimaryKey = viewPrimaryKey;
this.key = key;
this.dk = baseCfs.partitioner.decorateKey(key);
this.clusteringToRow = new HashMap<>();
}
public Iterator<TemporalRow> iterator()
{
return clusteringToRow.values().iterator();
}
public TemporalRow getClustering(Clustering clustering)
{
return clusteringToRow.get(clustering);
}
public void addRow(Row row, boolean isNew)
{
TemporalRow temporalRow = clusteringToRow.get(row.clustering());
if (temporalRow == null)
{
temporalRow = new TemporalRow(baseCfs, viewPrimaryKey, key, row, nowInSec, isNew);
clusteringToRow.put(row.clustering(), temporalRow);
}
for (Cell cell: row.cells())
{
temporalRow.addCell(cell, isNew);
}
}
public int size()
{
return clusteringToRow.size();
}
}
}

2
src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
View File

@ -87,7 +87,7 @@ public class KeyspaceMetrics
public final LatencyMetrics casPropose;
/** CAS Commit metrics */
public final LatencyMetrics casCommit;
public final MetricNameFactory factory;
private Keyspace keyspace;

42
src/java/org/apache/cassandra/metrics/MVWriteMetrics.java
View File

@ -1,42 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.metrics;
import com.codahale.metrics.Counter;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
public class MVWriteMetrics extends ClientRequestMetrics
{
public final Counter viewReplicasAttempted;
public final Counter viewReplicasSuccess;
public MVWriteMetrics(String scope) {
super(scope);
viewReplicasAttempted = Metrics.counter(factory.createMetricName("ViewReplicasAttempted"));
viewReplicasSuccess = Metrics.counter(factory.createMetricName("ViewReplicasSuccess"));
}
public void release()
{
super.release();
Metrics.remove(factory.createMetricName("ViewReplicasAttempted"));
Metrics.remove(factory.createMetricName("ViewReplicasSuccess"));
}
}

15
src/java/org/apache/cassandra/net/MessagingService.java
View File

@ -134,8 +134,6 @@ public final class MessagingService implements MessagingServiceMBean
PAXOS_PROPOSE,
PAXOS_COMMIT,
@Deprecated PAGED_RANGE,
BATCHLOG_MUTATION,
MATERIALIZED_VIEW_MUTATION,
// remember to add new verbs at the end, since we serialize by ordinal
UNUSED_1,
UNUSED_2,
@ -147,8 +145,6 @@ public final class MessagingService implements MessagingServiceMBean
{{
put(Verb.MUTATION, Stage.MUTATION);
put(Verb.COUNTER_MUTATION, Stage.COUNTER_MUTATION);
put(Verb.MATERIALIZED_VIEW_MUTATION, Stage.MATERIALIZED_VIEW_MUTATION);
put(Verb.BATCHLOG_MUTATION, Stage.BATCHLOG_MUTATION);
put(Verb.READ_REPAIR, Stage.MUTATION);
put(Verb.TRUNCATE, Stage.MUTATION);
put(Verb.PAXOS_PREPARE, Stage.MUTATION);
@ -207,8 +203,6 @@ public final class MessagingService implements MessagingServiceMBean
put(Verb.INTERNAL_RESPONSE, CallbackDeterminedSerializer.instance);
put(Verb.MUTATION, Mutation.serializer);
put(Verb.BATCHLOG_MUTATION, Mutation.serializer);
put(Verb.MATERIALIZED_VIEW_MUTATION, Mutation.serializer);
put(Verb.READ_REPAIR, Mutation.serializer);
put(Verb.READ, ReadCommand.serializer);
//put(Verb.RANGE_SLICE, ReadCommand.legacyRangeSliceCommandSerializer);
@ -235,8 +229,6 @@ public final class MessagingService implements MessagingServiceMBean
public static final EnumMap<Verb, IVersionedSerializer<?>> callbackDeserializers = new EnumMap<Verb, IVersionedSerializer<?>>(Verb.class)
{{
put(Verb.MUTATION, WriteResponse.serializer);
put(Verb.BATCHLOG_MUTATION, WriteResponse.serializer);
put(Verb.MATERIALIZED_VIEW_MUTATION, WriteResponse.serializer);
put(Verb.READ_REPAIR, WriteResponse.serializer);
put(Verb.COUNTER_MUTATION, WriteResponse.serializer);
put(Verb.RANGE_SLICE, ReadResponse.legacyRangeSliceReplySerializer);
@ -299,8 +291,6 @@ public final class MessagingService implements MessagingServiceMBean
*/
public static final EnumSet<Verb> DROPPABLE_VERBS = EnumSet.of(Verb._TRACE,
Verb.MUTATION,
Verb.BATCHLOG_MUTATION, //FIXME: should this be droppable??
Verb.MATERIALIZED_VIEW_MUTATION,
Verb.COUNTER_MUTATION,
Verb.READ_REPAIR,
Verb.READ,
@ -628,10 +618,7 @@ public final class MessagingService implements MessagingServiceMBean
ConsistencyLevel consistencyLevel,
boolean allowHints)
{
assert message.verb == Verb.MUTATION
|| message.verb == Verb.BATCHLOG_MUTATION
|| message.verb == Verb.MATERIALIZED_VIEW_MUTATION
|| message.verb == Verb.COUNTER_MUTATION;
assert message.verb == Verb.MUTATION || message.verb == Verb.COUNTER_MUTATION;
int messageId = nextId();
CallbackInfo previous = callbacks.put(messageId,

4
src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
View File

@ -307,9 +307,7 @@ public final class LegacySchemaMigrator
defaultValidator);
}
// The legacy schema did not have views, so we know that we are not loading a materialized view
boolean isMaterializedView = false;
CFMetaData cfm = CFMetaData.create(ksName, cfName, cfId, isDense, isCompound, isSuper, isCounter, isMaterializedView, columnDefs);
CFMetaData cfm = CFMetaData.create(ksName, cfName, cfId, isDense, isCompound, isSuper, isCounter, columnDefs);
cfm.readRepairChance(tableRow.getDouble("read_repair_chance"));
cfm.dcLocalReadRepairChance(tableRow.getDouble("local_read_repair_chance"));

149
src/java/org/apache/cassandra/schema/MaterializedViews.java
View File

@ -1,149 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.schema;
import java.util.Iterator;
import java.util.Optional;
import com.google.common.collect.ImmutableMap;
import org.apache.cassandra.config.MaterializedViewDefinition;
import static com.google.common.collect.Iterables.filter;
public final class MaterializedViews implements Iterable<MaterializedViewDefinition>
{
private final ImmutableMap<String, MaterializedViewDefinition> materializedViews;
private MaterializedViews(Builder builder)
{
materializedViews = builder.materializedViews.build();
}
public static Builder builder()
{
return new Builder();
}
public static MaterializedViews none()
{
return builder().build();
}
public Iterator<MaterializedViewDefinition> iterator()
{
return materializedViews.values().iterator();
}
public int size()
{
return materializedViews.size();
}
public boolean isEmpty()
{
return materializedViews.isEmpty();
}
/**
* Get the materialized view with the specified name
*
* @param name a non-qualified materialized view name
* @return an empty {@link Optional} if the materialized view name is not found; a non-empty optional of {@link MaterializedViewDefinition} otherwise
*/
public Optional<MaterializedViewDefinition> get(String name)
{
return Optional.ofNullable(materializedViews.get(name));
}
/**
* Create a MaterializedViews instance with the provided materialized view added
*/
public MaterializedViews with(MaterializedViewDefinition materializedView)
{
if (get(materializedView.viewName).isPresent())
throw new IllegalStateException(String.format("Materialized View %s already exists", materializedView.viewName));
return builder().add(this).add(materializedView).build();
}
/**
* Creates a MaterializedViews instance with the materializedView with the provided name removed
*/
public MaterializedViews without(String name)
{
MaterializedViewDefinition materializedView =
get(name).orElseThrow(() -> new IllegalStateException(String.format("Materialized View %s doesn't exists", name)));
return builder().add(filter(this, v -> v != materializedView)).build();
}
/**
* Creates a MaterializedViews instance which contains an updated materialized view
*/
public MaterializedViews replace(MaterializedViewDefinition materializedView)
{
return without(materializedView.viewName).with(materializedView);
}
@Override
public boolean equals(Object o)
{
return this == o || (o instanceof MaterializedViews && materializedViews.equals(((MaterializedViews) o).materializedViews));
}
@Override
public int hashCode()
{
return materializedViews.hashCode();
}
@Override
public String toString()
{
return materializedViews.values().toString();
}
public static final class Builder
{
final ImmutableMap.Builder<String, MaterializedViewDefinition> materializedViews = new ImmutableMap.Builder<>();
private Builder()
{
}
public MaterializedViews build()
{
return new MaterializedViews(this);
}
public Builder add(MaterializedViewDefinition materializedView)
{
materializedViews.put(materializedView.viewName, materializedView);
return this;
}
public Builder add(Iterable<MaterializedViewDefinition> materializedViews)
{
materializedViews.forEach(this::add);
return this;
}
}
}

158
src/java/org/apache/cassandra/schema/SchemaKeyspace.java
View File

@ -76,13 +76,12 @@ public final class SchemaKeyspace
public static final String COLUMNS = "columns";
public static final String DROPPED_COLUMNS = "dropped_columns";
public static final String TRIGGERS = "triggers";
public static final String MATERIALIZED_VIEWS = "materialized_views";
public static final String TYPES = "types";
public static final String FUNCTIONS = "functions";
public static final String AGGREGATES = "aggregates";
public static final List<String> ALL =
ImmutableList.of(KEYSPACES, TABLES, COLUMNS, TRIGGERS, MATERIALIZED_VIEWS, TYPES, FUNCTIONS, AGGREGATES);
ImmutableList.of(KEYSPACES, TABLES, COLUMNS, TRIGGERS, TYPES, FUNCTIONS, AGGREGATES);
private static final CFMetaData Keyspaces =
compile(KEYSPACES,
@ -153,18 +152,6 @@ public final class SchemaKeyspace
+ "trigger_options map<text, text>,"
+ "PRIMARY KEY ((keyspace_name), table_name, trigger_name))");
private static final CFMetaData MaterializedViews =
compile(MATERIALIZED_VIEWS,
"materialized views definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "table_name text,"
+ "view_name text,"
+ "target_columns list<text>,"
+ "clustering_columns list<text>,"
+ "included_columns list<text>,"
+ "PRIMARY KEY ((keyspace_name), table_name, view_name))");
private static final CFMetaData Types =
compile(TYPES,
"user defined type definitions",
@ -206,7 +193,7 @@ public final class SchemaKeyspace
+ "PRIMARY KEY ((keyspace_name), aggregate_name, signature))");
public static final List<CFMetaData> All =
ImmutableList.of(Keyspaces, Tables, Columns, Triggers, DroppedColumns, MaterializedViews, Types, Functions, Aggregates);
ImmutableList.of(Keyspaces, Tables, Columns, DroppedColumns, Triggers, Types, Functions, Aggregates);
private static CFMetaData compile(String name, String description, String schema)
{
@ -701,8 +688,6 @@ public final class SchemaKeyspace
Mutation mutation = new Mutation(NAME, getSchemaKSDecoratedKey(keyspace.name));
for (CFMetaData schemaTable : All)
mutation.add(PartitionUpdate.fullPartitionDelete(schemaTable, mutation.key(), timestamp, nowInSec));
mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.BuiltMaterializedViews, mutation.key(), timestamp, nowInSec));
mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.MaterializedViewsBuildsInProgress, mutation.key(), timestamp, nowInSec));
return mutation;
}
@ -845,9 +830,6 @@ public final class SchemaKeyspace
for (TriggerMetadata trigger : table.getTriggers())
addTriggerToSchemaMutation(table, trigger, timestamp, mutation);
for (MaterializedViewDefinition materializedView: table.getMaterializedViews())
addMaterializedViewToSchemaMutation(table, materializedView, timestamp, mutation);
}
}
@ -941,22 +923,6 @@ public final class SchemaKeyspace
for (TriggerMetadata trigger : triggerDiff.entriesOnlyOnRight().values())
addTriggerToSchemaMutation(newTable, trigger, timestamp, mutation);
MapDifference<String, MaterializedViewDefinition> materializedViewDiff = materializedViewsDiff(oldTable.getMaterializedViews(), newTable.getMaterializedViews());
// dropped materialized views
for (MaterializedViewDefinition materializedView : materializedViewDiff.entriesOnlyOnLeft().values())
dropMaterializedViewFromSchemaMutation(oldTable, materializedView, timestamp, mutation);
// newly created materialized views
for (MaterializedViewDefinition materializedView : materializedViewDiff.entriesOnlyOnRight().values())
addMaterializedViewToSchemaMutation(oldTable, materializedView, timestamp, mutation);
// updated materialized views need to be updated
for (MapDifference.ValueDifference<MaterializedViewDefinition> diff : materializedViewDiff.entriesDiffering().values())
{
addUpdatedMaterializedViewDefinitionToSchemaMutation(oldTable, diff.rightValue(), timestamp, mutation);
}
return mutation;
}
@ -971,17 +937,6 @@ public final class SchemaKeyspace
return Maps.difference(beforeMap, afterMap);
}
private static MapDifference<String, MaterializedViewDefinition> materializedViewsDiff(MaterializedViews before, MaterializedViews after)
{
Map<String, MaterializedViewDefinition> beforeMap = new HashMap<>();
before.forEach(v -> beforeMap.put(v.viewName, v));
Map<String, MaterializedViewDefinition> afterMap = new HashMap<>();
after.forEach(v -> afterMap.put(v.viewName, v));
return Maps.difference(beforeMap, afterMap);
}
public static Mutation makeDropTableMutation(KeyspaceMetadata keyspace, CFMetaData table, long timestamp)
{
// Include the serialized keyspace in case the target node missed a CREATE KEYSPACE migration (see CASSANDRA-5631).
@ -995,9 +950,6 @@ public final class SchemaKeyspace
for (TriggerMetadata trigger : table.getTriggers())
dropTriggerFromSchemaMutation(table, trigger, timestamp, mutation);
for (MaterializedViewDefinition materializedView : table.getMaterializedViews())
dropMaterializedViewFromSchemaMutation(table, materializedView, timestamp, mutation);
return mutation;
}
@ -1062,12 +1014,8 @@ public final class SchemaKeyspace
Triggers triggers =
readSchemaPartitionForTableAndApply(TRIGGERS, keyspace, table, SchemaKeyspace::createTriggersFromTriggersPartition);
MaterializedViews views =
readSchemaPartitionForTableAndApply(MATERIALIZED_VIEWS, keyspace, table, SchemaKeyspace::createMaterializedViewsFromMaterializedViewsPartition);
return createTableFromTableRowAndColumns(row, columns).droppedColumns(droppedColumns)
.triggers(triggers)
.materializedViews(views);
.triggers(triggers);
}
public static CFMetaData createTableFromTableRowAndColumns(UntypedResultSet.Row row, List<ColumnDefinition> columns)
@ -1084,9 +1032,8 @@ public final class SchemaKeyspace
boolean isCounter = flags.contains(CFMetaData.Flag.COUNTER);
boolean isDense = flags.contains(CFMetaData.Flag.DENSE);
boolean isCompound = flags.contains(CFMetaData.Flag.COMPOUND);
boolean isMaterializedView = flags.contains(CFMetaData.Flag.MATERIALIZEDVIEW);
CFMetaData cfm = CFMetaData.create(keyspace, table, id, isDense, isCompound, isSuper, isCounter, isMaterializedView, columns);
CFMetaData cfm = CFMetaData.create(keyspace, table, id, isDense, isCompound, isSuper, isCounter, columns);
Map<String, String> compaction = new HashMap<>(row.getTextMap("compaction"));
Class<? extends AbstractCompactionStrategy> compactionStrategyClass =
@ -1273,103 +1220,6 @@ public final class SchemaKeyspace
return new TriggerMetadata(name, classOption);
}
/*
* Global Index metadata serialization/deserialization.
*/
private static void addMaterializedViewToSchemaMutation(CFMetaData table, MaterializedViewDefinition materializedView, long timestamp, Mutation mutation)
{
RowUpdateBuilder builder = new RowUpdateBuilder(MaterializedViews, timestamp, mutation)
.clustering(table.cfName, materializedView.viewName);
for (ColumnIdentifier partitionColumn : materializedView.partitionColumns)
builder.addListEntry("target_columns", partitionColumn.toString());
for (ColumnIdentifier clusteringColumn : materializedView.clusteringColumns)
builder = builder.addListEntry("clustering_columns", clusteringColumn.toString());
for (ColumnIdentifier includedColumn : materializedView.included)
builder = builder.addListEntry("included_columns", includedColumn.toString());
builder.build();
}
private static void dropMaterializedViewFromSchemaMutation(CFMetaData table, MaterializedViewDefinition materializedView, long timestamp, Mutation mutation)
{
RowUpdateBuilder.deleteRow(MaterializedViews, timestamp, mutation, table.cfName, materializedView.viewName);
}
private static void addUpdatedMaterializedViewDefinitionToSchemaMutation(CFMetaData table, MaterializedViewDefinition materializedView, long timestamp, Mutation mutation)
{
RowUpdateBuilder builder = new RowUpdateBuilder(MaterializedViews, timestamp, mutation)
.clustering(table.cfName, materializedView.viewName);
builder.resetCollection("target_columns");
for (ColumnIdentifier partitionColumn : materializedView.partitionColumns)
builder.addListEntry("target_columns", partitionColumn.toString());
builder.resetCollection("clustering_columns");
for (ColumnIdentifier clusteringColumn : materializedView.clusteringColumns)
builder = builder.addListEntry("clustering_columns", clusteringColumn.toString());
builder.resetCollection("included_columns");
for (ColumnIdentifier includedColumn : materializedView.included)
builder = builder.addListEntry("included_columns", includedColumn.toString());
builder.build();
}
/**
* Deserialize materialized views from storage-level representation.
*
* @param partition storage-level partition containing the materialized view definitions
* @return the list of processed MaterializedViewDefinitions
*/
private static MaterializedViews createMaterializedViewsFromMaterializedViewsPartition(RowIterator partition)
{
MaterializedViews.Builder views = org.apache.cassandra.schema.MaterializedViews.builder();
String query = String.format("SELECT * FROM %s.%s", NAME, MATERIALIZED_VIEWS);
for (UntypedResultSet.Row row : QueryProcessor.resultify(query, partition))
{
MaterializedViewDefinition mv = createMaterializedViewFromMaterializedViewRow(row);
views.add(mv);
}
return views.build();
}
private static MaterializedViewDefinition createMaterializedViewFromMaterializedViewRow(UntypedResultSet.Row row)
{
String name = row.getString("view_name");
List<String> partitionColumnNames = row.getList("target_columns", UTF8Type.instance);
String cfName = row.getString("columnfamily_name");
List<String> clusteringColumnNames = row.getList("clustering_columns", UTF8Type.instance);
List<ColumnIdentifier> partitionColumns = new ArrayList<>();
for (String columnName : partitionColumnNames)
{
partitionColumns.add(ColumnIdentifier.getInterned(columnName, true));
}
List<ColumnIdentifier> clusteringColumns = new ArrayList<>();
for (String columnName : clusteringColumnNames)
{
clusteringColumns.add(ColumnIdentifier.getInterned(columnName, true));
}
List<String> includedColumnNames = row.getList("included_columns", UTF8Type.instance);
Set<ColumnIdentifier> includedColumns = new HashSet<>();
if (includedColumnNames != null)
{
for (String columnName : includedColumnNames)
includedColumns.add(ColumnIdentifier.getInterned(columnName, true));
}
return new MaterializedViewDefinition(cfName,
name,
partitionColumns,
clusteringColumns,
includedColumns);
}
/*
* UDF metadata serialization/deserialization.
*/

2
src/java/org/apache/cassandra/service/AbstractReadExecutor.java
View File

@ -196,7 +196,7 @@ public abstract class AbstractReadExecutor
return new SpeculatingReadExecutor(keyspace, cfs, command, consistencyLevel, targetReplicas);
}
public static class NeverSpeculatingReadExecutor extends AbstractReadExecutor
private static class NeverSpeculatingReadExecutor extends AbstractReadExecutor
{
public NeverSpeculatingReadExecutor(Keyspace keyspace, ReadCommand command, ConsistencyLevel consistencyLevel, List<InetAddress> targetReplicas)
{

2
src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java
View File

@ -47,7 +47,7 @@ public abstract class AbstractWriteResponseHandler<T> implements IAsyncCallbackW
public final ConsistencyLevel consistencyLevel;
protected final Runnable callback;
protected final Collection<InetAddress> pendingEndpoints;
protected final WriteType writeType;
private final WriteType writeType;
private static final AtomicIntegerFieldUpdater<AbstractWriteResponseHandler> failuresUpdater
= AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures");
private volatile int failures = 0;

121
src/java/org/apache/cassandra/service/BatchlogResponseHandler.java
View File

@ -1,121 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service;
import java.net.InetAddress;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.net.MessageIn;
public class BatchlogResponseHandler<T> extends AbstractWriteResponseHandler<T>
{
AbstractWriteResponseHandler<T> wrapped;
BatchlogCleanup cleanup;
protected volatile int requiredBeforeFinish;
private static final AtomicIntegerFieldUpdater<BatchlogResponseHandler> requiredBeforeFinishUpdater
= AtomicIntegerFieldUpdater.newUpdater(BatchlogResponseHandler.class, "requiredBeforeFinish");
public BatchlogResponseHandler(AbstractWriteResponseHandler<T> wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup)
{
super(wrapped.keyspace, wrapped.naturalEndpoints, wrapped.pendingEndpoints, wrapped.consistencyLevel, wrapped.callback, wrapped.writeType);
this.wrapped = wrapped;
this.requiredBeforeFinish = requiredBeforeFinish;
this.cleanup = cleanup;
}
protected int ackCount()
{
return wrapped.ackCount();
}
public void response(MessageIn<T> msg)
{
wrapped.response(msg);
if (requiredBeforeFinishUpdater.decrementAndGet(this) == 0)
cleanup.run();
}
public boolean isLatencyForSnitch()
{
return wrapped.isLatencyForSnitch();
}
public void onFailure(InetAddress from)
{
wrapped.onFailure(from);
}
public void assureSufficientLiveNodes()
{
wrapped.assureSufficientLiveNodes();
}
public void get() throws WriteTimeoutException, WriteFailureException
{
wrapped.get();
}
protected int totalBlockFor()
{
return wrapped.totalBlockFor();
}
protected int totalEndpoints()
{
return wrapped.totalEndpoints();
}
protected boolean waitingFor(InetAddress from)
{
return wrapped.waitingFor(from);
}
protected void signal()
{
wrapped.signal();
}
public static class BatchlogCleanup
{
private final BatchlogCleanupCallback callback;
protected volatile int mutationsWaitingFor;
private static final AtomicIntegerFieldUpdater<BatchlogCleanup> mutationsWaitingForUpdater
= AtomicIntegerFieldUpdater.newUpdater(BatchlogCleanup.class, "mutationsWaitingFor");
public BatchlogCleanup(int mutationsWaitingFor, BatchlogCleanupCallback callback)
{
this.mutationsWaitingFor = mutationsWaitingFor;
this.callback = callback;
}
public void run()
{
if (mutationsWaitingForUpdater.decrementAndGet(this) == 0)
callback.invoke();
}
}
public interface BatchlogCleanupCallback
{
void invoke();
}
}

18
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -287,24 +287,6 @@ public class CassandraDaemon
}
}
Runnable indexRebuild = new Runnable()
{
@Override
public void run()
{
for (Keyspace keyspace : Keyspace.all())
{
for (ColumnFamilyStore cf: keyspace.getColumnFamilyStores())
{
cf.materializedViewManager.buildAllViews();
}
}
}
};
ScheduledExecutors.optionalTasks.schedule(indexRebuild, StorageService.RING_DELAY, TimeUnit.MILLISECONDS);
SystemKeyspace.finishStartup();
// start server internals

244
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -31,10 +31,6 @@ import com.google.common.base.Predicate;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.*;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.cassandra.db.view.MaterializedViewManager;
import org.apache.cassandra.db.view.MaterializedViewUtils;
import org.apache.cassandra.metrics.*;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -62,6 +58,7 @@ import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.metrics.*;
import org.apache.cassandra.net.*;
import org.apache.cassandra.service.paxos.*;
import org.apache.cassandra.tracing.Tracing;
@ -94,7 +91,6 @@ public class StorageProxy implements StorageProxyMBean
private static final ClientRequestMetrics writeMetrics = new ClientRequestMetrics("Write");
private static final CASClientRequestMetrics casWriteMetrics = new CASClientRequestMetrics("CASWrite");
private static final CASClientRequestMetrics casReadMetrics = new CASClientRequestMetrics("CASRead");
private static final MVWriteMetrics mvWriteMetrics = new MVWriteMetrics("MVWrite");
private static final double CONCURRENT_SUBREQUESTS_MARGIN = 0.10;
@ -122,7 +118,7 @@ public class StorageProxy implements StorageProxyMBean
throws OverloadedException
{
assert mutation instanceof Mutation;
sendToHintedEndpoints((Mutation) mutation, targets, responseHandler, localDataCenter, Stage.MUTATION);
sendToHintedEndpoints((Mutation) mutation, targets, responseHandler, localDataCenter);
}
};
@ -626,80 +622,6 @@ public class StorageProxy implements StorageProxyMBean
Tracing.trace("Wrote hint to satisfy CL.ANY after no replicas acknowledged the write");
}
/**
* Use this method to have these Mutations applied
* across all replicas.
*
* @param mutations the mutations to be applied across the replicas
*/
public static void mutateMV(ByteBuffer dataKey, Collection<Mutation> mutations)
throws UnavailableException, OverloadedException, WriteTimeoutException
{
Tracing.trace("Determining replicas for mutation");
final String localDataCenter = DatabaseDescriptor.getEndpointSnitch().getDatacenter(FBUtilities.getBroadcastAddress());
long startTime = System.nanoTime();
List<WriteResponseHandlerWrapper> wrappers = new ArrayList<>(mutations.size());
try
{
Token baseToken = StorageService.getPartitioner().getToken(dataKey);
ConsistencyLevel consistencyLevel = ConsistencyLevel.ONE;
//Since the base -> view replication is 1:1 we only need to store the BL locally
final Collection<InetAddress> batchlogEndpoints = Collections.singleton(FBUtilities.getBroadcastAddress());
final UUID batchUUID = UUIDGen.getTimeUUID();
BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(),
() -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID));
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
{
String keyspaceName = mutation.getKeyspaceName();
Token tk = mutation.key().getToken();
List<InetAddress> naturalEndpoints = Lists.newArrayList(MaterializedViewUtils.getViewNaturalEndpoint(keyspaceName, baseToken, tk));
WriteResponseHandlerWrapper wrapper = wrapMVBatchResponseHandler(mutation,
consistencyLevel,
consistencyLevel,
naturalEndpoints,
WriteType.BATCH,
cleanup);
wrappers.add(wrapper);
//Apply to local batchlog memtable in this thread
BatchlogManager.getBatchlogMutationFor(mutations, batchUUID, MessagingService.current_version).apply();
}
// now actually perform the writes and wait for them to complete
asyncWriteBatchedMutations(wrappers, localDataCenter, Stage.MATERIALIZED_VIEW_MUTATION);
}
catch (WriteTimeoutException ex)
{
mvWriteMetrics.timeouts.mark();
Tracing.trace("Write timeout; received {} of {} required replies", ex.received, ex.blockFor);
throw ex;
}
catch (UnavailableException e)
{
mvWriteMetrics.unavailables.mark();
Tracing.trace("Unavailable");
throw e;
}
catch (OverloadedException e)
{
mvWriteMetrics.unavailables.mark();
Tracing.trace("Overloaded");
throw e;
}
finally
{
mvWriteMetrics.addNano(System.nanoTime() - startTime);
}
}
@SuppressWarnings("unchecked")
public static void mutateWithTriggers(Collection<? extends IMutation> mutations,
ConsistencyLevel consistencyLevel,
@ -708,17 +630,12 @@ public class StorageProxy implements StorageProxyMBean
{
Collection<Mutation> augmented = TriggerExecutor.instance.execute(mutations);
boolean updatesView = MaterializedViewManager.updatesAffectView(mutations, true);
if (augmented != null)
mutateAtomically(augmented, consistencyLevel, updatesView);
mutateAtomically(augmented, consistencyLevel);
else if (mutateAtomically)
mutateAtomically((Collection<Mutation>) mutations, consistencyLevel);
else
{
if (mutateAtomically || updatesView)
mutateAtomically((Collection<Mutation>) mutations, consistencyLevel, updatesView);
else
mutate(mutations, consistencyLevel);
}
mutate(mutations, consistencyLevel);
}
/**
@ -729,11 +646,8 @@ public class StorageProxy implements StorageProxyMBean
*
* @param mutations the Mutations to be applied across the replicas
* @param consistency_level the consistency level for the operation
* @param requireQuorumForRemove at least a quorum of nodes will see update before deleting batchlog
*/
public static void mutateAtomically(Collection<Mutation> mutations,
ConsistencyLevel consistency_level,
boolean requireQuorumForRemove)
public static void mutateAtomically(Collection<Mutation> mutations, ConsistencyLevel consistency_level)
throws UnavailableException, OverloadedException, WriteTimeoutException
{
Tracing.trace("Determining replicas for atomic batch");
@ -744,49 +658,25 @@ public class StorageProxy implements StorageProxyMBean
try
{
// If we are requiring quorum nodes for removal, we upgrade consistency level to QUORUM unless we already
// require ALL, or EACH_QUORUM. This is so that *at least* QUORUM nodes see the update.
ConsistencyLevel batchConsistencyLevel = requireQuorumForRemove
? ConsistencyLevel.QUORUM
: consistency_level;
switch (consistency_level)
{
case ALL:
case EACH_QUORUM:
batchConsistencyLevel = consistency_level;
}
final Collection<InetAddress> batchlogEndpoints = getBatchlogEndpoints(localDataCenter, batchConsistencyLevel);
final UUID batchUUID = UUIDGen.getTimeUUID();
BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(),
new BatchlogResponseHandler.BatchlogCleanupCallback()
{
public void invoke()
{
asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID);
}
});
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
{
WriteResponseHandlerWrapper wrapper = wrapBatchResponseHandler(mutation,
consistency_level,
batchConsistencyLevel,
WriteType.BATCH,
cleanup);
WriteResponseHandlerWrapper wrapper = wrapResponseHandler(mutation, consistency_level, WriteType.BATCH);
// exit early if we can't fulfill the CL at this time.
wrapper.handler.assureSufficientLiveNodes();
wrappers.add(wrapper);
}
// write to the batchlog
Collection<InetAddress> batchlogEndpoints = getBatchlogEndpoints(localDataCenter, consistency_level);
UUID batchUUID = UUIDGen.getTimeUUID();
syncWriteToBatchlog(mutations, batchlogEndpoints, batchUUID);
// now actually perform the writes and wait for them to complete
syncWriteBatchedMutations(wrappers, localDataCenter, Stage.MUTATION);
syncWriteBatchedMutations(wrappers, localDataCenter);
// remove the batchlog entries asynchronously
asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID);
}
catch (UnavailableException e)
{
@ -829,13 +719,13 @@ public class StorageProxy implements StorageProxyMBean
WriteType.BATCH_LOG);
MessageOut<Mutation> message = BatchlogManager.getBatchlogMutationFor(mutations, uuid, MessagingService.current_version)
.createMessage(MessagingService.Verb.BATCHLOG_MUTATION);
.createMessage();
for (InetAddress target : endpoints)
{
int targetVersion = MessagingService.instance().getVersion(target);
if (canDoLocalRequest(target))
{
insertLocal(Stage.BATCHLOG_MUTATION, message.payload, handler);
insertLocal(message.payload, handler);
}
else if (targetVersion == MessagingService.current_version)
{
@ -844,7 +734,7 @@ public class StorageProxy implements StorageProxyMBean
else
{
MessagingService.instance().sendRR(BatchlogManager.getBatchlogMutationFor(mutations, uuid, targetVersion)
.createMessage(MessagingService.Verb.BATCHLOG_MUTATION),
.createMessage(),
target,
handler,
false);
@ -864,43 +754,25 @@ public class StorageProxy implements StorageProxyMBean
WriteType.SIMPLE);
Mutation mutation = new Mutation(SystemKeyspace.NAME, StorageService.getPartitioner().decorateKey(UUIDType.instance.decompose(uuid)));
mutation.add(PartitionUpdate.fullPartitionDelete(SystemKeyspace.Batchlog, mutation.key(), FBUtilities.timestampMicros(), FBUtilities.nowInSeconds()));
MessageOut<Mutation> message = mutation.createMessage(MessagingService.Verb.BATCHLOG_MUTATION);
MessageOut<Mutation> message = mutation.createMessage();
for (InetAddress target : endpoints)
{
if (canDoLocalRequest(target))
insertLocal(Stage.BATCHLOG_MUTATION, message.payload, handler);
insertLocal(message.payload, handler);
else
MessagingService.instance().sendRR(message, target, handler, false);
}
}
private static void asyncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter, Stage stage)
{
for (WriteResponseHandlerWrapper wrapper : wrappers)
{
Iterable<InetAddress> endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints);
try
{
sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage);
}
catch (OverloadedException | WriteTimeoutException e)
{
wrapper.handler.onFailure(FBUtilities.getBroadcastAddress());
}
}
}
private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter, Stage stage)
private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter)
throws WriteTimeoutException, OverloadedException
{
for (WriteResponseHandlerWrapper wrapper : wrappers)
{
Iterable<InetAddress> endpoints = Iterables.concat(wrapper.handler.naturalEndpoints, wrapper.handler.pendingEndpoints);
sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter, stage);
sendToHintedEndpoints(wrapper.mutation, endpoints, wrapper.handler, localDataCenter);
}
for (WriteResponseHandlerWrapper wrapper : wrappers)
wrapper.handler.get();
}
@ -943,52 +815,25 @@ public class StorageProxy implements StorageProxyMBean
return responseHandler;
}
// same as performWrites except does not initiate writes (but does perform availability checks).
private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation,
ConsistencyLevel consistency_level,
ConsistencyLevel batchConsistencyLevel,
WriteType writeType,
BatchlogResponseHandler.BatchlogCleanup cleanup)
// same as above except does not initiate writes (but does perform availability checks).
private static WriteResponseHandlerWrapper wrapResponseHandler(Mutation mutation, ConsistencyLevel consistency_level, WriteType writeType)
{
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
AbstractReplicationStrategy rs = Keyspace.open(mutation.getKeyspaceName()).getReplicationStrategy();
String keyspaceName = mutation.getKeyspaceName();
Token tk = mutation.key().getToken();
List<InetAddress> naturalEndpoints = StorageService.instance.getNaturalEndpoints(keyspaceName, tk);
Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType);
BatchlogResponseHandler<IMutation> batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(keyspace), cleanup);
return new WriteResponseHandlerWrapper(batchHandler, mutation);
}
/**
* Same as performWrites except does not initiate writes (but does perform availability checks).
* Keeps track of MVWriteMetrics
*/
private static WriteResponseHandlerWrapper wrapMVBatchResponseHandler(Mutation mutation,
ConsistencyLevel consistency_level,
ConsistencyLevel batchConsistencyLevel,
List<InetAddress> naturalEndpoints,
WriteType writeType,
BatchlogResponseHandler.BatchlogCleanup cleanup)
{
Keyspace keyspace = Keyspace.open(mutation.getKeyspaceName());
AbstractReplicationStrategy rs = keyspace.getReplicationStrategy();
String keyspaceName = mutation.getKeyspaceName();
Token tk = mutation.key().getToken();
Collection<InetAddress> pendingEndpoints = StorageService.instance.getTokenMetadata().pendingEndpointsFor(tk, keyspaceName);
AbstractWriteResponseHandler<IMutation> writeHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType);
BatchlogResponseHandler<IMutation> batchHandler = new MVWriteMetricsWrapped(writeHandler, batchConsistencyLevel.blockFor(keyspace), cleanup);
return new WriteResponseHandlerWrapper(batchHandler, mutation);
AbstractWriteResponseHandler<IMutation> responseHandler = rs.getWriteResponseHandler(naturalEndpoints, pendingEndpoints, consistency_level, null, writeType);
return new WriteResponseHandlerWrapper(responseHandler, mutation);
}
// used by atomic_batch_mutate to decouple availability check from the write itself, caches consistency level and endpoints.
private static class WriteResponseHandlerWrapper
{
final BatchlogResponseHandler<IMutation> handler;
final AbstractWriteResponseHandler<IMutation> handler;
final Mutation mutation;
WriteResponseHandlerWrapper(BatchlogResponseHandler<IMutation> handler, Mutation mutation)
WriteResponseHandlerWrapper(AbstractWriteResponseHandler<IMutation> handler, Mutation mutation)
{
this.handler = handler;
this.mutation = mutation;
@ -1041,8 +886,7 @@ public class StorageProxy implements StorageProxyMBean
public static void sendToHintedEndpoints(final Mutation mutation,
Iterable<InetAddress> targets,
AbstractWriteResponseHandler<IMutation> responseHandler,
String localDataCenter,
Stage stage)
String localDataCenter)
throws OverloadedException
{
// extra-datacenter replicas, grouped by dc
@ -1106,7 +950,7 @@ public class StorageProxy implements StorageProxyMBean
}
if (insertLocal)
insertLocal(stage, mutation, responseHandler);
insertLocal(mutation, responseHandler);
if (dcGroups != null)
{
@ -1215,9 +1059,10 @@ public class StorageProxy implements StorageProxyMBean
}
}
private static void insertLocal(Stage stage, final Mutation mutation, final AbstractWriteResponseHandler<IMutation> responseHandler)
private static void insertLocal(final Mutation mutation, final AbstractWriteResponseHandler<IMutation> responseHandler)
{
StageManager.getStage(stage).maybeExecuteImmediately(new LocalMutationRunnable()
StageManager.getStage(Stage.MUTATION).maybeExecuteImmediately(new LocalMutationRunnable()
{
public void runMayThrow()
{
@ -1228,8 +1073,7 @@ public class StorageProxy implements StorageProxyMBean
}
catch (Exception ex)
{
if (!(ex instanceof WriteTimeoutException))
logger.error("Failed to apply mutation locally : {}", ex);
logger.error("Failed to apply mutation locally : {}", ex);
responseHandler.onFailure(FBUtilities.getBroadcastAddress());
}
}
@ -1349,7 +1193,7 @@ public class StorageProxy implements StorageProxyMBean
Set<InetAddress> remotes = Sets.difference(ImmutableSet.copyOf(targets),
ImmutableSet.of(FBUtilities.getBroadcastAddress()));
if (!remotes.isEmpty())
sendToHintedEndpoints(result, remotes, responseHandler, localDataCenter, Stage.COUNTER_MUTATION);
sendToHintedEndpoints(result, remotes, responseHandler, localDataCenter);
}
};
}
@ -2278,24 +2122,6 @@ public class StorageProxy implements StorageProxyMBean
ConsistencyLevel consistencyLevel) throws OverloadedException;
}
/**
* This class captures metrics for materialized views writes.
*/
private static class MVWriteMetricsWrapped extends BatchlogResponseHandler<IMutation>
{
public MVWriteMetricsWrapped(AbstractWriteResponseHandler<IMutation> writeHandler, int i, BatchlogCleanup cleanup)
{
super(writeHandler, i, cleanup);
mvWriteMetrics.viewReplicasAttempted.inc(totalEndpoints());
}
public void response(MessageIn<IMutation> msg)
{
super.response(msg);
mvWriteMetrics.viewReplicasSuccess.inc();
}
}
/**
* A Runnable that aborts if it doesn't start running before it times out
*/

24
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -297,8 +297,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
/* register the verb handlers */
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MUTATION, new MutationVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.BATCHLOG_MUTATION, new MutationVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MATERIALIZED_VIEW_MUTATION, new MutationVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ_REPAIR, new ReadRepairVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, new ReadCommandVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, new ReadCommandVerbHandler());
@ -631,14 +629,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public void runMayThrow() throws InterruptedException
{
inShutdownHook = true;
ExecutorService materializedViewMutationStage = StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION);
ExecutorService batchlogMutationStage = StageManager.getStage(Stage.BATCHLOG_MUTATION);
ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
ExecutorService mutationStage = StageManager.getStage(Stage.MUTATION);
if (mutationStage.isShutdown()
&& counterMutationStage.isShutdown()
&& batchlogMutationStage.isShutdown()
&& materializedViewMutationStage.isShutdown())
if (mutationStage.isShutdown() && counterMutationStage.isShutdown())
return; // drained already
if (daemon != null)
@ -649,12 +642,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
// In-progress writes originating here could generate hints to be written, so shut down MessagingService
// before mutation stage, so we can get all the hints saved before shutting down
MessagingService.instance().shutdown();
materializedViewMutationStage.shutdown();
batchlogMutationStage.shutdown();
counterMutationStage.shutdown();
mutationStage.shutdown();
materializedViewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
batchlogMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
counterMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
mutationStage.awaitTermination(3600, TimeUnit.SECONDS);
StorageProxy.instance.verifyNoHintsInProgress();
@ -3831,13 +3820,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
inShutdownHook = true;
ExecutorService counterMutationStage = StageManager.getStage(Stage.COUNTER_MUTATION);
ExecutorService batchlogMutationStage = StageManager.getStage(Stage.BATCHLOG_MUTATION);
ExecutorService materializedViewMutationStage = StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION);
ExecutorService mutationStage = StageManager.getStage(Stage.MUTATION);
if (mutationStage.isTerminated()
&& counterMutationStage.isTerminated()
&& batchlogMutationStage.isTerminated()
&& materializedViewMutationStage.isTerminated())
if (mutationStage.isTerminated() && counterMutationStage.isTerminated())
{
logger.warn("Cannot drain node (did it already happen?)");
return;
@ -3851,12 +3835,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
MessagingService.instance().shutdown();
setMode(Mode.DRAINING, "clearing mutation stage", false);
materializedViewMutationStage.shutdown();
batchlogMutationStage.shutdown();
counterMutationStage.shutdown();
mutationStage.shutdown();
materializedViewMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
batchlogMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
counterMutationStage.awaitTermination(3600, TimeUnit.SECONDS);
mutationStage.awaitTermination(3600, TimeUnit.SECONDS);

85
src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
View File

@ -24,25 +24,14 @@ import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.RowIterators;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.db.rows.UnfilteredRowIterators;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.concurrent.Refs;
@ -52,8 +41,6 @@ import org.apache.cassandra.utils.concurrent.Refs;
*/
public class StreamReceiveTask extends StreamTask
{
private static final Logger logger = LoggerFactory.getLogger(StreamReceiveTask.class);
private static final ExecutorService executor = Executors.newCachedThreadPool(new NamedThreadFactory("StreamReceiveTask"));
// number of files to receive
@ -133,69 +120,21 @@ public class StreamReceiveTask extends StreamTask
return;
}
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
boolean hasMaterializedViews = cfs.materializedViewManager.allViews().iterator().hasNext();
try
List<SSTableReader> readers = new ArrayList<>();
for (SSTableWriter writer : task.sstables)
readers.add(writer.finish(true));
task.txn.finish();
task.sstables.clear();
try (Refs<SSTableReader> refs = Refs.ref(readers))
{
List<SSTableReader> readers = new ArrayList<>();
for (SSTableWriter writer : task.sstables)
{
SSTableReader reader = writer.finish(true);
readers.add(reader);
task.txn.update(reader, false);
}
task.sstables.clear();
try (Refs<SSTableReader> refs = Refs.ref(readers))
{
//We have a special path for Materialized view.
//Since the MV requires cleaning up any pre-existing state, we must put
//all partitions through the same write path as normal mutations.
//This also ensures any 2is are also updated
if (hasMaterializedViews)
{
for (SSTableReader reader : readers)
{
try (ISSTableScanner scanner = reader.getScanner())
{
while (scanner.hasNext())
{
try (UnfilteredRowIterator rowIterator = scanner.next())
{
new Mutation(PartitionUpdate.fromIterator(rowIterator)).apply();
}
}
}
}
}
else
{
task.txn.finish();
// add sstables and build secondary indexes
cfs.addSSTables(readers);
cfs.indexManager.maybeBuildSecondaryIndexes(readers, cfs.indexManager.allIndexesNames());
}
}
catch (Throwable t)
{
logger.error("Error applying streamed sstable: ", t);
JVMStabilityInspector.inspectThrowable(t);
}
finally
{
//We don't keep the streamed sstables since we've applied them manually
//So we abort the txn and delete the streamed sstables
if (hasMaterializedViews)
task.txn.abort();
}
}
finally
{
task.session.taskCompleted(task);
// add sstables and build secondary indexes
cfs.addSSTables(readers);
cfs.indexManager.maybeBuildSecondaryIndexes(readers, cfs.indexManager.allIndexesNames());
}
task.session.taskCompleted(task);
}
}

28
src/java/org/apache/cassandra/thrift/CassandraServer.java
View File

@ -805,9 +805,6 @@ public class CassandraServer implements Cassandra.Iface
cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.MODIFY);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family, false);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
ThriftValidation.validateKey(metadata, key);
ThriftValidation.validateColumnParent(metadata, column_parent);
// SuperColumn field is usually optional, but not when we're inserting
@ -901,9 +898,6 @@ public class CassandraServer implements Cassandra.Iface
cState.hasColumnFamilyAccess(keyspace, column_family, Permission.SELECT);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family, false);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
ThriftValidation.validateKey(metadata, key);
if (metadata.isSuper())
throw new org.apache.cassandra.exceptions.InvalidRequestException("CAS does not support supercolumns");
@ -1096,9 +1090,6 @@ public class CassandraServer implements Cassandra.Iface
cState.hasColumnFamilyAccess(keyspace, cfName, Permission.MODIFY);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, cfName);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
ThriftValidation.validateKey(metadata, key);
if (metadata.isCounter())
ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata);
@ -1312,9 +1303,6 @@ public class CassandraServer implements Cassandra.Iface
cState.hasColumnFamilyAccess(keyspace, column_path.column_family, Permission.MODIFY);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_path.column_family, isCommutativeOp);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
ThriftValidation.validateKey(metadata, key);
ThriftValidation.validateColumnPathOrParent(metadata, column_path);
if (isCommutativeOp)
@ -1884,11 +1872,6 @@ public class CassandraServer implements Cassandra.Iface
{
String keyspace = cState.getKeyspace();
cState.hasColumnFamilyAccess(keyspace, column_family, Permission.DROP);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_family);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot drop Materialized Views from Thrift");
MigrationManager.announceColumnFamilyDrop(keyspace, column_family);
return Schema.instance.getVersion().toString();
}
@ -1999,11 +1982,6 @@ public class CassandraServer implements Cassandra.Iface
if (oldCfm == null)
throw new InvalidRequestException("Could not find table definition to modify.");
if (oldCfm.isMaterializedView())
throw new InvalidRequestException("Cannot modify Materialized View table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify Materialized View tables instead.");
if (!oldCfm.getMaterializedViews().isEmpty())
throw new InvalidRequestException("Cannot modify table with Materialized View " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify tables with Materialized Views instead.");
if (!oldCfm.isThriftCompatible())
throw new InvalidRequestException("Cannot modify CQL3 table " + oldCfm.cfName + " as it may break the schema. You should use cqlsh to modify CQL3 tables instead.");
@ -2031,9 +2009,6 @@ public class CassandraServer implements Cassandra.Iface
{
String keyspace = cState.getKeyspace();
cState.hasColumnFamilyAccess(keyspace, cfname, Permission.MODIFY);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, cfname, true);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot truncate Materialized Views");
if (startSessionIfRequested())
{
@ -2119,9 +2094,6 @@ public class CassandraServer implements Cassandra.Iface
cState.hasColumnFamilyAccess(keyspace, column_parent.column_family, Permission.MODIFY);
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family, true);
if (metadata.isMaterializedView())
throw new org.apache.cassandra.exceptions.InvalidRequestException("Cannot modify Materialized Views directly");
ThriftValidation.validateKey(metadata, key);
ThriftConversion.fromThrift(consistency_level).validateCounterForWrite(metadata);
ThriftValidation.validateColumnParent(metadata, column_parent);

5
src/java/org/apache/cassandra/thrift/ThriftConversion.java
View File

@ -274,10 +274,7 @@ public class ThriftConversion
subComparator,
defaultValidator);
// We do not allow Thrift materialized views, so we always set it to false
boolean isMaterializedView = false;
CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, isMaterializedView, defs);
CFMetaData newCFMD = CFMetaData.create(cf_def.keyspace, cf_def.name, cfId, isDense, isCompound, isSuper, isCounter, defs);
if (cf_def.isSetGc_grace_seconds())
newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds);

3
src/java/org/apache/cassandra/utils/NativeSSTableLoaderClient.java
View File

@ -118,7 +118,6 @@ public class NativeSSTableLoaderClient extends SSTableLoader.Client
boolean isCounter = flags.contains(CFMetaData.Flag.COUNTER);
boolean isDense = flags.contains(CFMetaData.Flag.DENSE);
boolean isCompound = flags.contains(CFMetaData.Flag.COMPOUND);
boolean isMaterializedView = flags.contains(CFMetaData.Flag.MATERIALIZEDVIEW);
String columnsQuery = String.format("SELECT * FROM %s.%s WHERE keyspace_name = ? AND table_name = ?",
SchemaKeyspace.NAME,
@ -128,7 +127,7 @@ public class NativeSSTableLoaderClient extends SSTableLoader.Client
for (Row colRow : session.execute(columnsQuery, keyspace, name))
defs.add(createDefinitionFromRow(colRow, keyspace, name));
tables.put(name, CFMetaData.create(keyspace, name, id, isDense, isCompound, isSuper, isCounter, isMaterializedView, defs));
tables.put(name, CFMetaData.create(keyspace, name, id, isDense, isCompound, isSuper, isCounter, defs));
}
return tables;

194
test/long/org/apache/cassandra/cql3/MaterializedViewLongTest.java
View File

@ -1,194 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
import com.datastax.driver.core.exceptions.WriteTimeoutException;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.db.BatchlogManager;
import org.apache.cassandra.utils.WrappedRunnable;
public class MaterializedViewLongTest extends CQLTester
{
int protocolVersion = 4;
private final List<String> materializedViews = new ArrayList<>();
@BeforeClass
public static void startup()
{
requireNetwork();
}
@Before
public void begin()
{
materializedViews.clear();
}
@After
public void end() throws Throwable
{
for (String viewName : materializedViews)
executeNet(protocolVersion, "DROP MATERIALIZED VIEW " + viewName);
}
private void createView(String name, String query) throws Throwable
{
executeNet(protocolVersion, String.format(query, name));
// If exception is thrown, the view will not be added to the list; since it shouldn't have been created, this is
// the desired behavior
materializedViews.add(name);
}
@Test
public void testConflictResolution() throws Throwable
{
final int writers = 96;
final int insertsPerWriter = 50;
final Map<Integer, Exception> failedWrites = new ConcurrentHashMap<>();
createTable("CREATE TABLE %s (" +
"a int," +
"b int," +
"c int," +
"PRIMARY KEY (a, b))");
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE c IS NOT NULL AND a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (c, a, b)");
CyclicBarrier semaphore = new CyclicBarrier(writers);
Thread[] threads = new Thread[writers];
for (int i = 0; i < writers; i++)
{
final int writer = i;
Thread t = new Thread(new WrappedRunnable()
{
public void runMayThrow()
{
try
{
int writerOffset = writer * insertsPerWriter;
semaphore.await();
for (int i = 0; i < insertsPerWriter; i++)
{
try
{
executeNet(protocolVersion, "INSERT INTO %s (a, b, c) VALUES (?, ?, ?)",
1,
1,
i + writerOffset);
}
catch (NoHostAvailableException|WriteTimeoutException e)
{
failedWrites.put(i + writerOffset, e);
}
}
}
catch (Throwable e)
{
throw new RuntimeException(e);
}
}
});
t.start();
threads[i] = t;
}
for (int i = 0; i < writers; i++)
threads[i].join();
for (int i = 0; i < writers * insertsPerWriter; i++)
{
if (executeNet(protocolVersion, "SELECT COUNT(*) FROM system.batchlog").one().getLong(0) == 0)
break;
try
{
// This will throw exceptions whenever there are exceptions trying to push the materialized view values
// out, caused by the view becoming overwhelmed.
BatchlogManager.instance.startBatchlogReplay().get();
}
catch (Throwable ignore)
{
}
}
while (!(((SEPExecutor) StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION)).getPendingTasks() == 0
&& ((SEPExecutor) StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION)).getActiveCount() == 0))
{
Thread.sleep(1);
}
int value = executeNet(protocolVersion, "SELECT c FROM %s WHERE a = 1 AND b = 1").one().getInt("c");
List<Row> rows = executeNet(protocolVersion, "SELECT c FROM " + keyspace() + ".mv").all();
boolean containsC = false;
StringBuilder others = new StringBuilder();
StringBuilder overlappingFailedWrites = new StringBuilder();
for (Row row : rows)
{
int c = row.getInt("c");
if (c == value)
containsC = true;
else
{
if (others.length() != 0)
others.append(' ');
others.append(c);
if (failedWrites.containsKey(c))
{
if (overlappingFailedWrites.length() != 0)
overlappingFailedWrites.append(' ');
overlappingFailedWrites.append(c)
.append(':')
.append(failedWrites.get(c).getMessage());
}
}
}
if (rows.size() > 1)
{
throw new AssertionError(String.format("Expected 1 row, but found %d; %s c = %d, and (%s) of which (%s) failed to insert", rows.size(), containsC ? "found row with" : "no rows contained", value, others, overlappingFailedWrites));
}
else if (rows.isEmpty())
{
throw new AssertionError(String.format("Could not find row with c = %d", value));
}
else if (rows.size() == 1 && !containsC)
{
throw new AssertionError(String.format("Single row had c = %d, expected %d", rows.get(0).getInt("c"), value));
}
}
}

114
src/java/org/apache/cassandra/db/AbstractReadCommandBuilder.java → test/unit/org/apache/cassandra/AbstractReadCommandBuilder.java
View File

@ -1,4 +1,5 @@
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -7,26 +8,25 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
package org.apache.cassandra;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.db.filter.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.marshal.*;
@ -41,9 +41,9 @@ public abstract class AbstractReadCommandBuilder
private int cqlLimit = -1;
private int pagingLimit = -1;
protected boolean reversed = false;
private boolean reversed = false;
protected Set<ColumnIdentifier> columns;
private Set<ColumnIdentifier> columns;
protected final RowFilter filter = RowFilter.create();
private Slice.Bound lowerClusteringBound;
@ -181,13 +181,13 @@ public abstract class AbstractReadCommandBuilder
protected ColumnFilter makeColumnFilter()
{
if (columns == null || columns.isEmpty())
if (columns == null)
return ColumnFilter.all(cfs.metadata);
ColumnFilter.Builder filter = ColumnFilter.selectionBuilder();
ColumnFilter.Builder builder = ColumnFilter.allColumnsBuilder(cfs.metadata);
for (ColumnIdentifier column : columns)
filter.add(cfs.metadata.getColumnDefinition(column));
return filter.build();
builder.add(cfs.metadata.getColumnDefinition(column));
return builder.build();
}
protected ClusteringIndexFilter makeFilter()
@ -212,13 +212,33 @@ public abstract class AbstractReadCommandBuilder
return limits;
}
public Row getOnlyRow()
{
return Util.getOnlyRow(build());
}
public Row getOnlyRowUnfiltered()
{
return Util.getOnlyRowUnfiltered(build());
}
public FilteredPartition getOnlyPartition()
{
return Util.getOnlyPartition(build());
}
public Partition getOnlyPartitionUnfiltered()
{
return Util.getOnlyPartitionUnfiltered(build());
}
public abstract ReadCommand build();
public static class SinglePartitionBuilder extends AbstractReadCommandBuilder
{
private final DecoratedKey partitionKey;
public SinglePartitionBuilder(ColumnFamilyStore cfs, DecoratedKey key)
SinglePartitionBuilder(ColumnFamilyStore cfs, DecoratedKey key)
{
super(cfs);
this.partitionKey = key;
@ -231,37 +251,6 @@ public abstract class AbstractReadCommandBuilder
}
}
public static class SinglePartitionSliceBuilder extends AbstractReadCommandBuilder
{
private final DecoratedKey partitionKey;
private Slices.Builder sliceBuilder;
public SinglePartitionSliceBuilder(ColumnFamilyStore cfs, DecoratedKey key)
{
super(cfs);
this.partitionKey = key;
sliceBuilder = new Slices.Builder(cfs.getComparator());
}
public SinglePartitionSliceBuilder addSlice(Slice slice)
{
sliceBuilder.add(slice);
return this;
}
@Override
protected ClusteringIndexFilter makeFilter()
{
return new ClusteringIndexSliceFilter(sliceBuilder.build(), reversed);
}
@Override
public ReadCommand build()
{
return SinglePartitionSliceCommand.create(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), partitionKey, makeFilter());
}
}
public static class PartitionRangeBuilder extends AbstractReadCommandBuilder
{
private DecoratedKey startKey;
@ -269,7 +258,7 @@ public abstract class AbstractReadCommandBuilder
private DecoratedKey endKey;
private boolean endInclusive;
public PartitionRangeBuilder(ColumnFamilyStore cfs)
PartitionRangeBuilder(ColumnFamilyStore cfs)
{
super(cfs);
}
@ -278,7 +267,7 @@ public abstract class AbstractReadCommandBuilder
{
assert startKey == null;
this.startInclusive = true;
this.startKey = makeKey(cfs.metadata, values);
this.startKey = Util.makeKey(cfs.metadata, values);
return this;
}
@ -286,7 +275,7 @@ public abstract class AbstractReadCommandBuilder
{
assert startKey == null;
this.startInclusive = false;
this.startKey = makeKey(cfs.metadata, values);
this.startKey = Util.makeKey(cfs.metadata, values);
return this;
}
@ -294,7 +283,7 @@ public abstract class AbstractReadCommandBuilder
{
assert endKey == null;
this.endInclusive = true;
this.endKey = makeKey(cfs.metadata, values);
this.endKey = Util.makeKey(cfs.metadata, values);
return this;
}
@ -302,7 +291,7 @@ public abstract class AbstractReadCommandBuilder
{
assert endKey == null;
this.endInclusive = false;
this.endKey = makeKey(cfs.metadata, values);
this.endKey = Util.makeKey(cfs.metadata, values);
return this;
}
@ -321,27 +310,18 @@ public abstract class AbstractReadCommandBuilder
end = StorageService.getPartitioner().getMinimumToken().maxKeyBound();
endInclusive = true;
}
AbstractBounds<PartitionPosition> bounds;
if (startInclusive && endInclusive)
bounds = new Bounds<>(start, end);
bounds = new Bounds<PartitionPosition>(start, end);
else if (startInclusive && !endInclusive)
bounds = new IncludingExcludingBounds<>(start, end);
bounds = new IncludingExcludingBounds<PartitionPosition>(start, end);
else if (!startInclusive && endInclusive)
bounds = new Range<>(start, end);
bounds = new Range<PartitionPosition>(start, end);
else
bounds = new ExcludingBounds<>(start, end);
bounds = new ExcludingBounds<PartitionPosition>(start, end);
return new PartitionRangeReadCommand(cfs.metadata, nowInSeconds, makeColumnFilter(), filter, makeLimits(), new DataRange(bounds, makeFilter()));
}
static DecoratedKey makeKey(CFMetaData metadata, Object... partitionKey)
{
if (partitionKey.length == 1 && partitionKey[0] instanceof DecoratedKey)
return (DecoratedKey)partitionKey[0];
ByteBuffer key = CFMetaData.serializePartitionKey(metadata.getKeyValidatorAsClusteringComparator().make(partitionKey));
return StorageService.getPartitioner().decorateKey(key);
}
}
}

4
test/unit/org/apache/cassandra/cql3/CQLTester.java
View File

@ -760,8 +760,8 @@ public abstract class CQLTester
if (!Objects.equal(expectedByteValue, actualValue))
{
Object actualValueDecoded = actualValue == null ? null : column.type.getSerializer().deserialize(actualValue);
if (!expected[j].equals(actualValueDecoded))
Object actualValueDecoded = column.type.getSerializer().deserialize(actualValue);
if (!actualValueDecoded.equals(expected[j]))
Assert.fail(String.format("Invalid value for row %d column %d (%s of type %s), expected <%s> but got <%s>",
i,
j,

955
test/unit/org/apache/cassandra/cql3/MaterializedViewTest.java
View File

@ -1,955 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.cql3;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.InetAddress;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.datastax.driver.core.*;
import com.datastax.driver.core.exceptions.InvalidQueryException;
import junit.framework.Assert;
import org.apache.cassandra.concurrent.SEPExecutor;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.serializers.SimpleDateSerializer;
import org.apache.cassandra.serializers.TimeSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
public class MaterializedViewTest extends CQLTester
{
int protocolVersion = 4;
private final List<String> materializedViews = new ArrayList<>();
@BeforeClass
public static void startup()
{
requireNetwork();
}
@Before
public void begin()
{
materializedViews.clear();
}
@After
public void end() throws Throwable
{
for (String viewName : materializedViews)
executeNet(protocolVersion, "DROP MATERIALIZED VIEW " + viewName);
}
private void createView(String name, String query) throws Throwable
{
executeNet(protocolVersion, String.format(query, name));
// If exception is thrown, the view will not be added to the list; since it shouldn't have been created, this is
// the desired behavior
materializedViews.add(name);
}
private void updateMV(String query, Object... params) throws Throwable
{
executeNet(protocolVersion, query, params);
while (!(((SEPExecutor) StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION)).getPendingTasks() == 0
&& ((SEPExecutor) StageManager.getStage(Stage.MATERIALIZED_VIEW_MUTATION)).getActiveCount() == 0))
{
Thread.sleep(1);
}
}
@Test
public void testAccessAndSchema() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"asciival ascii, " +
"bigintval bigint, " +
"PRIMARY KEY((k, asciival)))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv1_test", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE bigintval IS NOT NULL AND k IS NOT NULL AND asciival IS NOT NULL PRIMARY KEY (bigintval, k, asciival)");
updateMV("INSERT INTO %s(k,asciival,bigintval)VALUES(?,?,?)", 0, "foo", 1L);
try
{
updateMV("INSERT INTO mv1_test(k,asciival,bigintval) VALUES(?,?,?)", 1, "foo", 2L);
Assert.fail("Shouldn't be able to modify a MV directly");
}
catch (Exception e)
{
}
try
{
executeNet(protocolVersion, "ALTER TABLE mv1_test ADD foo text");
Assert.fail("Should not be able to use alter table with MV");
}
catch (Exception e)
{
}
try
{
executeNet(protocolVersion, "ALTER TABLE mv1_test WITH compaction = { 'class' : 'LeveledCompactionStrategy' }");
Assert.fail("Should not be able to use alter table with MV");
}
catch (Exception e)
{
}
executeNet(protocolVersion, "ALTER MATERIALIZED VIEW mv1_test WITH compaction = { 'class' : 'LeveledCompactionStrategy' }");
//Test alter add
executeNet(protocolVersion, "ALTER TABLE %s ADD foo text");
CFMetaData metadata = Schema.instance.getCFMetaData(keyspace(), "mv1_test");
Assert.assertNotNull(metadata.getColumnDefinition(ByteBufferUtil.bytes("foo")));
updateMV("INSERT INTO %s(k,asciival,bigintval,foo)VALUES(?,?,?,?)", 0, "foo", 1L, "bar");
assertRows(execute("SELECT foo from %s"), row("bar"));
//Test alter rename
executeNet(protocolVersion, "ALTER TABLE %s RENAME asciival TO bar");
assertRows(execute("SELECT bar from %s"), row("foo"));
metadata = Schema.instance.getCFMetaData(keyspace(), "mv1_test");
Assert.assertNotNull(metadata.getColumnDefinition(ByteBufferUtil.bytes("bar")));
}
@Test
public void testStaticTable() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"c int, " +
"sval text static, " +
"val text, " +
"PRIMARY KEY(k,c))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
try
{
createView("mv_static", "CREATE MATERIALIZED VIEW %%s AS SELECT * FROM %s WHERE sval IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (sval,k,c)");
Assert.fail("MV on static should fail");
}
catch (InvalidQueryException e)
{
}
createView("mv_static", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
for (int i = 0; i < 100; i++)
updateMV("INSERT into %s (k,c,sval,val)VALUES(?,?,?,?)", 0, i % 2, "bar" + i, "baz");
Assert.assertEquals(2, execute("select * from %s").size());
assertRows(execute("SELECT sval from %s"), row("bar99"), row("bar99"));
Assert.assertEquals(2, execute("select * from mv_static").size());
assertInvalid("SELECT sval from mv_static");
}
@Test
public void testOldTimestamps() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"c int, " +
"val text, " +
"PRIMARY KEY(k,c))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv_tstest", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE val IS NOT NULL AND k IS NOT NULL AND c IS NOT NULL PRIMARY KEY (val,k,c)");
for (int i = 0; i < 100; i++)
updateMV("INSERT into %s (k,c,val)VALUES(?,?,?)", 0, i % 2, "baz");
Keyspace.open(keyspace()).getColumnFamilyStore(currentTable()).forceBlockingFlush();
Assert.assertEquals(2, execute("select * from %s").size());
Assert.assertEquals(2, execute("select * from mv_tstest").size());
assertRows(execute("SELECT val from %s where k = 0 and c = 0"), row("baz"));
assertRows(execute("SELECT c from mv_tstest where k = 0 and val = ?", "baz"), row(0), row(1));
//Make sure an old TS does nothing
updateMV("UPDATE %s USING TIMESTAMP 100 SET val = ? where k = ? AND c = ?", "bar", 0, 0);
assertRows(execute("SELECT val from %s where k = 0 and c = 0"), row("baz"));
assertRows(execute("SELECT c from mv_tstest where k = 0 and val = ?", "baz"), row(0), row(1));
assertRows(execute("SELECT c from mv_tstest where k = 0 and val = ?", "bar"));
//Latest TS
updateMV("UPDATE %s SET val = ? where k = ? AND c = ?", "bar", 0, 0);
assertRows(execute("SELECT val from %s where k = 0 and c = 0"), row("bar"));
assertRows(execute("SELECT c from mv_tstest where k = 0 and val = ?", "bar"), row(0));
assertRows(execute("SELECT c from mv_tstest where k = 0 and val = ?", "baz"), row(1));
}
@Test
public void testCountersTable() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int PRIMARY KEY, " +
"count counter)");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
try
{
createView("mv_counter", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE count IS NOT NULL AND k IS NOT NULL PRIMARY KEY (count,k)");
Assert.fail("MV on counter should fail");
}
catch (InvalidQueryException e)
{
}
}
@Test
public void testRangeTombstone() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"asciival ascii, " +
"bigintval bigint, " +
"textval1 text, " +
"textval2 text, " +
"PRIMARY KEY((k, asciival), bigintval, textval1)" +
")");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv_test1", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE textval2 IS NOT NULL AND k IS NOT NULL AND asciival IS NOT NULL AND bigintval IS NOT NULL AND textval1 IS NOT NULL PRIMARY KEY ((textval2, k), asciival, bigintval, textval1)");
for (int i = 0; i < 100; i++)
updateMV("INSERT into %s (k,asciival,bigintval,textval1,textval2)VALUES(?,?,?,?,?)", 0, "foo", (long) i % 2, "bar" + i, "baz");
Assert.assertEquals(50, execute("select * from %s where k = 0 and asciival = 'foo' and bigintval = 0").size());
Assert.assertEquals(50, execute("select * from %s where k = 0 and asciival = 'foo' and bigintval = 1").size());
Assert.assertEquals(100, execute("select * from mv_test1").size());
//Check the builder works
createView("mv_test2", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE textval2 IS NOT NULL AND k IS NOT NULL AND asciival IS NOT NULL AND bigintval IS NOT NULL AND textval1 IS NOT NULL PRIMARY KEY ((textval2, k), asciival, bigintval, textval1)");
while (!SystemKeyspace.isViewBuilt(keyspace(), "mv_test2"))
Thread.sleep(1000);
Assert.assertEquals(100, execute("select * from mv_test2").size());
createView("mv_test3", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE textval2 IS NOT NULL AND k IS NOT NULL AND asciival IS NOT NULL AND bigintval IS NOT NULL AND textval1 IS NOT NULL PRIMARY KEY ((textval2, k), bigintval, textval1, asciival)");
while (!SystemKeyspace.isViewBuilt(keyspace(), "mv_test3"))
Thread.sleep(1000);
Assert.assertEquals(100, execute("select * from mv_test3").size());
Assert.assertEquals(100, execute("select asciival from mv_test3 where textval2 = ? and k = ?", "baz", 0).size());
//Write a RT and verify the data is removed from index
updateMV("DELETE FROM %s WHERE k = ? AND asciival = ? and bigintval = ?", 0, "foo", 0L);
Assert.assertEquals(50, execute("select asciival from mv_test3 where textval2 = ? and k = ?", "baz", 0).size());
}
@Test
public void testRangeTombstone2() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"asciival ascii, " +
"bigintval bigint, " +
"textval1 text, " +
"PRIMARY KEY((k, asciival), bigintval)" +
")");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE textval1 IS NOT NULL AND k IS NOT NULL AND asciival IS NOT NULL AND bigintval IS NOT NULL PRIMARY KEY ((textval1, k), asciival, bigintval)");
for (int i = 0; i < 100; i++)
updateMV("INSERT into %s (k,asciival,bigintval,textval1)VALUES(?,?,?,?)", 0, "foo", (long) i % 2, "bar" + i);
Assert.assertEquals(1, execute("select * from %s where k = 0 and asciival = 'foo' and bigintval = 0").size());
Assert.assertEquals(1, execute("select * from %s where k = 0 and asciival = 'foo' and bigintval = 1").size());
Assert.assertEquals(2, execute("select * from %s").size());
Assert.assertEquals(2, execute("select * from mv").size());
//Write a RT and verify the data is removed from index
updateMV("DELETE FROM %s WHERE k = ? AND asciival = ? and bigintval = ?", 0, "foo", 0L);
Assert.assertEquals(1, execute("select * from %s").size());
Assert.assertEquals(1, execute("select * from mv").size());
}
@Test
public void testCompoundPartitionKey() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"asciival ascii, " +
"bigintval bigint, " +
"PRIMARY KEY((k, asciival)))");
CFMetaData metadata = currentTableMetadata();
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
for (ColumnDefinition def : new HashSet<>(metadata.allColumns()))
{
try
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ("
+ def.name + ", k" + (def.name.toString().equals("asciival") ? "" : ", asciival") + ")";
createView("mv1_" + def.name, query);
if (def.type.isMultiCell())
Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
Assert.fail("MV creation failed on " + def);
}
try
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + " PRIMARY KEY ("
+ def.name + ", asciival" + (def.name.toString().equals("k") ? "" : ", k") + ")";
createView("mv2_" + def.name, query);
if (def.type.isMultiCell())
Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
Assert.fail("MV creation failed on " + def);
}
try
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ((" + def.name + ", k), asciival)";
createView("mv3_" + def.name, query);
if (def.type.isMultiCell())
Assert.fail("MV on a multicell should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
Assert.fail("MV creation failed on " + def);
}
try
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ((" + def.name + ", k), asciival)";
createView("mv3_" + def.name, query);
Assert.fail("Should fail on duplicate name");
}
catch (Exception e)
{
}
try
{
String query = "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL "
+ (def.name.toString().equals("asciival") ? "" : "AND asciival IS NOT NULL ") + "PRIMARY KEY ((" + def.name + ", k), nonexistentcolumn)";
createView("mv3_" + def.name, query);
Assert.fail("Should fail with unknown base column");
}
catch (InvalidQueryException e)
{
}
}
updateMV("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, fromJson(?))", 0, "ascii text", "123123123123");
updateMV("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT bigintval FROM %s WHERE k = ? and asciival = ?", 0, "ascii text"), row(123123123123L));
//Check the MV
assertRows(execute("SELECT k, bigintval from mv1_asciival WHERE asciival = ?", "ascii text"), row(0, 123123123123L));
assertRows(execute("SELECT k, bigintval from mv2_k WHERE asciival = ? and k = ?", "ascii text", 0), row(0, 123123123123L));
assertRows(execute("SELECT k from mv1_bigintval WHERE bigintval = ?", 123123123123L), row(0));
assertRows(execute("SELECT asciival from mv3_bigintval where bigintval = ? AND k = ?", 123123123123L, 0), row("ascii text"));
//UPDATE BASE
updateMV("INSERT INTO %s (k, asciival, bigintval) VALUES (?, ?, fromJson(?))", 0, "ascii text", "1");
assertRows(execute("SELECT bigintval FROM %s WHERE k = ? and asciival = ?", 0, "ascii text"), row(1L));
//Check the MV
assertRows(execute("SELECT k, bigintval from mv1_asciival WHERE asciival = ?", "ascii text"), row(0, 1L));
assertRows(execute("SELECT k, bigintval from mv2_k WHERE asciival = ? and k = ?", "ascii text", 0), row(0, 1L));
assertRows(execute("SELECT k from mv1_bigintval WHERE bigintval = ?", 123123123123L));
assertRows(execute("SELECT asciival from mv3_bigintval where bigintval = ? AND k = ?", 123123123123L, 0));
assertRows(execute("SELECT asciival from mv3_bigintval where bigintval = ? AND k = ?", 1L, 0), row("ascii text"));
//test truncate also truncates all MV
updateMV("TRUNCATE %s");
assertRows(execute("SELECT bigintval FROM %s WHERE k = ? and asciival = ?", 0, "ascii text"));
assertRows(execute("SELECT k, bigintval from mv1_asciival WHERE asciival = ?", "ascii text"));
assertRows(execute("SELECT k, bigintval from mv2_k WHERE asciival = ? and k = ?", "ascii text", 0));
assertRows(execute("SELECT asciival from mv3_bigintval where bigintval = ? AND k = ?", 1L, 0));
}
@Test
public void testCollections() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"intval int, " +
"listval list<int>, " +
"PRIMARY KEY (k))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND intval IS NOT NULL PRIMARY KEY (intval, k)");
updateMV("INSERT INTO %s (k, intval, listval) VALUES (?, ?, fromJson(?))", 0, 0, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
assertRows(execute("SELECT k, listval from mv WHERE intval = ?", 0), row(0, list(1, 2, 3)));
updateMV("INSERT INTO %s (k, intval) VALUES (?, ?)", 1, 1);
updateMV("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 1, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 1), row(1, list(1, 2, 3)));
assertRows(execute("SELECT k, listval from mv WHERE intval = ?", 1), row(1, list(1, 2, 3)));
}
@Test
public void testUpdate() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"intval int, " +
"PRIMARY KEY (k))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND intval IS NOT NULL PRIMARY KEY (intval, k)");
updateMV("INSERT INTO %s (k, intval) VALUES (?, ?)", 0, 0);
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 0));
assertRows(execute("SELECT k, intval from mv WHERE intval = ?", 0), row(0, 0));
updateMV("INSERT INTO %s (k, intval) VALUES (?, ?)", 0, 1);
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 1));
assertRows(execute("SELECT k, intval from mv WHERE intval = ?", 1), row(0, 1));
}
@Test
public void testDecimalUpdate() throws Throwable
{
createTable("CREATE TABLE %s (" +
"k int, " +
"decimalval decimal, " +
"asciival ascii, " +
"PRIMARY KEY (k))");
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE k IS NOT NULL AND decimalval IS NOT NULL PRIMARY KEY (decimalval, k)");
updateMV("INSERT INTO %s (k, asciival) VALUES (?, ?)", 0, "ascii text");
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123")));
assertRows(execute("SELECT k, asciival from mv WHERE decimalval = fromJson(?)", "123123.123123"));
assertRows(execute("SELECT k, asciival from mv WHERE decimalval = fromJson(?)", "123123"), row(0, "ascii text"));
// accept strings for numbers that cannot be represented as doubles
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"123123.123123\"");
assertRows(execute("SELECT k, decimalval, asciival FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123"), "ascii text"));
assertRows(execute("SELECT k, asciival from mv WHERE decimalval = fromJson(?)", "\"123123.123123\""), row(0, "ascii text"));
}
@Test
public void testAllTypes() throws Throwable
{
String myType = createType("CREATE TYPE %s (a int, b uuid, c set<text>)");
createTable("CREATE TABLE %s (" +
"k int PRIMARY KEY, " +
"asciival ascii, " +
"bigintval bigint, " +
"blobval blob, " +
"booleanval boolean, " +
"dateval date, " +
"decimalval decimal, " +
"doubleval double, " +
"floatval float, " +
"inetval inet, " +
"intval int, " +
"textval text, " +
"timeval time, " +
"timestampval timestamp, " +
"timeuuidval timeuuid, " +
"uuidval uuid," +
"varcharval varchar, " +
"varintval varint, " +
"listval list<int>, " +
"frozenlistval frozen<list<int>>, " +
"setval set<uuid>, " +
"frozensetval frozen<set<uuid>>, " +
"mapval map<ascii, int>," +
"frozenmapval frozen<map<ascii, int>>," +
"tupleval frozen<tuple<int, ascii, uuid>>," +
"udtval frozen<" + myType + ">)");
CFMetaData metadata = currentTableMetadata();
execute("USE " + keyspace());
executeNet(protocolVersion, "USE " + keyspace());
for (ColumnDefinition def : new HashSet<>(metadata.allColumns()))
{
try
{
createView("mv_" + def.name, "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE " + def.name + " IS NOT NULL AND k IS NOT NULL PRIMARY KEY (" + def.name + ",k)");
if (def.type.isMultiCell())
Assert.fail("MV on a multicell should fail " + def);
if (def.isPartitionKey())
Assert.fail("MV on partition key should fail " + def);
}
catch (InvalidQueryException e)
{
if (!def.type.isMultiCell() && !def.isPartitionKey())
Assert.fail("MV creation failed on " + def);
}
}
// fromJson() can only be used when the receiver type is known
assertInvalidMessage("fromJson() cannot be used in the selection clause", "SELECT fromJson(asciival) FROM %s", 0, 0);
String func1 = createFunction(KEYSPACE, "int", "CREATE FUNCTION %s (a int) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return a.toString(); $$");
createFunctionOverload(func1, "int", "CREATE FUNCTION %s (a text) CALLED ON NULL INPUT RETURNS text LANGUAGE java AS $$ return new String(a); $$");
// ================ ascii ================
updateMV("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii \\\" text\"");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0), row(0, "ascii \" text"));
// test that we can use fromJson() in other valid places in queries
assertRows(execute("SELECT asciival FROM %s WHERE k = fromJson(?)", "0"), row("ascii \" text"));
//Check the MV
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii text"));
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"), row(0, null));
updateMV("UPDATE %s SET asciival = fromJson(?) WHERE k = fromJson(?)", "\"ascii \\\" text\"", "0");
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"), row(0, null));
updateMV("DELETE FROM %s WHERE k = fromJson(?)", "0");
assertRows(execute("SELECT k, asciival FROM %s WHERE k = ?", 0));
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii \" text"));
updateMV("INSERT INTO %s (k, asciival) VALUES (?, fromJson(?))", 0, "\"ascii text\"");
assertRows(execute("SELECT k, udtval from mv_asciival WHERE asciival = ?", "ascii text"), row(0, null));
// ================ bigint ================
updateMV("INSERT INTO %s (k, bigintval) VALUES (?, fromJson(?))", 0, "123123123123");
assertRows(execute("SELECT k, bigintval FROM %s WHERE k = ?", 0), row(0, 123123123123L));
assertRows(execute("SELECT k, asciival from mv_bigintval WHERE bigintval = ?", 123123123123L), row(0, "ascii text"));
// ================ blob ================
updateMV("INSERT INTO %s (k, blobval) VALUES (?, fromJson(?))", 0, "\"0x00000001\"");
assertRows(execute("SELECT k, blobval FROM %s WHERE k = ?", 0), row(0, ByteBufferUtil.bytes(1)));
assertRows(execute("SELECT k, asciival from mv_blobval WHERE blobval = ?", ByteBufferUtil.bytes(1)), row(0, "ascii text"));
// ================ boolean ================
updateMV("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "true");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, true));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", true), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, booleanval) VALUES (?, fromJson(?))", 0, "false");
assertRows(execute("SELECT k, booleanval FROM %s WHERE k = ?", 0), row(0, false));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", true));
assertRows(execute("SELECT k, asciival from mv_booleanval WHERE booleanval = ?", false), row(0, "ascii text"));
// ================ date ================
updateMV("INSERT INTO %s (k, dateval) VALUES (?, fromJson(?))", 0, "\"1987-03-23\"");
assertRows(execute("SELECT k, dateval FROM %s WHERE k = ?", 0), row(0, SimpleDateSerializer.dateStringToDays("1987-03-23")));
assertRows(execute("SELECT k, asciival from mv_dateval WHERE dateval = fromJson(?)", "\"1987-03-23\""), row(0, "ascii text"));
// ================ decimal ================
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123.123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "123123");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123")));
assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123.123123"));
assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "123123"), row(0, "ascii text"));
// accept strings for numbers that cannot be represented as doubles
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"123123.123123\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("123123.123123")));
updateMV("INSERT INTO %s (k, decimalval) VALUES (?, fromJson(?))", 0, "\"-1.23E-12\"");
assertRows(execute("SELECT k, decimalval FROM %s WHERE k = ?", 0), row(0, new BigDecimal("-1.23E-12")));
assertRows(execute("SELECT k, asciival from mv_decimalval WHERE decimalval = fromJson(?)", "\"-1.23E-12\""), row(0, "ascii text"));
// ================ double ================
updateMV("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123.123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.123123d));
assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, doubleval) VALUES (?, fromJson(?))", 0, "123123");
assertRows(execute("SELECT k, doubleval FROM %s WHERE k = ?", 0), row(0, 123123.0d));
assertRows(execute("SELECT k, asciival from mv_doubleval WHERE doubleval = fromJson(?)", "123123"), row(0, "ascii text"));
// ================ float ================
updateMV("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123.123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.123123f));
assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = fromJson(?)", "123123.123123"), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, floatval) VALUES (?, fromJson(?))", 0, "123123");
assertRows(execute("SELECT k, floatval FROM %s WHERE k = ?", 0), row(0, 123123.0f));
assertRows(execute("SELECT k, asciival from mv_floatval WHERE floatval = fromJson(?)", "123123"), row(0, "ascii text"));
// ================ inet ================
updateMV("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"127.0.0.1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("127.0.0.1")));
assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"127.0.0.1\""), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, inetval) VALUES (?, fromJson(?))", 0, "\"::1\"");
assertRows(execute("SELECT k, inetval FROM %s WHERE k = ?", 0), row(0, InetAddress.getByName("::1")));
assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"127.0.0.1\""));
assertRows(execute("SELECT k, asciival from mv_inetval WHERE inetval = fromJson(?)", "\"::1\""), row(0, "ascii text"));
// ================ int ================
updateMV("INSERT INTO %s (k, intval) VALUES (?, fromJson(?))", 0, "123123");
assertRows(execute("SELECT k, intval FROM %s WHERE k = ?", 0), row(0, 123123));
assertRows(execute("SELECT k, asciival from mv_intval WHERE intval = fromJson(?)", "123123"), row(0, "ascii text"));
// ================ text (varchar) ================
updateMV("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"some \\\" text\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "some \" text"));
updateMV("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"\\u2013\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "\u2013"));
assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = fromJson(?)", "\"\\u2013\""), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, textval) VALUES (?, fromJson(?))", 0, "\"abcd\"");
assertRows(execute("SELECT k, textval FROM %s WHERE k = ?", 0), row(0, "abcd"));
assertRows(execute("SELECT k, asciival from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, "ascii text"));
// ================ time ================
updateMV("INSERT INTO %s (k, timeval) VALUES (?, fromJson(?))", 0, "\"07:35:07.000111222\"");
assertRows(execute("SELECT k, timeval FROM %s WHERE k = ?", 0), row(0, TimeSerializer.timeStringToLong("07:35:07.000111222")));
assertRows(execute("SELECT k, asciival from mv_timeval WHERE timeval = fromJson(?)", "\"07:35:07.000111222\""), row(0, "ascii text"));
// ================ timestamp ================
updateMV("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "123123123123");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new Date(123123123123L)));
assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = fromJson(?)", "123123123123"), row(0, "ascii text"));
updateMV("INSERT INTO %s (k, timestampval) VALUES (?, fromJson(?))", 0, "\"2014-01-01\"");
assertRows(execute("SELECT k, timestampval FROM %s WHERE k = ?", 0), row(0, new SimpleDateFormat("y-M-d").parse("2014-01-01")));
assertRows(execute("SELECT k, asciival from mv_timestampval WHERE timestampval = fromJson(?)", "\"2014-01-01\""), row(0, "ascii text"));
// ================ timeuuid ================
updateMV("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
updateMV("INSERT INTO %s (k, timeuuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, timeuuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
assertRows(execute("SELECT k, asciival from mv_timeuuidval WHERE timeuuidval = fromJson(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
// ================ uuidval ================
updateMV("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6bddc89a-5644-11e4-97fc-56847afe9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
updateMV("INSERT INTO %s (k, uuidval) VALUES (?, fromJson(?))", 0, "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\"");
assertRows(execute("SELECT k, uuidval FROM %s WHERE k = ?", 0), row(0, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")));
assertRows(execute("SELECT k, asciival from mv_uuidval WHERE uuidval = fromJson(?)", "\"6BDDC89A-5644-11E4-97FC-56847AFE9799\""), row(0, "ascii text"));
// ================ varint ================
updateMV("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "123123123123");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("123123123123")));
assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = fromJson(?)", "123123123123"), row(0, "ascii text"));
// accept strings for numbers that cannot be represented as longs
updateMV("INSERT INTO %s (k, varintval) VALUES (?, fromJson(?))", 0, "\"1234567890123456789012345678901234567890\"");
assertRows(execute("SELECT k, varintval FROM %s WHERE k = ?", 0), row(0, new BigInteger("1234567890123456789012345678901234567890")));
assertRows(execute("SELECT k, asciival from mv_varintval WHERE varintval = fromJson(?)", "\"1234567890123456789012345678901234567890\""), row(0, "ascii text"));
// ================ lists ================
updateMV("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2, 3)));
updateMV("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[1]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1)));
updateMV("UPDATE %s SET listval = listval + fromJson(?) WHERE k = ?", "[2]", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(1, 2)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2)));
updateMV("UPDATE %s SET listval = fromJson(?) + listval WHERE k = ?", "[0]", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 1, 2)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 1, 2)));
updateMV("UPDATE %s SET listval[1] = fromJson(?) WHERE k = ?", "10", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 10, 2)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 10, 2)));
updateMV("DELETE listval[1] FROM %s WHERE k = ?", 0);
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, list(0, 2)));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(0, 2)));
updateMV("INSERT INTO %s (k, listval) VALUES (?, fromJson(?))", 0, "[]");
assertRows(execute("SELECT k, listval FROM %s WHERE k = ?", 0), row(0, null));
assertRows(execute("SELECT k, listval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, null));
// frozen
updateMV("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[1, 2, 3]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list(1, 2, 3)));
assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(1, 2, 3)));
assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[1, 2, 3]"), row(0, "abcd"));
updateMV("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[3, 2, 1]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list(3, 2, 1)));
assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[1, 2, 3]"));
assertRows(execute("SELECT k, textval from mv_frozenlistval where frozenlistval = fromJson(?)", "[3, 2, 1]"), row(0, "abcd"));
assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list(3, 2, 1)));
updateMV("INSERT INTO %s (k, frozenlistval) VALUES (?, fromJson(?))", 0, "[]");
assertRows(execute("SELECT k, frozenlistval FROM %s WHERE k = ?", 0), row(0, list()));
assertRows(execute("SELECT k, frozenlistval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, list()));
// ================ sets ================
updateMV("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
// duplicates are okay, just like in CQL
updateMV("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
updateMV("UPDATE %s SET setval = setval + fromJson(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-0000-97fc-56847afe9799"), UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-0000-97fc-56847afe9799"), UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
updateMV("UPDATE %s SET setval = setval - fromJson(?) WHERE k = ?", "[\"6bddc89a-5644-0000-97fc-56847afe9799\"]", 0);
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
updateMV("INSERT INTO %s (k, setval) VALUES (?, fromJson(?))", 0, "[]");
assertRows(execute("SELECT k, setval FROM %s WHERE k = ?", 0), row(0, null));
assertRows(execute("SELECT k, setval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, null));
// frozen
updateMV("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
0, "[\"6bddc89a-5644-11e4-97fc-56847afe9798\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))))
);
assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))));
updateMV("INSERT INTO %s (k, frozensetval) VALUES (?, fromJson(?))",
0, "[\"6bddc89a-0000-11e4-97fc-56847afe9799\", \"6bddc89a-5644-11e4-97fc-56847afe9798\"]");
assertRows(execute("SELECT k, frozensetval FROM %s WHERE k = ?", 0),
row(0, set(UUID.fromString("6bddc89a-0000-11e4-97fc-56847afe9799"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798"))))
);
assertRows(execute("SELECT k, frozensetval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, set(UUID.fromString("6bddc89a-0000-11e4-97fc-56847afe9799"), (UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9798")))));
// ================ maps ================
updateMV("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""), row(0, map("a", 1, "b", 2)));
updateMV("UPDATE %s SET mapval[?] = ? WHERE k = ?", "c", 3, 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "b", 2, "c", 3))
);
assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, map("a", 1, "b", 2, "c", 3)));
updateMV("UPDATE %s SET mapval[?] = ? WHERE k = ?", "b", 10, 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "b", 10, "c", 3))
);
assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, map("a", 1, "b", 10, "c", 3)));
updateMV("DELETE mapval[?] FROM %s WHERE k = ?", "b", 0);
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0),
row(0, map("a", 1, "c", 3))
);
assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, map("a", 1, "c", 3)));
updateMV("INSERT INTO %s (k, mapval) VALUES (?, fromJson(?))", 0, "{}");
assertRows(execute("SELECT k, mapval FROM %s WHERE k = ?", 0), row(0, null));
assertRows(execute("SELECT k, mapval from mv_textval WHERE textval = fromJson(?)", "\"abcd\""),
row(0, null));
// frozen
updateMV("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": 2}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 1, "b", 2)));
assertRows(execute("SELECT k, textval FROM mv_frozenmapval WHERE frozenmapval = fromJson(?)", "{\"a\": 1, \"b\": 2}"), row(0, "abcd"));
updateMV("INSERT INTO %s (k, frozenmapval) VALUES (?, fromJson(?))", 0, "{\"b\": 2, \"a\": 3}");
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 3, "b", 2)));
assertRows(execute("SELECT k, frozenmapval FROM %s WHERE k = ?", 0), row(0, map("a", 3, "b", 2)));
// ================ tuples ================
updateMV("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, \"foobar\", \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
assertRows(execute("SELECT k, textval FROM mv_tupleval WHERE tupleval = ?", tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))),
row(0, "abcd"));
updateMV("INSERT INTO %s (k, tupleval) VALUES (?, fromJson(?))", 0, "[1, null, \"6bddc89a-5644-11e4-97fc-56847afe9799\"]");
assertRows(execute("SELECT k, tupleval FROM %s WHERE k = ?", 0),
row(0, tuple(1, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799")))
);
assertRows(execute("SELECT k, textval FROM mv_tupleval WHERE tupleval = ?", tuple(1, "foobar", UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))));
assertRows(execute("SELECT k, textval FROM mv_tupleval WHERE tupleval = ?", tuple(1, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"))),
row(0, "abcd"));
// ================ UDTs ================
updateMV("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// order of fields shouldn't matter
updateMV("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"a\": 1, \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// test nulls
updateMV("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, null, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), set("bar", "foo"))
);
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"),
row(0, "abcd"));
// test missing fields
updateMV("INSERT INTO %s (k, udtval) VALUES (?, fromJson(?))", 0, "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}");
assertRows(execute("SELECT k, udtval.a, udtval.b, udtval.c FROM %s WHERE k = ?", 0),
row(0, 1, UUID.fromString("6bddc89a-5644-11e4-97fc-56847afe9799"), null)
);
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": null, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\", \"c\": [\"foo\", \"bar\"]}"));
assertRows(execute("SELECT k, textval FROM mv_udtval WHERE udtval = fromJson(?)", "{\"a\": 1, \"b\": \"6bddc89a-5644-11e4-97fc-56847afe9799\"}"),
row(0, "abcd"));
}
@Test
public void ttlTest() throws Throwable
{
createTable("CREATE TABLE %s (" +
"a int," +
"b int," +
"c int," +
"d int," +
"PRIMARY KEY (a, b))");
executeNet(protocolVersion, "USE " + keyspace());
createView("mv", "CREATE MATERIALIZED VIEW %s AS SELECT * FROM %%s WHERE c IS NOT NULL AND a IS NOT NULL AND b IS NOT NULL PRIMARY KEY (c, a, b)");
updateMV("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?) USING TTL 5", 1, 1, 1, 1);
Thread.sleep(TimeUnit.SECONDS.toMillis(3));
updateMV("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", 1, 1, 2);
Thread.sleep(TimeUnit.SECONDS.toMillis(3));
List<Row> results = executeNet(protocolVersion, "SELECT d FROM mv WHERE c = 2 AND a = 1 AND b = 1").all();
Assert.assertEquals(1, results.size());
Assert.assertTrue("There should be a null result given back due to ttl expiry", results.get(0).isNull(0));
}
}

1
test/unit/org/apache/cassandra/db/RangeTombstoneTest.java
View File

@ -33,6 +33,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.AbstractReadCommandBuilder;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.*;
import org.apache.cassandra.Util;

115
test/unit/org/apache/cassandra/db/view/MaterializedViewUtilsTest.java
View File

@ -1,115 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.view;
import java.net.InetAddress;
import java.util.HashMap;
import java.util.Map;
import org.junit.BeforeClass;
import org.junit.Test;
import junit.framework.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.OrderPreservingPartitioner.StringToken;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.IEndpointSnitch;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.PropertyFileSnitch;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
public class MaterializedViewUtilsTest
{
@BeforeClass
public static void setUp() throws ConfigurationException
{
IEndpointSnitch snitch = new PropertyFileSnitch();
DatabaseDescriptor.setEndpointSnitch(snitch);
Keyspace.setInitialized();
}
@Test
public void testGetIndexNaturalEndpoint() throws Exception
{
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
// DC2
metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
Map<String, String> replicationMap = new HashMap<>();
replicationMap.put(KeyspaceParams.Replication.CLASS, NetworkTopologyStrategy.class.getName());
replicationMap.put("DC1", "1");
replicationMap.put("DC2", "1");
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap));
Schema.instance.setKeyspaceMetadata(meta);
InetAddress naturalEndpoint = MaterializedViewUtils.getViewNaturalEndpoint("Keyspace1",
new StringToken("CA"),
new StringToken("BB"));
Assert.assertEquals(InetAddress.getByName("127.0.0.2"), naturalEndpoint);
}
@Test
public void testLocalHostPreference() throws Exception
{
TokenMetadata metadata = StorageService.instance.getTokenMetadata();
metadata.clearUnsafe();
// DC1
metadata.updateNormalToken(new StringToken("A"), InetAddress.getByName("127.0.0.1"));
metadata.updateNormalToken(new StringToken("C"), InetAddress.getByName("127.0.0.2"));
// DC2
metadata.updateNormalToken(new StringToken("B"), InetAddress.getByName("127.0.0.4"));
metadata.updateNormalToken(new StringToken("D"), InetAddress.getByName("127.0.0.5"));
Map<String, String> replicationMap = new HashMap<>();
replicationMap.put(KeyspaceParams.Replication.CLASS, NetworkTopologyStrategy.class.getName());
replicationMap.put("DC1", "2");
replicationMap.put("DC2", "2");
Keyspace.clear("Keyspace1");
KeyspaceMetadata meta = KeyspaceMetadata.create("Keyspace1", KeyspaceParams.create(false, replicationMap));
Schema.instance.setKeyspaceMetadata(meta);
InetAddress naturalEndpoint = MaterializedViewUtils.getViewNaturalEndpoint("Keyspace1",
new StringToken("CA"),
new StringToken("BB"));
Assert.assertEquals(InetAddress.getByName("127.0.0.1"), naturalEndpoint);
}
}