natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow concurrent schema migrations 

patch by Pavel Yaskevich; reviewed by Jonathan Ellis for CASSANDRA-1391
This commit is contained in:
Pavel Yaskevich 2012-01-22 02:46:02 +02:00
parent e594e0dca8
commit 37b079352d
39 changed files with 2101 additions and 1616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -50,6 +50,7 @@
* Allow rangeSlice queries to be start/end inclusive/exclusive (CASSANDRA-3749)
* Fix BulkLoader to support new SSTable layout and add stream
throttling to prevent an NPE when there is no yaml config (CASSANDRA-3752)
* Allow concurrent schema migrations (CASSANDRA-1391)
1.0.8

343
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -18,30 +18,38 @@
package org.apache.cassandra.config;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.*;
import com.google.common.base.Objects;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.db.migration.avro.ColumnDef;
import org.apache.cassandra.io.IColumnSerializer;
import org.apache.cassandra.io.compress.CompressionParameters;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnDef;
import org.apache.cassandra.thrift.IndexType;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.cassandra.db.migration.MigrationHelper.*;
public final class CFMetaData
{
//
@ -63,11 +71,51 @@ public final class CFMetaData
public static final CFMetaData StatusCf = newSystemMetadata(SystemTable.STATUS_CF, 0, "persistent metadata for the local node", BytesType.instance, null);
public static final CFMetaData HintsCf = newSystemMetadata(HintedHandOffManager.HINTS_CF, 1, "hinted handoff data", BytesType.instance, BytesType.instance);
@Deprecated
public static final CFMetaData MigrationsCf = newSystemMetadata(Migration.MIGRATIONS_CF, 2, "individual schema mutations", TimeUUIDType.instance, null);
@Deprecated
public static final CFMetaData SchemaCf = newSystemMetadata(Migration.SCHEMA_CF, 3, "current state of the schema", UTF8Type.instance, null);
public static final CFMetaData IndexCf = newSystemMetadata(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null);
public static final CFMetaData NodeIdCf = newSystemMetadata(SystemTable.NODE_ID_CF, 6, "nodeId and their metadata", TimeUUIDType.instance, null);
public static final CFMetaData VersionCf = newSystemMetadata(SystemTable.VERSION_CF, 7, "server version information", UTF8Type.instance, null);
public static final CFMetaData SchemaKeyspacesCf = schemaCFDefinition(SystemTable.SCHEMA_KEYSPACES_CF, 8, "keyspace attributes of the schema", AsciiType.instance, 1);
public static final CFMetaData SchemaColumnFamiliesCf = schemaCFDefinition(SystemTable.SCHEMA_COLUMNFAMILIES_CF, 9, "ColumnFamily attributes of the schema", AsciiType.instance, 2);
public static final CFMetaData SchemaColumnsCf = schemaCFDefinition(SystemTable.SCHEMA_COLUMNS_CF, 10, "ColumnFamily column attributes of the schema", AsciiType.instance, 3);
private static CFMetaData schemaCFDefinition(String name, int index, String comment, AbstractType<?> comp, int nestingLevel)
{
try
{
AbstractType<?> comparator;
if (nestingLevel == 1)
{
comparator = comp;
}
else
{
List<AbstractType<?>> composite = new ArrayList<AbstractType<?>>(nestingLevel);
for (int i = 0; i < nestingLevel; i++)
composite.add(comp);
comparator = CompositeType.getInstance(composite);
}
return newSystemMetadata(name,
index,
comment,
comparator,
null)
.keyValidator(AsciiType.instance)
.defaultValidator(UTF8Type.instance);
}
catch (ConfigurationException e)
{
throw new RuntimeException(e);
}
}
static
{
try
@ -277,47 +325,7 @@ public final class CFMetaData
return cfName + Directories.SECONDARY_INDEX_NAME_SEPARATOR + (info.getIndexName() == null ? ByteBufferUtil.bytesToHex(info.name) : info.getIndexName());
}
// converts CFM to avro CfDef
public org.apache.cassandra.db.migration.avro.CfDef toAvro()
{
org.apache.cassandra.db.migration.avro.CfDef cf = new org.apache.cassandra.db.migration.avro.CfDef();
cf.id = cfId;
cf.keyspace = new Utf8(ksName);
cf.name = new Utf8(cfName);
cf.column_type = new Utf8(cfType.name());
cf.comparator_type = new Utf8(comparator.toString());
if (subcolumnComparator != null)
{
assert cfType == ColumnFamilyType.Super
: String.format("%s CF %s should not have subcomparator %s defined", cfType, cfName, subcolumnComparator);
cf.subcomparator_type = new Utf8(subcolumnComparator.toString());
}
cf.comment = new Utf8(enforceCommentNotNull(comment));
cf.read_repair_chance = readRepairChance;
cf.replicate_on_write = replicateOnWrite;
cf.gc_grace_seconds = gcGraceSeconds;
cf.default_validation_class = defaultValidator == null ? null : new Utf8(defaultValidator.toString());
cf.key_validation_class = new Utf8(keyValidator.toString());
cf.min_compaction_threshold = minCompactionThreshold;
cf.max_compaction_threshold = maxCompactionThreshold;
cf.merge_shards_chance = mergeShardsChance;
cf.key_alias = keyAlias;
cf.column_metadata = new ArrayList<ColumnDef>(column_metadata.size());
for (ColumnDefinition cd : column_metadata.values())
cf.column_metadata.add(cd.toAvro());
cf.compaction_strategy = new Utf8(compactionStrategyClass.getName());
if (compactionStrategyOptions != null)
{
cf.compaction_strategy_options = new HashMap<CharSequence, CharSequence>();
for (Map.Entry<String, String> e : compactionStrategyOptions.entrySet())
cf.compaction_strategy_options.put(new Utf8(e.getKey()), new Utf8(e.getValue()));
}
cf.compression_options = compressionParameters.asAvroOptions();
cf.bloom_filter_fp_chance = bloomFilterFpChance;
cf.caching = new Utf8(caching.toString());
return cf;
}
@Deprecated
public static CFMetaData fromAvro(org.apache.cassandra.db.migration.avro.CfDef cf)
{
AbstractType<?> comparator;
@ -338,7 +346,7 @@ public final class CFMetaData
throw new RuntimeException("Could not inflate CFMetaData for " + cf, ex);
}
Map<ByteBuffer, ColumnDefinition> column_metadata = new TreeMap<ByteBuffer, ColumnDefinition>(BytesType.instance);
for (ColumnDef aColumn_metadata : cf.column_metadata)
for (org.apache.cassandra.db.migration.avro.ColumnDef aColumn_metadata : cf.column_metadata)
{
ColumnDefinition cd = ColumnDefinition.fromAvro(aColumn_metadata);
if (cd.getIndexType() != null && cd.getIndexName() == null)
@ -627,22 +635,45 @@ public final class CFMetaData
.validate();
}
/** updates CFMetaData in-place to match cf_def */
public void apply(org.apache.cassandra.db.migration.avro.CfDef cf_def) throws ConfigurationException
public void reload() throws IOException
{
Row cfDefRow = SystemTable.readSchemaRow(ksName, cfName);
if (cfDefRow.cf == null || cfDefRow.cf.isEmpty())
throw new IOException(String.format("%s not found in the schema definitions table.", ksName + ":" + cfName));
try
{
apply(fromSchema(cfDefRow.cf));
}
catch (ConfigurationException e)
{
throw new IOException(e);
}
}
/**
* Updates CFMetaData in-place to match cf_def
*
* *Note*: This method left public only for DefsTest, don't use directly!
*
* @throws ConfigurationException if ks/cf names or cf ids didn't match
*/
public void apply(CfDef cf_def) throws ConfigurationException
{
logger.debug("applying {} to {}", cf_def, this);
// validate
if (!cf_def.keyspace.toString().equals(ksName))
if (!cf_def.keyspace.equals(ksName))
throw new ConfigurationException(String.format("Keyspace mismatch (found %s; expected %s)",
cf_def.keyspace, ksName));
if (!cf_def.name.toString().equals(cfName))
if (!cf_def.name.equals(cfName))
throw new ConfigurationException(String.format("Column family mismatch (found %s; expected %s)",
cf_def.name, cfName));
if (!cf_def.id.equals(cfId))
if (cf_def.id != cfId)
throw new ConfigurationException(String.format("Column family ID mismatch (found %s; expected %s)",
cf_def.id, cfId));
if (!cf_def.column_type.toString().equals(cfType.name()))
if (!cf_def.column_type.equals(cfType.name()))
throw new ConfigurationException("types do not match.");
if (comparator != TypeParser.parse(cf_def.comparator_type))
throw new ConfigurationException("comparators do not match.");
@ -667,15 +698,18 @@ public final class CFMetaData
maxCompactionThreshold = cf_def.max_compaction_threshold;
mergeShardsChance = cf_def.merge_shards_chance;
keyAlias = cf_def.key_alias;
if (cf_def.bloom_filter_fp_chance != null)
if (cf_def.isSetBloom_filter_fp_chance())
bloomFilterFpChance = cf_def.bloom_filter_fp_chance;
caching = Caching.fromString(cf_def.caching.toString());
caching = Caching.fromString(cf_def.caching);
if (!cf_def.isSetColumn_metadata())
cf_def.setColumn_metadata(new ArrayList<ColumnDef>());
// adjust column definitions. figure out who is coming and going.
Set<ByteBuffer> toRemove = new HashSet<ByteBuffer>();
Set<ByteBuffer> newColumns = new HashSet<ByteBuffer>();
Set<org.apache.cassandra.db.migration.avro.ColumnDef> toAdd = new HashSet<org.apache.cassandra.db.migration.avro.ColumnDef>();
for (org.apache.cassandra.db.migration.avro.ColumnDef def : cf_def.column_metadata)
Set<ColumnDef> toAdd = new HashSet<ColumnDef>();
for (ColumnDef def : cf_def.column_metadata)
{
newColumns.add(def.name);
if (!column_metadata.containsKey(def.name))
@ -691,36 +725,36 @@ public final class CFMetaData
column_metadata.remove(indexName);
}
// update the ones staying
for (org.apache.cassandra.db.migration.avro.ColumnDef def : cf_def.column_metadata)
for (ColumnDef def : cf_def.column_metadata)
{
ColumnDefinition oldDef = column_metadata.get(def.name);
if (oldDef == null)
continue;
oldDef.setValidator(TypeParser.parse(def.validation_class));
oldDef.setIndexType(def.index_type == null ? null : org.apache.cassandra.thrift.IndexType.valueOf(def.index_type.name()),
ColumnDefinition.getStringMap(def.index_options));
oldDef.setIndexName(def.index_name == null ? null : def.index_name.toString());
oldDef.setIndexType(def.index_type == null ? null : IndexType.valueOf(def.index_type.name()),
def.index_options);
oldDef.setIndexName(def.index_name == null ? null : def.index_name);
}
// add the new ones coming in.
for (org.apache.cassandra.db.migration.avro.ColumnDef def : toAdd)
for (ColumnDef def : toAdd)
{
AbstractType<?> dValidClass = TypeParser.parse(def.validation_class);
ColumnDefinition cd = new ColumnDefinition(def.name,
dValidClass,
def.index_type == null ? null : org.apache.cassandra.thrift.IndexType.valueOf(def.index_type.toString()),
ColumnDefinition.getStringMap(def.index_options),
def.index_name == null ? null : def.index_name.toString());
def.index_type == null ? null : IndexType.valueOf(def.index_type.name()),
def.index_options,
def.index_name == null ? null : def.index_name);
column_metadata.put(cd.name, cd);
}
if (cf_def.compaction_strategy != null)
compactionStrategyClass = createCompactionStrategy(cf_def.compaction_strategy.toString());
compactionStrategyClass = createCompactionStrategy(cf_def.compaction_strategy);
if (null != cf_def.compaction_strategy_options)
{
compactionStrategyOptions = new HashMap<String, String>();
for (Map.Entry<CharSequence, CharSequence> e : cf_def.compaction_strategy_options.entrySet())
compactionStrategyOptions.put(e.getKey().toString(), e.getValue().toString());
for (Map.Entry<String, String> e : cf_def.compaction_strategy_options.entrySet())
compactionStrategyOptions.put(e.getKey(), e.getValue());
}
compressionParameters = CompressionParameters.create(cf_def.compression_options);
@ -749,9 +783,7 @@ public final class CFMetaData
ColumnFamilyStore.class,
Map.class // options
});
return (AbstractCompactionStrategy)constructor.newInstance(new Object[] {
cfs,
compactionStrategyOptions});
return (AbstractCompactionStrategy)constructor.newInstance(cfs, compactionStrategyOptions);
}
catch (NoSuchMethodException e)
{
@ -788,7 +820,7 @@ public final class CFMetaData
def.setRead_repair_chance(readRepairChance);
def.setReplicate_on_write(replicateOnWrite);
def.setGc_grace_seconds(gcGraceSeconds);
def.setDefault_validation_class(defaultValidator.toString());
def.setDefault_validation_class(defaultValidator == null ? null : defaultValidator.toString());
def.setKey_validation_class(keyValidator.toString());
def.setMin_compaction_threshold(minCompactionThreshold);
def.setMax_compaction_threshold(maxCompactionThreshold);
@ -815,9 +847,9 @@ public final class CFMetaData
return def;
}
public static void validateMinMaxCompactionThresholds(org.apache.cassandra.db.migration.avro.CfDef cf_def) throws ConfigurationException
public static void validateMinMaxCompactionThresholds(CfDef cf_def) throws ConfigurationException
{
if (cf_def.min_compaction_threshold != null && cf_def.max_compaction_threshold != null)
if (cf_def.isSetMin_compaction_threshold() && cf_def.isSetMax_compaction_threshold())
{
if ((cf_def.min_compaction_threshold > cf_def.max_compaction_threshold) &&
cf_def.max_compaction_threshold != 0)
@ -825,15 +857,15 @@ public final class CFMetaData
throw new ConfigurationException("min_compaction_threshold cannot be greater than max_compaction_threshold");
}
}
else if (cf_def.min_compaction_threshold != null)
else if (cf_def.isSetMin_compaction_threshold())
{
if (cf_def.min_compaction_threshold > DEFAULT_MAX_COMPACTION_THRESHOLD)
{
throw new ConfigurationException("min_compaction_threshold cannot be greather than max_compaction_threshold (default " +
throw new ConfigurationException("min_compaction_threshold cannot be greater than max_compaction_threshold (default " +
DEFAULT_MAX_COMPACTION_THRESHOLD + ")");
}
}
else if (cf_def.max_compaction_threshold != null)
else if (cf_def.isSetMax_compaction_threshold())
{
if (cf_def.max_compaction_threshold < DEFAULT_MIN_COMPACTION_THRESHOLD && cf_def.max_compaction_threshold != 0) {
throw new ConfigurationException("max_compaction_threshold cannot be less than min_compaction_threshold");
@ -923,6 +955,167 @@ public final class CFMetaData
return this;
}
/**
* Calculate the difference between current metadata and given and serialize it as schema RowMutation
*
* @param newState The new metadata (for the same CF)
* @param modificationTimestamp Timestamp to use for mutation
*
* @return Difference between attributes in form of schema mutation
*
* @throws ConfigurationException if any of the attributes didn't pass validation
*/
public RowMutation diff(CfDef newState, long modificationTimestamp) throws ConfigurationException
{
CfDef curState = toThrift();
RowMutation m = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(ksName));
for (CfDef._Fields field : CfDef._Fields.values())
{
if (field.equals(CfDef._Fields.COLUMN_METADATA))
continue; // deal with columns after main attributes
Object curValue = curState.getFieldValue(field);
Object newValue = newState.getFieldValue(field);
if (Objects.equal(curValue, newValue))
continue;
m.add(new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF, null, compositeNameFor(curState.name, field.getFieldName())),
valueAsBytes(newValue),
modificationTimestamp);
}
AbstractType nameComparator = cfType.equals(ColumnFamilyType.Super)
? subcolumnComparator
: comparator;
MapDifference<ByteBuffer, ColumnDefinition> columnDiff = Maps.difference(column_metadata, ColumnDefinition.fromThrift(newState.column_metadata));
Map<ByteBuffer, ColumnDef> columnDefMap = ColumnDefinition.toMap(newState.column_metadata);
// columns that are no longer needed
for (ByteBuffer name : columnDiff.entriesOnlyOnLeft().keySet())
ColumnDefinition.deleteFromSchema(m, curState.name, nameComparator, name, modificationTimestamp);
// newly added columns
for (ByteBuffer name : columnDiff.entriesOnlyOnRight().keySet())
ColumnDefinition.addToSchema(m, curState.name, nameComparator, columnDefMap.get(name), modificationTimestamp);
// old columns with updated attributes
for (ByteBuffer name : columnDiff.entriesDiffering().keySet())
ColumnDefinition.addToSchema(m, curState.name, nameComparator, columnDefMap.get(name), modificationTimestamp);
return m;
}
/**
* Remove all CF attributes from schema
*
* @param timestamp Timestamp to use
*
* @return RowMutation to use to completely remove cf from schema
*/
public RowMutation dropFromSchema(long timestamp)
{
RowMutation m = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(ksName));
for (CfDef._Fields field : CfDef._Fields.values())
m.delete(new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF, null, compositeNameFor(cfName, field.getFieldName())), timestamp);
for (ColumnDefinition columnDefinition : column_metadata.values())
ColumnDefinition.deleteFromSchema(m, cfName, comparator, columnDefinition.name, timestamp);
return m;
}
/**
* Convert current metadata into schema mutation
*
* @param timestamp Timestamp to use
*
* @return Low-level representation of the CF
*
* @throws ConfigurationException if any of the attributes didn't pass validation
*/
public RowMutation toSchema(long timestamp) throws ConfigurationException
{
RowMutation mutation = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(ksName));
toSchema(mutation, toThrift(), timestamp);
return mutation;
}
/**
* Convert given Thrift-serialized metadata into schema mutation
*
* @param mutation The mutation to include ColumnFamily attributes into (can contain keyspace attributes already)
* @param cfDef Thrift-serialized metadata to use as source for schema mutation
* @param timestamp Timestamp to use
*
* @throws ConfigurationException if any of the attributes didn't pass validation
*/
public static void toSchema(RowMutation mutation, CfDef cfDef, long timestamp) throws ConfigurationException
{
applyImplicitDefaults(cfDef);
for (CfDef._Fields field : CfDef._Fields.values())
{
if (field.equals(CfDef._Fields.COLUMN_METADATA))
continue;
mutation.add(new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF, null, compositeNameFor(cfDef.name, field.getFieldName())),
valueAsBytes(cfDef.getFieldValue(field)),
timestamp);
}
if (!cfDef.isSetColumn_metadata())
return;
AbstractType comparator = TypeParser.parse(cfDef.column_type.equals("Super")
? cfDef.subcomparator_type
: cfDef.comparator_type);
for (ColumnDef columnDef : cfDef.column_metadata)
ColumnDefinition.addToSchema(mutation, cfDef.name, comparator, columnDef, timestamp);
}
/**
* Deserialize CF metadata from low-level representation
*
* @param serializedCfDef The data to use for deserialization
*
* @return Thrift-based metadata deserialized from schema
*
* @throws IOException on any I/O related error
*/
public static CfDef fromSchema(ColumnFamily serializedCfDef) throws IOException
{
assert serializedCfDef != null;
CfDef cfDef = new CfDef();
AbstractType sysComparator = serializedCfDef.getComparator();
for (IColumn cfAttr : serializedCfDef.getSortedColumns())
{
if (cfAttr == null || cfAttr.isMarkedForDelete())
continue;
// column name format is <cf>:<attribute name>
String[] attr = sysComparator.getString(cfAttr.name()).split(":");
assert attr.length == 2;
CfDef._Fields field = CfDef._Fields.findByName(attr[1]);
cfDef.setFieldValue(field, deserializeValue(cfAttr.value(), getValueClass(CfDef.class, field.getFieldName())));
}
for (ColumnDef columnDef : ColumnDefinition.fromSchema(cfDef.keyspace, cfDef.name))
cfDef.addToColumn_metadata(columnDef);
return cfDef;
}
@Override
public String toString()
{
@ -951,4 +1144,4 @@ public final class CFMetaData
.append("caching", caching)
.toString();
}
}
}

180
src/java/org/apache/cassandra/config/ColumnDefinition.java
View File

@ -24,16 +24,22 @@ package org.apache.cassandra.config;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.avro.util.Utf8;
import com.google.common.collect.Maps;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.db.migration.MigrationHelper;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.ColumnDef;
import org.apache.cassandra.thrift.IndexType;
import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.db.migration.MigrationHelper.*;
public class ColumnDefinition
{
public final ByteBuffer name;
private AbstractType<?> validator;
private IndexType index_type;
@ -80,19 +86,7 @@ public class ColumnDefinition
return result;
}
public org.apache.cassandra.db.migration.avro.ColumnDef toAvro()
{
org.apache.cassandra.db.migration.avro.ColumnDef cd = new org.apache.cassandra.db.migration.avro.ColumnDef();
cd.name = ByteBufferUtil.clone(name);
cd.validation_class = new Utf8(validator.toString());
cd.index_type = index_type == null
? null
: org.apache.cassandra.db.migration.avro.IndexType.valueOf(index_type.name());
cd.index_name = index_name == null ? null : new Utf8(index_name);
cd.index_options = getCharSequenceMap(index_options);
return cd;
}
@Deprecated
public static ColumnDefinition fromAvro(org.apache.cassandra.db.migration.avro.ColumnDef cd)
{
IndexType index_type = cd.index_type == null ? null : Enum.valueOf(IndexType.class, cd.index_type.name());
@ -108,6 +102,22 @@ public class ColumnDefinition
}
}
public ColumnDef toThrift()
{
ColumnDef cd = new ColumnDef();
cd.setName(ByteBufferUtil.clone(name));
cd.setValidation_class(validator.toString());
cd.setIndex_type(index_type == null
? null
: IndexType.valueOf(index_type.name()));
cd.setIndex_name(index_name == null ? null : index_name);
cd.setIndex_options(index_options == null ? null : Maps.newHashMap(index_options));
return cd;
}
public static ColumnDefinition fromThrift(ColumnDef thriftColumnDef) throws ConfigurationException
{
return new ColumnDefinition(ByteBufferUtil.clone(thriftColumnDef.name),
@ -129,6 +139,133 @@ public class ColumnDefinition
return cds;
}
public static Map<ByteBuffer, ColumnDef> toMap(List<ColumnDef> columnDefs)
{
Map<ByteBuffer, ColumnDef> map = new HashMap<ByteBuffer, ColumnDef>();
if (columnDefs == null)
return map;
for (ColumnDef columnDef : columnDefs)
map.put(columnDef.name, columnDef);
return map;
}
/**
* Drop specified column from the schema using given row mutation.
*
* @param mutation The schema row mutation
* @param cfName The name of the parent ColumnFamily
* @param comparator The comparator to serialize column name in human-readable format
* @param columnName The column name as String
* @param timestamp The timestamp to use for column modification
*/
public static void deleteFromSchema(RowMutation mutation, String cfName, AbstractType comparator, ByteBuffer columnName, long timestamp)
{
toSchema(mutation, comparator, cfName, columnName, null, timestamp, true);
}
/**
* Add new/update column to/in the schema.
*
* @param mutation The schema row mutation
* @param cfName The name of the parent ColumnFamily
* @param comparator The comparator to serialize column name in human-readable format
* @param columnDef The Thrift-based column definition that contains all attributes
* @param timestamp The timestamp to use for column modification
*/
public static void addToSchema(RowMutation mutation, String cfName, AbstractType comparator, ColumnDef columnDef, long timestamp)
{
toSchema(mutation, comparator, cfName, columnDef.name, columnDef, timestamp, false);
}
/**
* Serialize given ColumnDef into given schema row mutation to add or drop it.
*
* @param mutation The mutation to use for serialization
* @param comparator The comparator to serialize column name in human-readable format
* @param cfName The name of the parent ColumnFamily
* @param columnName The column name as String
* @param columnDef The Thrift-based column definition that contains all attributes
* @param timestamp The timestamp to use for column modification
* @param delete The flag which indicates if column should be deleted or added to the schema
*/
private static void toSchema(RowMutation mutation, AbstractType comparator, String cfName, ByteBuffer columnName, ColumnDef columnDef, long timestamp, boolean delete)
{
for (ColumnDef._Fields field : ColumnDef._Fields.values())
{
QueryPath path = new QueryPath(SystemTable.SCHEMA_COLUMNS_CF,
null,
compositeNameFor(cfName,
readableColumnName(columnName, comparator),
field.getFieldName()));
if (delete)
mutation.delete(path, timestamp);
else
mutation.add(path, valueAsBytes(columnDef.getFieldValue(field)), timestamp);
}
}
/**
* Deserialize columns from low-level representation
*
* @param ksName The corresponding Keyspace
* @param cfName The name of the parent ColumnFamily
*
* @return Thrift-based deserialized representation of the column
*/
public static List<ColumnDef> fromSchema(String ksName, String cfName)
{
DecoratedKey key = StorageService.getPartitioner().decorateKey(SystemTable.getSchemaKSKey(ksName));
ColumnFamilyStore columnsStore = SystemTable.schemaCFS(SystemTable.SCHEMA_COLUMNS_CF);
ColumnFamily columns = columnsStore.getColumnFamily(key,
new QueryPath(SystemTable.SCHEMA_COLUMNS_CF),
MigrationHelper.searchComposite(cfName, true),
MigrationHelper.searchComposite(cfName, false),
false,
Integer.MAX_VALUE);
if (columns == null || columns.isEmpty())
return Collections.emptyList();
// contenders to be a valid columns, re-check is done after all attributes
// were read from serialized state, if ColumnDef has all required fields it gets promoted to be returned
Map<String, ColumnDef> contenders = new HashMap<String, ColumnDef>();
for (IColumn column : columns.getSortedColumns())
{
if (column.isMarkedForDelete())
continue;
// column name format <cf>:<column name>:<attribute name>
String[] components = columns.getComparator().getString(column.name()).split(":");
assert components.length == 3;
ColumnDef columnDef = contenders.get(components[1]);
if (columnDef == null)
{
columnDef = new ColumnDef();
contenders.put(components[1], columnDef);
}
ColumnDef._Fields field = ColumnDef._Fields.findByName(components[2]);
columnDef.setFieldValue(field, deserializeValue(column.value(), getValueClass(ColumnDef.class, field.getFieldName())));
}
List<ColumnDef> columnDefs = new ArrayList<ColumnDef>();
for (ColumnDef columnDef : contenders.values())
{
if (columnDef.isSetName() && columnDef.isSetValidation_class())
columnDefs.add(columnDef);
}
return columnDefs;
}
@Override
public String toString()
{
@ -189,17 +326,4 @@ public class ColumnDefinition
return stringMap;
}
private static Map<CharSequence, CharSequence> getCharSequenceMap(Map<String,String> stringMap)
{
if (stringMap == null)
return null;
Map<CharSequence, CharSequence> charMap = new HashMap<CharSequence, CharSequence>();
for (Map.Entry<String, String> entry : stringMap.entrySet())
charMap.put(new Utf8(entry.getKey()), new Utf8(entry.getValue()));
return charMap;
}
}

96
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -36,7 +36,9 @@ import org.apache.cassandra.auth.IAuthenticator;
import org.apache.cassandra.auth.IAuthority;
import org.apache.cassandra.cache.IRowCacheProvider;
import org.apache.cassandra.config.Config.RequestSchedulerId;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DefsTable;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.FileUtils;
@ -425,6 +427,9 @@ public class DatabaseDescriptor
Schema.instance.load(CFMetaData.IndexCf);
Schema.instance.load(CFMetaData.NodeIdCf);
Schema.instance.load(CFMetaData.VersionCf);
Schema.instance.load(CFMetaData.SchemaKeyspacesCf);
Schema.instance.load(CFMetaData.SchemaColumnFamiliesCf);
Schema.instance.load(CFMetaData.SchemaColumnsCf);
Schema.instance.addSystemTable(systemMeta);
@ -471,61 +476,72 @@ public class DatabaseDescriptor
/** load keyspace (table) definitions, but do not initialize the table instances. */
public static void loadSchemas() throws IOException
{
// we can load tables from local storage if a version is set in the system table and that acutally maps to
// real data in the definitions table. If we do end up loading from xml, store the defintions so that we
// don't load from xml anymore.
UUID uuid = Migration.getLastMigrationId();
if (uuid == null)
ColumnFamilyStore schemaCFS = SystemTable.schemaCFS(SystemTable.SCHEMA_KEYSPACES_CF);
// if table with definitions is empty try loading the old way
if (schemaCFS.estimateKeys() == 0)
{
logger.info("Couldn't detect any schema definitions in local storage.");
// peek around the data directories to see if anything is there.
boolean hasExistingTables = false;
for (String dataDir : getAllDataFileLocations())
// we can load tables from local storage if a version is set in the system table and that actually maps to
// real data in the definitions table. If we do end up loading from xml, store the definitions so that we
// don't load from xml anymore.
UUID uuid = Migration.getLastMigrationId();
if (uuid == null)
{
File dataPath = new File(dataDir);
if (dataPath.exists() && dataPath.isDirectory())
logger.info("Couldn't detect any schema definitions in local storage.");
// peek around the data directories to see if anything is there.
if (hasExistingNoSystemTables())
logger.info("Found table data in data directories. Consider using the CLI to define your schema.");
else
logger.info("To create keyspaces and column families, see 'help create keyspace' in the CLI, or set up a schema using the thrift system_* calls.");
}
else
{
logger.info("Loading schema version " + uuid.toString());
Collection<KSMetaData> tableDefs = DefsTable.loadFromStorage(uuid);
// happens when someone manually deletes all tables and restarts.
if (tableDefs.size() == 0)
{
// see if there are other directories present.
int dirCount = dataPath.listFiles(new FileFilter()
{
public boolean accept(File pathname)
{
return pathname.isDirectory();
}
}).length;
if (dirCount > 0)
hasExistingTables = true;
logger.warn("No schema definitions were found in local storage.");
}
if (hasExistingTables)
else // if non-system tables where found, trying to load them
{
break;
Schema.instance.load(tableDefs);
}
}
if (hasExistingTables)
logger.info("Found table data in data directories. Consider using the CLI to define your schema.");
else
logger.info("To create keyspaces and column families, see 'help create keyspace' in the CLI, or set up a schema using the thrift system_* calls.");
}
else
{
logger.info("Loading schema version " + uuid.toString());
Collection<KSMetaData> tableDefs = DefsTable.loadFromStorage(uuid);
Schema.instance.load(DefsTable.loadFromTable());
}
// happens when someone manually deletes all tables and restarts.
if (tableDefs.size() == 0)
Schema.instance.updateVersion();
Schema.instance.fixCFMaxId();
}
private static boolean hasExistingNoSystemTables()
{
for (String dataDir : getAllDataFileLocations())
{
File dataPath = new File(dataDir);
if (dataPath.exists() && dataPath.isDirectory())
{
logger.warn("No schema definitions were found in local storage.");
// set version so that migrations leading up to emptiness aren't replayed.
Schema.instance.setVersion(uuid);
}
else // if non-system tables where found, trying to load them
{
Schema.instance.load(tableDefs, uuid);
// see if there are other directories present.
int dirCount = dataPath.listFiles(new FileFilter()
{
public boolean accept(File pathname)
{
return pathname.isDirectory();
}
}).length;
if (dirCount > 0)
return true;
}
}
Schema.instance.fixCFMaxId();
return false;
}
public static IAuthenticator getAuthenticator()

234
src/java/org/apache/cassandra/config/KSMetaData.java
View File

@ -18,19 +18,25 @@
package org.apache.cassandra.config;
import java.io.IOException;
import java.util.*;
import com.google.common.base.Objects;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.LocalStrategy;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.KsDef;
import org.apache.cassandra.thrift.ColumnDef;
import static org.apache.cassandra.db.migration.MigrationHelper.*;
public final class KSMetaData
{
@ -65,7 +71,10 @@ public final class KSMetaData
CFMetaData.SchemaCf,
CFMetaData.IndexCf,
CFMetaData.NodeIdCf,
CFMetaData.VersionCf);
CFMetaData.VersionCf,
CFMetaData.SchemaKeyspacesCf,
CFMetaData.SchemaColumnFamiliesCf,
CFMetaData.SchemaColumnsCf);
return new KSMetaData(Table.SYSTEM_TABLE, LocalStrategy.class, optsWithRF(1), true, cfDefs);
}
@ -117,28 +126,6 @@ public final class KSMetaData
{
return cfMetaData;
}
public org.apache.cassandra.db.migration.avro.KsDef toAvro()
{
org.apache.cassandra.db.migration.avro.KsDef ks = new org.apache.cassandra.db.migration.avro.KsDef();
ks.name = new Utf8(name);
ks.strategy_class = new Utf8(strategyClass.getName());
if (strategyOptions != null)
{
ks.strategy_options = new HashMap<CharSequence, CharSequence>();
for (Map.Entry<String, String> e : strategyOptions.entrySet())
{
ks.strategy_options.put(new Utf8(e.getKey()), new Utf8(e.getValue()));
}
}
ks.cf_defs = SerDeUtils.createArray(cfMetaData.size(), org.apache.cassandra.db.migration.avro.CfDef.SCHEMA$);
for (CFMetaData cfm : cfMetaData.values())
ks.cf_defs.add(cfm.toAvro());
ks.durable_writes = durableWrites;
return ks;
}
@Override
public String toString()
@ -154,6 +141,7 @@ public final class KSMetaData
return sb.toString();
}
@Deprecated
public static KSMetaData fromAvro(org.apache.cassandra.db.migration.avro.KsDef ks)
{
Class<? extends AbstractReplicationStrategy> repStratClass;
@ -230,4 +218,198 @@ public final class KSMetaData
return ksdef;
}
public RowMutation diff(KsDef newState, long modificationTimestamp)
{
KsDef curState = toThrift();
RowMutation m = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
for (KsDef._Fields field : KsDef._Fields.values())
{
if (field.equals(KsDef._Fields.CF_DEFS))
continue;
Object curValue = curState.getFieldValue(field);
Object newValue = newState.getFieldValue(field);
if (Objects.equal(curValue, newValue))
continue;
m.add(new QueryPath(SystemTable.SCHEMA_KEYSPACES_CF, null, AsciiType.instance.fromString(field.getFieldName())),
valueAsBytes(newValue),
modificationTimestamp);
}
return m;
}
public KSMetaData reloadAttributes() throws IOException
{
Row ksDefRow = SystemTable.readSchemaRow(name);
if (ksDefRow.cf == null || ksDefRow.cf.isEmpty())
throw new IOException(String.format("%s not found in the schema definitions table (%s).", name, SystemTable.SCHEMA_KEYSPACES_CF));
return fromSchema(ksDefRow.cf, null);
}
public List<RowMutation> dropFromSchema(long timestamp)
{
List<RowMutation> mutations = new ArrayList<RowMutation>();
RowMutation ksMutation = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
ksMutation.delete(new QueryPath(SystemTable.SCHEMA_KEYSPACES_CF), timestamp);
mutations.add(ksMutation);
for (CFMetaData cfm : cfMetaData.values())
mutations.add(cfm.dropFromSchema(timestamp));
return mutations;
}
public static RowMutation toSchema(KsDef ksDef, long timestamp) throws IOException
{
RowMutation mutation = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(ksDef.name));
for (KsDef._Fields field : KsDef._Fields.values())
{
if (field.equals(KsDef._Fields.CF_DEFS))
continue;
mutation.add(new QueryPath(SystemTable.SCHEMA_KEYSPACES_CF,
null,
AsciiType.instance.fromString(field.getFieldName())),
valueAsBytes(ksDef.getFieldValue(field)),
timestamp);
}
if (!ksDef.isSetCf_defs())
return mutation;
for (CfDef cf : ksDef.cf_defs)
{
try
{
CFMetaData.toSchema(mutation, cf, timestamp);
}
catch (ConfigurationException e)
{
throw new IOException(e);
}
}
return mutation;
}
public RowMutation toSchema(long timestamp) throws IOException
{
return toSchema(toThrift(), timestamp);
}
/**
* Deserialize only Keyspace attributes without nested ColumnFamilies
*
* @param serializedKsDef Keyspace attributes in serialized form
*
* @return deserialized keyspace without cf_defs
*
* @throws IOException if deserialization failed
*/
public static KsDef fromSchema(ColumnFamily serializedKsDef) throws IOException
{
KsDef ksDef = new KsDef();
AbstractType comparator = serializedKsDef.getComparator();
for (IColumn ksAttr : serializedKsDef.getSortedColumns())
{
if (ksAttr == null || ksAttr.isMarkedForDelete())
continue;
KsDef._Fields field = KsDef._Fields.findByName(comparator.getString(ksAttr.name()));
ksDef.setFieldValue(field, deserializeValue(ksAttr.value(), getValueClass(KsDef.class, field.getFieldName())));
}
return ksDef.name == null ? null : ksDef;
}
/**
* Deserialize Keyspace with nested ColumnFamilies
*
* @param serializedKsDef Keyspace in serialized form
* @param serializedCFs Collection of the serialized ColumnFamilies
*
* @return deserialized keyspace with cf_defs
*
* @throws IOException if deserialization failed
*/
public static KSMetaData fromSchema(ColumnFamily serializedKsDef, ColumnFamily serializedCFs) throws IOException
{
KsDef ksDef = fromSchema(serializedKsDef);
assert ksDef != null;
Map<String, CfDef> cfs = deserializeColumnFamilies(serializedCFs);
try
{
CFMetaData[] cfms = new CFMetaData[cfs.size()];
int index = 0;
for (CfDef cfDef : cfs.values())
cfms[index++] = CFMetaData.fromThrift(cfDef);
return fromThrift(ksDef, cfms);
}
catch (Exception e)
{
// this is critical because indicates that something is wrong with serialized schema
throw new IOException(e);
}
}
/**
* Deserialize ColumnFamilies from low-level schema representation, all of them belong to the same keyspace
*
* @param serializedColumnFamilies ColumnFamilies in the serialized form
*
* @return map containing name of the ColumnFamily and it's metadata for faster lookup
*/
public static Map<String, CfDef> deserializeColumnFamilies(ColumnFamily serializedColumnFamilies)
{
Map<String, CfDef> cfs = new HashMap<String, CfDef>();
if (serializedColumnFamilies == null)
return cfs;
AbstractType<?> comparator = serializedColumnFamilies.getComparator();
for (IColumn column : serializedColumnFamilies.getSortedColumns())
{
if (column == null || column.isMarkedForDelete())
continue;
String[] attr = comparator.getString(column.name()).split(":");
assert attr.length == 2;
CfDef cfDef = cfs.get(attr[0]);
if (cfDef == null)
{
cfDef = new CfDef();
cfs.put(attr[0], cfDef);
}
CfDef._Fields field = CfDef._Fields.findByName(attr[1]);
cfDef.setFieldValue(field, deserializeValue(column.value(), getValueClass(CfDef.class, field.getFieldName())));
}
for (CfDef cfDef : cfs.values())
{
for (ColumnDef columnDef : ColumnDefinition.fromSchema(cfDef.keyspace, cfDef.name))
cfDef.addToColumn_metadata(columnDef);
}
return cfs;
}
}

141
src/java/org/apache/cassandra/config/Schema.java
View File

@ -20,30 +20,33 @@ package org.apache.cassandra.config;
import java.io.IOError;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.cassandra.db.ColumnFamilyType;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.Pair;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyType;
import org.apache.cassandra.db.Row;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.Pair;
import org.cliffc.high_scale_lib.NonBlockingHashMap;
public class Schema
{
private static final Logger logger = LoggerFactory.getLogger(Schema.class);
public static final UUID INITIAL_VERSION = new UUID(4096, 0); // has type nibble set to 1, everything else to zero.
public static final Schema instance = new Schema(INITIAL_VERSION);
public static final Schema instance = new Schema();
private static final int MIN_CF_ID = 1000;
private final AtomicInteger cfIdGen = new AtomicInteger(MIN_CF_ID);
@ -58,50 +61,60 @@ public class Schema
private final BiMap<Pair<String, String>, Integer> cfIdMap = HashBiMap.create();
private volatile UUID version;
private final ReadWriteLock versionLock = new ReentrantReadWriteLock();
/**
* Initialize empty schema object with given version
* @param initialVersion The initial version of the schema
* Initialize empty schema object
*/
public Schema(UUID initialVersion)
{
version = initialVersion;
}
public Schema()
{}
/**
* Load up non-system tables and set schema version to the given value
* Load up non-system tables
*
* @param tableDefs The non-system table definitions
* @param version The version of the schema
*
* @return self to support chaining calls
*/
public Schema load(Collection<KSMetaData> tableDefs, UUID version)
public Schema load(Collection<KSMetaData> tableDefs)
{
for (KSMetaData def : tableDefs)
load(def);
return this;
}
/**
* Load specific keyspace into Schema
*
* @param keyspaceDef The keyspace to load up
*
* @return self to support chaining calls
*/
public Schema load(KSMetaData keyspaceDef)
{
if (!Migration.isLegalName(keyspaceDef.name))
throw new RuntimeException("invalid keyspace name: " + keyspaceDef.name);
for (CFMetaData cfm : keyspaceDef.cfMetaData().values())
{
if (!Migration.isLegalName(def.name))
throw new RuntimeException("invalid keyspace name: " + def.name);
if (!Migration.isLegalName(cfm.cfName))
throw new RuntimeException("invalid column family name: " + cfm.cfName);
for (CFMetaData cfm : def.cfMetaData().values())
try
{
if (!Migration.isLegalName(cfm.cfName))
throw new RuntimeException("invalid column family name: " + cfm.cfName);
try
{
load(cfm);
}
catch (ConfigurationException ex)
{
throw new IOError(ex);
}
load(cfm);
}
catch (ConfigurationException ex)
{
throw new IOError(ex);
}
setTableDefinition(def, version);
}
setVersion(version);
setTableDefinition(keyspaceDef);
fixCFMaxId();
return this;
}
@ -146,15 +159,13 @@ public class Schema
}
/**
* Remove table definition from system and update schema version
* Remove table definition from system
*
* @param ksm The table definition to remove
* @param newVersion New version of the system
*/
public void clearTableDefinition(KSMetaData ksm, UUID newVersion)
public void clearTableDefinition(KSMetaData ksm)
{
tables.remove(ksm.name);
version = newVersion;
}
/**
@ -319,16 +330,14 @@ public class Schema
}
/**
* Update (or insert) new table definition and change schema version
* Update (or insert) new table definition
*
* @param ksm The metadata about table
* @param newVersion New schema version
*/
public void setTableDefinition(KSMetaData ksm, UUID newVersion)
public void setTableDefinition(KSMetaData ksm)
{
if (ksm != null)
tables.put(ksm.name, ksm);
version = newVersion;
}
/**
@ -381,6 +390,8 @@ public class Schema
logger.debug("Adding {} to cfIdMap", cfm);
cfIdMap.put(key, cfm.cfId);
fixCFMaxId();
}
/**
@ -417,15 +428,47 @@ public class Schema
*/
public UUID getVersion()
{
return version;
versionLock.readLock().lock();
try
{
return version;
}
finally
{
versionLock.readLock().unlock();
}
}
/**
* Set new version of the schema
* @param newVersion New version of the schema
* Read schema from system table and calculate MD5 digest of every row, resulting digest
* will be converted into UUID which would act as content-based version of the schema.
*/
public void setVersion(UUID newVersion)
public void updateVersion()
{
version = newVersion;
versionLock.writeLock().lock();
try
{
MessageDigest versionDigest = MessageDigest.getInstance("MD5");
for (Row row : SystemTable.serializedSchema())
{
if (row.cf == null || row.cf.getColumnCount() == 0)
continue;
row.cf.updateDigest(versionDigest);
}
version = UUID.nameUUIDFromBytes(versionDigest.digest());
}
catch (Exception e)
{
throw new RuntimeException(e);
}
finally
{
versionLock.writeLock().unlock();
}
}
}

45
src/java/org/apache/cassandra/cql/AlterTableStatement.java
View File

@ -20,11 +20,10 @@
*/
package org.apache.cassandra.cql;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.db.migration.avro.CfDef;
import org.apache.cassandra.db.migration.avro.ColumnDef;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnDef;
import org.apache.cassandra.thrift.InvalidRequestException;
import java.nio.ByteBuffer;
@ -72,7 +71,7 @@ public class AlterTableStatement
{
CFMetaData meta = Schema.instance.getCFMetaData(keyspace, columnFamily);
CfDef cfDef = meta.toAvro();
CfDef cfDef = meta.toThrift();
ByteBuffer columnName = this.oType == OperationType.OPTS ? null
: meta.comparator.fromString(this.columnName);
@ -89,20 +88,27 @@ public class AlterTableStatement
TypeParser.parse(validator),
null,
null,
null).toAvro());
null).toThrift());
break;
case ALTER:
ColumnDefinition column = meta.getColumnDefinition(columnName);
ColumnDef toUpdate = null;
if (column == null)
for (ColumnDef columnDef : cfDef.column_metadata)
{
if (columnDef.name.equals(columnName))
{
toUpdate = columnDef;
break;
}
}
if (toUpdate == null)
throw new InvalidRequestException(String.format("Column '%s' was not found in CF '%s'",
this.columnName,
columnFamily));
column.setValidator(TypeParser.parse(validator));
cfDef.column_metadata.add(column.toAvro());
toUpdate.setValidation_class(TypeParser.parse(validator).toString());
break;
case DROP:
@ -121,9 +127,6 @@ public class AlterTableStatement
this.columnName,
columnFamily));
// it is impossible to use ColumnDefinition.deflate() in remove() method
// it will throw java.lang.ClassCastException: java.lang.String cannot be cast to org.apache.avro.util.Utf8
// some where deep inside of Avro
cfDef.column_metadata.remove(toDelete);
break;
@ -156,13 +159,13 @@ public class AlterTableStatement
}
if (cfProps.hasProperty(CFPropDefs.KW_COMMENT))
{
cfDef.comment = new Utf8(cfProps.getProperty(CFPropDefs.KW_COMMENT));
cfDef.comment = cfProps.getProperty(CFPropDefs.KW_COMMENT);
}
if (cfProps.hasProperty(CFPropDefs.KW_DEFAULTVALIDATION))
{
try
{
cfDef.default_validation_class = new Utf8(cfProps.getValidator().toString());
cfDef.default_validation_class = cfProps.getValidator().toString();
}
catch (ConfigurationException e)
{
@ -179,20 +182,16 @@ public class AlterTableStatement
if (!cfProps.compactionStrategyOptions.isEmpty())
{
cfDef.compaction_strategy_options = new HashMap<CharSequence, CharSequence>();
cfDef.compaction_strategy_options = new HashMap<String, String>();
for (Map.Entry<String, String> entry : cfProps.compactionStrategyOptions.entrySet())
{
cfDef.compaction_strategy_options.put(new Utf8(entry.getKey()), new Utf8(entry.getValue()));
}
cfDef.compaction_strategy_options.put(entry.getKey(), entry.getValue());
}
if (!cfProps.compressionParameters.isEmpty())
{
cfDef.compression_options = new HashMap<CharSequence, CharSequence>();
cfDef.compression_options = new HashMap<String, String>();
for (Map.Entry<String, String> entry : cfProps.compressionParameters.entrySet())
{
cfDef.compression_options.put(new Utf8(entry.getKey()), new Utf8(entry.getValue()));
}
cfDef.compression_options.put(entry.getKey(), entry.getValue());
}
}
}

17
src/java/org/apache/cassandra/cql/DropIndexStatement.java
View File

@ -22,20 +22,19 @@ package org.apache.cassandra.cql;
import java.io.IOException;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.migration.avro.CfDef;
import org.apache.cassandra.db.migration.avro.ColumnDef;
import org.apache.cassandra.db.migration.UpdateColumnFamily;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnDef;
import org.apache.cassandra.thrift.InvalidRequestException;
public class DropIndexStatement
{
public final CharSequence index;
public final String index;
public DropIndexStatement(String indexName)
{
index = new Utf8(indexName);
index = indexName;
}
public UpdateColumnFamily generateMutation(String keyspace)
@ -47,7 +46,7 @@ public class DropIndexStatement
for (CFMetaData cfm : ksm.cfMetaData().values())
{
cfDef = getUpdatedCFDef(cfm.toAvro());
cfDef = getUpdatedCFDef(cfm.toThrift());
if (cfDef != null)
break;
}
@ -62,10 +61,10 @@ public class DropIndexStatement
{
for (ColumnDef column : cfDef.column_metadata)
{
if (column.index_type != null && column.index_name != null && column.index_name.equals(index))
if (column.isSetIndex_type() && column.isSetIndex_name() && column.index_name.equals(index))
{
column.index_name = null;
column.index_type = null;
column.unsetIndex_name();
column.unsetIndex_type();
return cfDef;
}
}

47
src/java/org/apache/cassandra/cql/QueryProcessor.java
View File

@ -731,12 +731,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;
@ -758,12 +752,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.toString());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;
@ -802,16 +790,7 @@ public class QueryProcessor
ThriftValidation.validateCfDef(cf_def, oldCfm);
try
{
org.apache.cassandra.db.migration.avro.CfDef result1;
try
{
result1 = CFMetaData.fromThrift(cf_def).toAvro();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
applyMigrationOnStage(new UpdateColumnFamily(result1));
applyMigrationOnStage(new UpdateColumnFamily(cf_def));
}
catch (ConfigurationException e)
{
@ -819,12 +798,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.toString());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;
@ -870,12 +843,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;
@ -895,12 +862,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;
@ -922,12 +883,6 @@ public class QueryProcessor
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
result.type = CqlResultType.VOID;
return result;

19
src/java/org/apache/cassandra/db/ColumnFamily.java
View File

@ -23,6 +23,8 @@ import static org.apache.cassandra.db.DBConstants.*;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.filter.QueryPath;
@ -30,6 +32,7 @@ import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.IColumnSerializer;
import org.apache.cassandra.utils.Allocator;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.HeapAllocator;
@ -255,13 +258,25 @@ public class ColumnFamily extends AbstractColumnContainer
@Override
public int hashCode()
{
throw new RuntimeException("Not implemented.");
return new HashCodeBuilder(373, 75437)
.append(cfm)
.append(getMarkedForDeleteAt())
.append(columns).toHashCode();
}
@Override
public boolean equals(Object o)
{
throw new RuntimeException("Not implemented.");
if (this == o)
return true;
if (o == null || this.getClass() != o.getClass())
return false;
ColumnFamily comparison = (ColumnFamily) o;
return cfm.equals(comparison.cfm)
&& getMarkedForDeleteAt() == comparison.getMarkedForDeleteAt()
&& ByteBufferUtil.compareUnsigned(digest(this), digest(comparison)) == 0;
}
@Override

67
src/java/org/apache/cassandra/db/DefinitionsUpdateVerbHandler.java
View File

@ -18,76 +18,35 @@
package org.apache.cassandra.db;
import java.io.IOError;
import java.io.IOException;
import java.util.Collection;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.WrappedRunnable;
/**
* Called when node receives updated schema state from the schema migration coordinator node.
* Such happens when user makes local schema migration on one of the nodes in the ring
* (which is going to act as coordinator) and that node sends (pushes) it's updated schema state
* (in form of row mutations) to all the alive nodes in the cluster.
*/
public class DefinitionsUpdateVerbHandler implements IVerbHandler
{
private static final Logger logger = LoggerFactory.getLogger(DefinitionsUpdateVerbHandler.class);
/** someone sent me their data definitions */
public void doVerb(final Message message, String id)
{
try
logger.debug("Received schema mutation push from " + message.getFrom());
StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
{
// these are the serialized row mutations that I must apply.
// check versions at every step along the way to make sure migrations are not applied out of order.
Collection<Column> cols = MigrationManager.makeColumns(message);
for (Column col : cols)
public void runMayThrow() throws Exception
{
final UUID version = UUIDGen.getUUID(col.name());
if (version.timestamp() > Schema.instance.getVersion().timestamp())
{
final Migration m = Migration.deserialize(col.value(), message.getVersion());
assert m.getVersion().equals(version);
StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
{
protected void runMayThrow() throws Exception
{
// check to make sure the current version is before this one.
if (Schema.instance.getVersion().timestamp() == version.timestamp())
logger.debug("Not appling (equal) " + version.toString());
else if (Schema.instance.getVersion().timestamp() > version.timestamp())
logger.debug("Not applying (before)" + version.toString());
else
{
logger.debug("Applying {} from {}", m.getClass().getSimpleName(), message.getFrom());
try
{
m.apply();
// update gossip, but don't contact nodes directly
m.passiveAnnounce();
}
catch (ConfigurationException ex)
{
// Trying to apply the same migration twice. This happens as a result of gossip.
logger.debug("Migration not applied " + ex.getMessage());
}
}
}
});
}
DefsTable.mergeRemoteSchema(message.getMessageBody(), message.getVersion());
}
}
catch (IOException ex)
{
throw new IOError(ex);
}
});
}
}
}

403
src/java/org/apache/cassandra/db/DefsTable.java
View File

@ -18,85 +18,384 @@
package org.apache.cassandra.db;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.avro.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificRecord;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.db.migration.MigrationHelper;
import org.apache.cassandra.db.migration.avro.KsDef;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
/**
* SCHEMA_{KEYSPACES, COLUMNFAMILIES, COLUMNS}_CF are used to store Keyspace/ColumnFamily attributes to make schema
* load/distribution easy, it replaces old mechanism when local migrations where serialized, stored in system.Migrations
* and used for schema distribution.
*
* SCHEMA_KEYSPACES_CF layout:
*
* <key (AsciiType)>
* ascii => json_serialized_value
* ...
* </key>
*
* Where <key> is a name of keyspace e.g. "ks".
*
* SCHEMA_COLUMNFAMILIES_CF layout:
*
* <key (AsciiType)>
* composite(ascii, ascii) => json_serialized_value
* </key>
*
* Where <key> is a name of keyspace e.g. "ks"., first component of the column name is name of the ColumnFamily, last
* component is the name of the ColumnFamily attribute.
*
* SCHEMA_COLUMNS_CF layout:
*
* <key (AsciiType)>
* composite(ascii, ascii, ascii) => json_serialized value
* </key>
*
* Where <key> is a name of keyspace e.g. "ks".
*
* Column names where made composite to support 3-level nesting which represents following structure:
* "ColumnFamily name":"column name":"column attribute" => "value"
*
* Example of schema (using CLI):
*
* schema_keyspaces
* ----------------
* RowKey: ks
* => (column=durable_writes, value=true, timestamp=1327061028312185000)
* => (column=name, value="ks", timestamp=1327061028312185000)
* => (column=replication_factor, value=0, timestamp=1327061028312185000)
* => (column=strategy_class, value="org.apache.cassandra.locator.NetworkTopologyStrategy", timestamp=1327061028312185000)
* => (column=strategy_options, value={"datacenter1":"1"}, timestamp=1327061028312185000)
*
* schema_columnfamilies
* ---------------------
* RowKey: ks
* => (column=cf:bloom_filter_fp_chance, value=0.0, timestamp=1327061105833119000)
* => (column=cf:caching, value="NONE", timestamp=1327061105833119000)
* => (column=cf:column_type, value="Standard", timestamp=1327061105833119000)
* => (column=cf:comment, value="ColumnFamily", timestamp=1327061105833119000)
* => (column=cf:default_validation_class, value="org.apache.cassandra.db.marshal.BytesType", timestamp=1327061105833119000)
* => (column=cf:gc_grace_seconds, value=864000, timestamp=1327061105833119000)
* => (column=cf:id, value=1000, timestamp=1327061105833119000)
* => (column=cf:key_alias, value="S0VZ", timestamp=1327061105833119000)
* ... part of the output omitted.
*
* schema_columns
* --------------
* RowKey: ks
* => (column=cf:c:index_name, value=null, timestamp=1327061105833119000)
* => (column=cf:c:index_options, value=null, timestamp=1327061105833119000)
* => (column=cf:c:index_type, value=null, timestamp=1327061105833119000)
* => (column=cf:c:name, value="aGVsbG8=", timestamp=1327061105833119000)
* => (column=cf:c:validation_class, value="org.apache.cassandra.db.marshal.AsciiType", timestamp=1327061105833119000)
*/
public class DefsTable
{
private final static Logger logger = LoggerFactory.getLogger(DefsTable.class);
// unbuffered decoders
private final static DecoderFactory DIRECT_DECODERS = new DecoderFactory().configureDirectDecoder(true);
// column name for the schema storing serialized keyspace definitions
// NB: must be an invalid keyspace name
public static final ByteBuffer DEFINITION_SCHEMA_COLUMN_NAME = ByteBufferUtil.bytes("Avro/Schema");
/** dumps current keyspace definitions to storage */
public static synchronized void dumpToStorage(UUID version) throws IOException
/* dumps current keyspace definitions to storage */
public static synchronized void dumpToStorage(Collection<KSMetaData> keyspaces) throws IOException
{
final ByteBuffer versionKey = Migration.toUTF8Bytes(version);
long timestamp = System.currentTimeMillis();
// build a list of keyspaces
Collection<String> ksnames = org.apache.cassandra.config.Schema.instance.getNonSystemTables();
// persist keyspaces under new version
RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, versionKey);
long now = System.currentTimeMillis();
for (String ksname : ksnames)
{
KSMetaData ksm = org.apache.cassandra.config.Schema.instance.getTableDefinition(ksname);
rm.add(new QueryPath(Migration.SCHEMA_CF, null, ByteBufferUtil.bytes(ksm.name)), SerDeUtils.serialize(ksm.toAvro()), now);
}
// add the schema
rm.add(new QueryPath(Migration.SCHEMA_CF,
null,
DEFINITION_SCHEMA_COLUMN_NAME),
ByteBufferUtil.bytes(org.apache.cassandra.db.migration.avro.KsDef.SCHEMA$.toString()),
now);
rm.apply();
// apply new version
rm = new RowMutation(Table.SYSTEM_TABLE, Migration.LAST_MIGRATION_KEY);
rm.add(new QueryPath(Migration.SCHEMA_CF, null, Migration.LAST_MIGRATION_KEY),
ByteBuffer.wrap(UUIDGen.decompose(version)),
now);
rm.apply();
for (KSMetaData ksMetaData : keyspaces)
ksMetaData.toSchema(timestamp).apply();
}
/** loads a version of keyspace definitions from storage */
public static synchronized Collection<KSMetaData> loadFromStorage(UUID version) throws IOException
/**
* Load keyspace definitions for the system keyspace (system.SCHEMA_KEYSPACES_CF)
*
* @return Collection of found keyspace definitions
*
* @throws IOException if failed to read SCHEMA_KEYSPACES_CF
*/
public static Collection<KSMetaData> loadFromTable() throws IOException
{
DecoratedKey vkey = StorageService.getPartitioner().decorateKey(Migration.toUTF8Bytes(version));
Table defs = Table.open(Table.SYSTEM_TABLE);
ColumnFamilyStore cfStore = defs.getColumnFamilyStore(Migration.SCHEMA_CF);
QueryFilter filter = QueryFilter.getIdentityFilter(vkey, new QueryPath(Migration.SCHEMA_CF));
ColumnFamily cf = cfStore.getColumnFamily(filter);
IColumn avroschema = cf.getColumn(DEFINITION_SCHEMA_COLUMN_NAME);
if (avroschema == null)
// TODO: more polite way to handle this?
throw new RuntimeException("Cannot read system table! Are you upgrading a pre-release version?");
List<Row> serializedSchema = SystemTable.serializedSchema(SystemTable.SCHEMA_KEYSPACES_CF);
ByteBuffer value = avroschema.value();
Schema schema = Schema.parse(ByteBufferUtil.string(value));
List<KSMetaData> keyspaces = new ArrayList<KSMetaData>();
// deserialize keyspaces using schema
Collection<KSMetaData> keyspaces = new ArrayList<KSMetaData>();
for (IColumn column : cf.getSortedColumns())
for (Row row : serializedSchema)
{
if (column.name().equals(DEFINITION_SCHEMA_COLUMN_NAME))
if (row.cf == null || row.cf.isEmpty() || row.cf.isMarkedForDelete())
continue;
org.apache.cassandra.db.migration.avro.KsDef ks = SerDeUtils.deserialize(schema, column.value(), new org.apache.cassandra.db.migration.avro.KsDef());
keyspaces.add(KSMetaData.fromAvro(ks));
keyspaces.add(KSMetaData.fromSchema(row.cf, serializedColumnFamilies(row.key)));
}
return keyspaces;
}
private static ColumnFamily serializedColumnFamilies(DecoratedKey ksNameKey)
{
ColumnFamilyStore cfsStore = SystemTable.schemaCFS(SystemTable.SCHEMA_COLUMNFAMILIES_CF);
return cfsStore.getColumnFamily(QueryFilter.getIdentityFilter(ksNameKey, new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF)));
}
/**
* Loads a version of keyspace definitions from storage (using old SCHEMA_CF as a data source)
* Note: If definitions where found in SCHEMA_CF this method would load them into new schema handling table KEYSPACE_CF
*
* @param version The version of the latest migration.
*
* @return Collection of found keyspace definitions
*
* @throws IOException if failed to read SCHEMA_CF or failed to deserialize Avro schema
*/
public static synchronized Collection<KSMetaData> loadFromStorage(UUID version) throws IOException
{
DecoratedKey vkey = StorageService.getPartitioner().decorateKey(toUTF8Bytes(version));
Table defs = Table.open(Table.SYSTEM_TABLE);
ColumnFamilyStore cfStore = defs.getColumnFamilyStore(Migration.SCHEMA_CF);
ColumnFamily cf = cfStore.getColumnFamily(QueryFilter.getIdentityFilter(vkey, new QueryPath(Migration.SCHEMA_CF)));
IColumn avroschema = cf.getColumn(DEFINITION_SCHEMA_COLUMN_NAME);
Collection<KSMetaData> keyspaces = Collections.emptyList();
if (avroschema != null)
{
ByteBuffer value = avroschema.value();
org.apache.avro.Schema schema = org.apache.avro.Schema.parse(ByteBufferUtil.string(value));
// deserialize keyspaces using schema
keyspaces = new ArrayList<KSMetaData>();
for (IColumn column : cf.getSortedColumns())
{
if (column.name().equals(DEFINITION_SCHEMA_COLUMN_NAME))
continue;
KsDef ks = deserializeAvro(schema, column.value(), new KsDef());
keyspaces.add(KSMetaData.fromAvro(ks));
}
// store deserialized keyspaces into new place
dumpToStorage(keyspaces);
logger.info("Truncating deprecated system column families (migrations, schema)...");
MigrationHelper.dropColumnFamily(Table.SYSTEM_TABLE, Migration.MIGRATIONS_CF);
MigrationHelper.dropColumnFamily(Table.SYSTEM_TABLE, Migration.SCHEMA_CF);
}
return keyspaces;
}
/**
* Merge remote schema in form of row mutations with local and mutate ks/cf metadata objects
* (which also involves fs operations on add/drop ks/cf)
*
* @param data The data of the message from remote node with schema information
* @param version The version of the message
*
* @throws ConfigurationException If one of metadata attributes has invalid value
* @throws IOException If data was corrupted during transportation or failed to apply fs operations
*/
public static void mergeRemoteSchema(byte[] data, int version) throws ConfigurationException, IOException
{
// save current state of the schema
Map<DecoratedKey, ColumnFamily> oldKeyspaces = SystemTable.getSchema(SystemTable.SCHEMA_KEYSPACES_CF);
Map<DecoratedKey, ColumnFamily> oldColumnFamilies = SystemTable.getSchema(SystemTable.SCHEMA_COLUMNFAMILIES_CF);
// apply remote mutations
for (RowMutation mutation : MigrationManager.deserializeMigrationMessage(data, version))
mutation.apply();
if (!StorageService.instance.isClientMode())
MigrationHelper.flushSchemaCFs();
Schema.instance.updateVersion();
Set<String> keyspacesToDrop = mergeKeyspaces(oldKeyspaces, SystemTable.getSchema(SystemTable.SCHEMA_KEYSPACES_CF));
mergeColumnFamilies(oldColumnFamilies, SystemTable.getSchema(SystemTable.SCHEMA_COLUMNFAMILIES_CF));
// it is save to drop a keyspace only when all nested ColumnFamilies where deleted
for (String keyspaceToDrop : keyspacesToDrop)
MigrationHelper.dropKeyspace(keyspaceToDrop);
}
private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated)
throws ConfigurationException, IOException
{
// calculate the difference between old and new states (note that entriesOnlyLeft() will be always empty)
MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated);
/**
* At first step we check if any new keyspaces were added.
*/
for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
{
ColumnFamily ksAttrs = entry.getValue();
// we don't care about nested ColumnFamilies here because those are going to be processed separately
if (!ksAttrs.isEmpty())
MigrationHelper.addKeyspace(KSMetaData.fromSchema(entry.getValue(), null));
}
/**
* At second step we check if there were any keyspaces re-created, in this context
* re-created means that they were previously deleted but still exist in the low-level schema as empty keys
*/
Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries = diff.entriesDiffering();
// instead of looping over all modified entries and skipping processed keys all the time
// we would rather store "left to process" items and iterate over them removing already met keys
List<DecoratedKey> leftToProcess = new ArrayList<DecoratedKey>();
for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> entry : modifiedEntries.entrySet())
{
ColumnFamily prevValue = entry.getValue().leftValue();
ColumnFamily newValue = entry.getValue().rightValue();
if (prevValue.isEmpty())
{
MigrationHelper.addKeyspace(KSMetaData.fromSchema(newValue, null));
continue;
}
leftToProcess.add(entry.getKey());
}
if (leftToProcess.size() == 0)
return Collections.emptySet();
/**
* At final step we updating modified keyspaces and saving keyspaces drop them later
*/
Set<String> keyspacesToDrop = new HashSet<String>();
for (DecoratedKey key : leftToProcess)
{
MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(key);
ColumnFamily newState = valueDiff.rightValue();
if (newState.isEmpty())
keyspacesToDrop.add(AsciiType.instance.getString(key.key));
else
MigrationHelper.updateKeyspace(KSMetaData.fromSchema(newState));
}
return keyspacesToDrop;
}
private static void mergeColumnFamilies(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey, ColumnFamily> updated)
throws ConfigurationException, IOException
{
// calculate the difference between old and new states (note that entriesOnlyLeft() will be always empty)
MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated);
// check if any new Keyspaces with ColumnFamilies were added.
for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
{
ColumnFamily cfAttrs = entry.getValue();
if (!cfAttrs.isEmpty())
{
Map<String, CfDef> cfDefs = KSMetaData.deserializeColumnFamilies(cfAttrs);
for (CfDef cfDef : cfDefs.values())
MigrationHelper.addColumnFamily(cfDef);
}
}
// deal with modified ColumnFamilies (remember that all of the keyspace nested ColumnFamilies are put to the single row)
Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries = diff.entriesDiffering();
for (DecoratedKey keyspace : modifiedEntries.keySet())
{
MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(keyspace);
ColumnFamily prevValue = valueDiff.leftValue(); // state before external modification
ColumnFamily newValue = valueDiff.rightValue(); // updated state
if (prevValue.isEmpty()) // whole keyspace was deleted and now it's re-created
{
for (CfDef cfDef : KSMetaData.deserializeColumnFamilies(newValue).values())
MigrationHelper.addColumnFamily(cfDef);
}
else if (newValue.isEmpty()) // whole keyspace is deleted
{
for (CfDef cfDef : KSMetaData.deserializeColumnFamilies(prevValue).values())
MigrationHelper.dropColumnFamily(cfDef.keyspace, cfDef.name);
}
else // has modifications in the nested ColumnFamilies, need to perform nested diff to determine what was really changed
{
String ksName = AsciiType.instance.getString(keyspace.key);
Map<String, CfDef> oldCfDefs = new HashMap<String, CfDef>();
for (CFMetaData cfm : Schema.instance.getKSMetaData(ksName).cfMetaData().values())
oldCfDefs.put(cfm.cfName, cfm.toThrift());
Map<String, CfDef> newCfDefs = KSMetaData.deserializeColumnFamilies(newValue);
MapDifference<String, CfDef> cfDefDiff = Maps.difference(oldCfDefs, newCfDefs);
for (CfDef cfDef : cfDefDiff.entriesOnlyOnRight().values())
MigrationHelper.addColumnFamily(cfDef);
for (CfDef cfDef : cfDefDiff.entriesOnlyOnLeft().values())
MigrationHelper.dropColumnFamily(cfDef.keyspace, cfDef.name);
for (MapDifference.ValueDifference<CfDef> cfDef : cfDefDiff.entriesDiffering().values())
MigrationHelper.updateColumnFamily(cfDef.rightValue());
}
}
}
private static ByteBuffer toUTF8Bytes(UUID version)
{
return ByteBufferUtil.bytes(version.toString());
}
/**
* Deserialize a single object based on the given Schema.
*
* @param writer writer's schema
* @param bytes Array to deserialize from
* @param ob An empty object to deserialize into (must not be null).
*
* @return serialized Avro object
*
* @throws IOException if deserialization failed
*/
public static <T extends SpecificRecord> T deserializeAvro(org.apache.avro.Schema writer, ByteBuffer bytes, T ob) throws IOException
{
BinaryDecoder dec = DIRECT_DECODERS.createBinaryDecoder(ByteBufferUtil.getArray(bytes), null);
SpecificDatumReader<T> reader = new SpecificDatumReader<T>(writer);
reader.setExpected(ob.getSchema());
return reader.read(ob, dec);
}
}

53
src/java/org/apache/cassandra/db/MigrationRequestVerbHandler.java Normal file
View File

@ -0,0 +1,53 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.io.IOError;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.MigrationManager;
/**
* Sends it's current schema state in form of row mutations in reply to the remote node's request.
* Such a request is made when one of the nodes, by means of Gossip, detects schema disagreement in the ring.
*/
public class MigrationRequestVerbHandler implements IVerbHandler
{
private static final Logger logger = LoggerFactory.getLogger(MigrationRequestVerbHandler.class);
public void doVerb(Message message, String id)
{
logger.debug("Received migration request from {}.", message.getFrom());
try
{
Message response = message.getInternalReply(MigrationManager.serializeSchema(SystemTable.serializeSchema(), message.getVersion()), message.getVersion());
MessagingService.instance().sendReply(response, id, message.getFrom());
}
catch (IOException e)
{
throw new IOError(e);
}
}
}

118
src/java/org/apache/cassandra/db/SystemTable.java
View File

@ -23,11 +23,7 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.*;
import java.util.concurrent.ExecutionException;
import org.slf4j.Logger;
@ -36,10 +32,14 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql.QueryProcessor;
import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.migration.MigrationHelper;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.Constants;
@ -54,6 +54,11 @@ public class SystemTable
public static final String INDEX_CF = "IndexInfo";
public static final String NODE_ID_CF = "NodeIdInfo";
public static final String VERSION_CF = "Versions";
// see layout description in the DefsTable class header
public static final String SCHEMA_KEYSPACES_CF = "schema_keyspaces";
public static final String SCHEMA_COLUMNFAMILIES_CF = "schema_columnfamilies";
public static final String SCHEMA_COLUMNS_CF = "schema_columns";
private static final ByteBuffer LOCATION_KEY = ByteBufferUtil.bytes("L");
private static final ByteBuffer RING_KEY = ByteBufferUtil.bytes("Ring");
private static final ByteBuffer BOOTSTRAP_KEY = ByteBufferUtil.bytes("Bootstrap");
@ -506,4 +511,107 @@ public class SystemTable
}
return l;
}
/**
* @param cfName The name of the ColumnFamily responsible for part of the schema (keyspace, ColumnFamily, columns)
* @return CFS responsible to hold low-level serialized schema
*/
public static ColumnFamilyStore schemaCFS(String cfName)
{
return Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(cfName);
}
public static List<Row> serializedSchema()
{
List<Row> schema = new ArrayList<Row>();
schema.addAll(serializedSchema(SCHEMA_KEYSPACES_CF));
schema.addAll(serializedSchema(SCHEMA_COLUMNFAMILIES_CF));
schema.addAll(serializedSchema(SCHEMA_COLUMNS_CF));
return schema;
}
/**
* @param schemaCfName The name of the ColumnFamily responsible for part of the schema (keyspace, ColumnFamily, columns)
* @return low-level schema representation (each row represents individual Keyspace or ColumnFamily)
*/
public static List<Row> serializedSchema(String schemaCfName)
{
Token minToken = StorageService.getPartitioner().getMinimumToken();
return schemaCFS(schemaCfName).getRangeSlice(null,
new Range<RowPosition>(minToken.minKeyBound(),
minToken.maxKeyBound()),
Integer.MAX_VALUE,
new IdentityQueryFilter(),
null);
}
public static Collection<RowMutation> serializeSchema()
{
Map<DecoratedKey, RowMutation> mutationMap = new HashMap<DecoratedKey, RowMutation>();
serializeSchema(mutationMap, SCHEMA_KEYSPACES_CF);
serializeSchema(mutationMap, SCHEMA_COLUMNFAMILIES_CF);
serializeSchema(mutationMap, SCHEMA_COLUMNS_CF);
return mutationMap.values();
}
private static void serializeSchema(Map<DecoratedKey, RowMutation> mutationMap, String schemaCfName)
{
for (Row schemaRow : serializedSchema(schemaCfName))
{
RowMutation mutation = mutationMap.get(schemaRow.key);
if (mutation == null)
{
mutationMap.put(schemaRow.key, new RowMutation(Table.SYSTEM_TABLE, schemaRow));
continue;
}
mutation.add(schemaRow.cf);
}
}
public static Map<DecoratedKey, ColumnFamily> getSchema(String cfName)
{
Map<DecoratedKey, ColumnFamily> schema = new HashMap<DecoratedKey, ColumnFamily>();
for (Row schemaEntity : SystemTable.serializedSchema(cfName))
schema.put(schemaEntity.key, schemaEntity.cf);
return schema;
}
public static ByteBuffer getSchemaKSKey(String ksName)
{
return AsciiType.instance.fromString(ksName);
}
public static Row readSchemaRow(String ksName)
{
DecoratedKey key = StorageService.getPartitioner().decorateKey(getSchemaKSKey(ksName));
ColumnFamilyStore schemaCFS = SystemTable.schemaCFS(SCHEMA_KEYSPACES_CF);
ColumnFamily result = schemaCFS.getColumnFamily(QueryFilter.getIdentityFilter(key, new QueryPath(SCHEMA_KEYSPACES_CF)));
return new Row(key, result);
}
public static Row readSchemaRow(String ksName, String cfName)
{
DecoratedKey key = StorageService.getPartitioner().decorateKey(getSchemaKSKey(ksName));
ColumnFamilyStore schemaCFS = SystemTable.schemaCFS(SCHEMA_COLUMNFAMILIES_CF);
ColumnFamily result = schemaCFS.getColumnFamily(key,
new QueryPath(SCHEMA_COLUMNFAMILIES_CF),
MigrationHelper.searchComposite(cfName, true),
MigrationHelper.searchComposite(cfName, false),
false,
Integer.MAX_VALUE);
return new Row(key, result);
}
}

95
src/java/org/apache/cassandra/db/migration/AddColumnFamily.java
View File

@ -1,17 +1,3 @@
package org.apache.cassandra.db.migration;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import com.google.common.collect.Iterables;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -29,79 +15,38 @@ import org.apache.cassandra.utils.UUIDGen;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.migration;
import java.io.IOException;
public class AddColumnFamily extends Migration
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
public class AddColumnFamily extends Migration
{
private CFMetaData cfm;
private final CFMetaData cfm;
/** Required no-arg constructor */
protected AddColumnFamily() { /* pass */ }
public AddColumnFamily(CFMetaData cfm) throws ConfigurationException, IOException
public AddColumnFamily(CFMetaData cfm) throws ConfigurationException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
this.cfm = cfm;
KSMetaData ksm = schema.getTableDefinition(cfm.ksName);
super(System.nanoTime());
KSMetaData ksm = Schema.instance.getTableDefinition(cfm.ksName);
if (ksm == null)
throw new ConfigurationException("No such keyspace: " + cfm.ksName);
throw new ConfigurationException(String.format("Can't add ColumnFamily '%s' to Keyspace '%s': Keyspace does not exist.", cfm.cfName, cfm.ksName));
else if (ksm.cfMetaData().containsKey(cfm.cfName))
throw new ConfigurationException(String.format("%s already exists in keyspace %s",
cfm.cfName,
cfm.ksName));
throw new ConfigurationException(String.format("Can't add ColumnFamily '%s' to Keyspace '%s': Already exists.", cfm.cfName, cfm.ksName));
else if (!Migration.isLegalName(cfm.cfName))
throw new ConfigurationException("Invalid column family name: " + cfm.cfName);
for (Map.Entry<ByteBuffer, ColumnDefinition> entry : cfm.getColumn_metadata().entrySet())
{
String indexName = entry.getValue().getIndexName();
if (indexName != null && !Migration.isLegalName(indexName))
throw new ConfigurationException("Invalid index name: " + indexName);
}
throw new ConfigurationException("Can't add ColumnFamily '%s' to Keyspace '%s': Invalid ColumnFamily name.");
// clone ksm but include the new cf def.
KSMetaData newKsm = makeNewKeyspaceDefinition(ksm);
rm = makeDefinitionMutation(newKsm, null, newVersion);
}
private KSMetaData makeNewKeyspaceDefinition(KSMetaData ksm)
{
return KSMetaData.cloneWith(ksm, Iterables.concat(ksm.cfMetaData().values(), Collections.singleton(cfm)));
}
public void applyModels() throws IOException
{
// reinitialize the table.
KSMetaData ksm = schema.getTableDefinition(cfm.ksName);
ksm = makeNewKeyspaceDefinition(ksm);
try
{
schema.load(cfm);
}
catch (ConfigurationException ex)
{
throw new IOException(ex);
}
Table.open(cfm.ksName, schema); // make sure it's init-ed w/ the old definitions first, since we're going to call initCf on the new one manually
schema.setTableDefinition(ksm, newVersion);
// these definitions could have come from somewhere else.
schema.fixCFMaxId();
if (!StorageService.instance.isClientMode())
Table.open(ksm.name, schema).initCf(cfm.cfId, cfm.cfName);
this.cfm = cfm;
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
protected void applyImpl() throws ConfigurationException, IOException
{
org.apache.cassandra.db.migration.avro.AddColumnFamily acf = new org.apache.cassandra.db.migration.avro.AddColumnFamily();
acf.cf = cfm.toAvro();
mi.migration = acf;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.AddColumnFamily acf = (org.apache.cassandra.db.migration.avro.AddColumnFamily)mi.migration;
cfm = CFMetaData.fromAvro(acf.cf);
MigrationHelper.addColumnFamily(cfm, timestamp);
}
@Override

69
src/java/org/apache/cassandra/db/migration/AddKeyspace.java
View File

@ -20,70 +20,33 @@ package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
public class AddKeyspace extends Migration
{
private KSMetaData ksm;
private final KSMetaData ksm;
/** Required no-arg constructor */
protected AddKeyspace() { /* pass */ }
public AddKeyspace(KSMetaData ksm) throws ConfigurationException, IOException
public AddKeyspace(KSMetaData ksm) throws ConfigurationException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
if (schema.getTableDefinition(ksm.name) != null)
throw new ConfigurationException("Keyspace already exists.");
if (!Migration.isLegalName(ksm.name))
throw new ConfigurationException("Invalid keyspace name: " + ksm.name);
super(System.nanoTime());
if (Schema.instance.getTableDefinition(ksm.name) != null)
throw new ConfigurationException(String.format("Can't add Keyspace '%s': Already exists.", ksm.name));
else if (!Migration.isLegalName(ksm.name))
throw new ConfigurationException(String.format("Can't add Keyspace '%s': Invalid name.", ksm.name));
for (CFMetaData cfm : ksm.cfMetaData().values())
if (!Migration.isLegalName(cfm.cfName))
throw new ConfigurationException("Invalid column family name: " + cfm.cfName);
throw new ConfigurationException(String.format("Can't add Keyspace '%s': Invalid ColumnFamily name '%s'.", ksm.name, cfm.cfName));
this.ksm = ksm;
rm = makeDefinitionMutation(ksm, null, newVersion);
}
public void applyModels() throws IOException
protected void applyImpl() throws ConfigurationException, IOException
{
for (CFMetaData cfm : ksm.cfMetaData().values())
{
try
{
schema.load(cfm);
}
catch (ConfigurationException ex)
{
// throw RTE since this indicates a table,cf maps to an existing ID. It shouldn't if this is really a
// new keyspace.
throw new RuntimeException(ex);
}
}
schema.setTableDefinition(ksm, newVersion);
// these definitions could have come from somewhere else.
schema.fixCFMaxId();
if (!StorageService.instance.isClientMode())
{
Table.open(ksm.name, schema);
}
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.AddKeyspace aks = new org.apache.cassandra.db.migration.avro.AddKeyspace();
aks.ks = ksm.toAvro();
mi.migration = aks;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.AddKeyspace aks = (org.apache.cassandra.db.migration.avro.AddKeyspace)mi.migration;
ksm = KSMetaData.fromAvro(aks.ks);
MigrationHelper.addKeyspace(ksm, timestamp);
}
@Override

91
src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
View File

@ -1,17 +1,3 @@
package org.apache.cassandra.db.migration;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -29,78 +15,41 @@ import org.apache.cassandra.utils.UUIDGen;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
public class DropColumnFamily extends Migration
{
private String tableName;
private String cfName;
private final String ksName;
private final String cfName;
/** Required no-arg constructor */
protected DropColumnFamily() { /* pass */ }
public DropColumnFamily(String tableName, String cfName) throws ConfigurationException, IOException
public DropColumnFamily(String ksName, String cfName) throws ConfigurationException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
this.tableName = tableName;
this.cfName = cfName;
KSMetaData ksm = schema.getTableDefinition(tableName);
super(System.nanoTime());
KSMetaData ksm = Schema.instance.getTableDefinition(ksName);
if (ksm == null)
throw new ConfigurationException("No such keyspace: " + tableName);
throw new ConfigurationException("Can't drop ColumnFamily: No such keyspace '" + ksName + "'.");
else if (!ksm.cfMetaData().containsKey(cfName))
throw new ConfigurationException("CF is not defined in that keyspace.");
KSMetaData newKsm = makeNewKeyspaceDefinition(ksm);
rm = makeDefinitionMutation(newKsm, null, newVersion);
throw new ConfigurationException(String.format("Can't drop ColumnFamily (ks=%s, cf=%s) : Not defined in that keyspace.", ksName, cfName));
this.ksName = ksName;
this.cfName = cfName;
}
private KSMetaData makeNewKeyspaceDefinition(KSMetaData ksm)
protected void applyImpl() throws ConfigurationException, IOException
{
// clone ksm but do not include the new def
CFMetaData cfm = ksm.cfMetaData().get(cfName);
List<CFMetaData> newCfs = new ArrayList<CFMetaData>(ksm.cfMetaData().values());
newCfs.remove(cfm);
assert newCfs.size() == ksm.cfMetaData().size() - 1;
return KSMetaData.cloneWith(ksm, newCfs);
}
public void applyModels() throws IOException
{
ColumnFamilyStore cfs = Table.open(tableName, schema).getColumnFamilyStore(cfName);
// reinitialize the table.
KSMetaData existing = schema.getTableDefinition(tableName);
CFMetaData cfm = existing.cfMetaData().get(cfName);
KSMetaData ksm = makeNewKeyspaceDefinition(existing);
schema.purge(cfm);
schema.setTableDefinition(ksm, newVersion);
if (!StorageService.instance.isClientMode())
{
cfs.snapshot(Table.getTimestampedSnapshotName(cfs.columnFamily));
Table.open(ksm.name, schema).dropCf(cfm.cfId);
}
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.DropColumnFamily dcf = new org.apache.cassandra.db.migration.avro.DropColumnFamily();
dcf.ksname = new org.apache.avro.util.Utf8(tableName);
dcf.cfname = new org.apache.avro.util.Utf8(cfName);
mi.migration = dcf;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.DropColumnFamily dcf = (org.apache.cassandra.db.migration.avro.DropColumnFamily)mi.migration;
tableName = dcf.ksname.toString();
cfName = dcf.cfname.toString();
MigrationHelper.dropColumnFamily(ksName, cfName, timestamp);
}
@Override
public String toString()
{
return String.format("Drop column family: %s.%s", tableName, cfName);
return String.format("Drop column family: %s.%s", ksName, cfName);
}
}

63
src/java/org/apache/cassandra/db/migration/DropKeyspace.java
View File

@ -20,65 +20,28 @@ package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
public class DropKeyspace extends Migration
{
private String name;
private final String name;
/** Required no-arg constructor */
protected DropKeyspace() { /* pass */ }
public DropKeyspace(String name) throws ConfigurationException, IOException
public DropKeyspace(String name) throws ConfigurationException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
this.name = name;
KSMetaData ksm = schema.getTableDefinition(name);
super(System.nanoTime());
KSMetaData ksm = Schema.instance.getTableDefinition(name);
if (ksm == null)
throw new ConfigurationException("Keyspace does not exist.");
rm = makeDefinitionMutation(null, ksm, newVersion);
throw new ConfigurationException("Can't drop keyspace '" + name + "' because it does not exist.");
this.name = name;
}
public void applyModels() throws IOException
protected void applyImpl() throws ConfigurationException, IOException
{
String snapshotName = Table.getTimestampedSnapshotName(name);
KSMetaData ksm = schema.getTableDefinition(name);
// remove all cfs from the table instance.
for (CFMetaData cfm : ksm.cfMetaData().values())
{
ColumnFamilyStore cfs = Table.open(ksm.name, schema).getColumnFamilyStore(cfm.cfName);
schema.purge(cfm);
if (!StorageService.instance.isClientMode())
{
cfs.snapshot(snapshotName);
Table.open(ksm.name, schema).dropCf(cfm.cfId);
}
}
// remove the table from the static instances.
Table.clear(ksm.name, schema);
// reset defs.
schema.clearTableDefinition(ksm, newVersion);
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.DropKeyspace dks = new org.apache.cassandra.db.migration.avro.DropKeyspace();
dks.ksname = new org.apache.avro.util.Utf8(name);
mi.migration = dks;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.DropKeyspace dks = (org.apache.cassandra.db.migration.avro.DropKeyspace)mi.migration;
name = dks.ksname.toString();
MigrationHelper.dropKeyspace(name, timestamp);
}
@Override

269
src/java/org/apache/cassandra/db/migration/Migration.java
View File

@ -19,48 +19,31 @@
package org.apache.cassandra.db.migration;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.net.MessagingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.QueryFilter;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.service.MigrationManager;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.UUIDGen;
/**
* A migration represents a single metadata mutation (cf dropped, added, etc.). Migrations can be applied locally, or
* serialized and sent to another machine where it can be applied there. Each migration has a version represented by
* a TimeUUID that can be used to look up both the Migration itself (see getLocalMigrations) as well as a serialization
* of the Keyspace definition that was modified.
* A migration represents a single metadata mutation (cf dropped, added, etc.).
*
* There are two parts to a migration (think of it as a schema update):
* 1. data is written to the schema cf (SCHEMA_KEYSPACES_CF).
* 2. updated models are applied to the cassandra instance.
*
* There are three parts to a migration (think of it as a schema update):
* 1. data is written to the schema cf.
* 2. the migration is serialized to the migrations cf.
* 3. updated models are applied to the cassandra instance.
*
* Since steps 1, 2 and 3 are not committed atomically, care should be taken to ensure that a node/cluster is reasonably
* quiescent with regard to the keyspace or columnfamily whose schema is being modified.
*
* Each class that extends Migration is required to implement a no arg constructor, which will be used to inflate the
* object from it's serialized form.
* Since all steps are not committed atomically, care should be taken to ensure that a node/cluster is reasonably
* quiescent with regard to the Keyspace or ColumnFamily whose schema is being modified.
*/
public abstract class Migration
{
@ -69,107 +52,52 @@ public abstract class Migration
public static final String NAME_VALIDATOR_REGEX = "\\w+";
public static final String MIGRATIONS_CF = "Migrations";
public static final String SCHEMA_CF = "Schema";
public static final ByteBuffer MIGRATIONS_KEY = ByteBufferUtil.bytes("Migrations Key");
public static final ByteBuffer LAST_MIGRATION_KEY = ByteBufferUtil.bytes("Last Migration");
protected RowMutation rm;
protected UUID newVersion;
protected UUID lastVersion;
// the migration in column form, used when announcing to others
private IColumn column;
protected final long timestamp;
// cast of the schema at the time when migration was initialized
protected final Schema schema = Schema.instance;
/** Subclasses must have a matching constructor */
protected Migration() { }
Migration(UUID newVersion, UUID lastVersion)
Migration(long modificationTimestamp)
{
this();
this.newVersion = newVersion;
this.lastVersion = lastVersion;
timestamp = modificationTimestamp;
}
/** apply changes */
public final void apply() throws IOException, ConfigurationException
public final void apply() throws ConfigurationException, IOException
{
// ensure migration is serial. don't apply unless the previous version matches.
if (!schema.getVersion().equals(lastVersion))
throw new ConfigurationException("Previous version mismatch. cannot apply.");
if (newVersion.timestamp() <= lastVersion.timestamp())
throw new ConfigurationException("New version timestamp is not newer than the current version timestamp.");
// write to schema
assert rm != null;
applyImpl();
if (!StorageService.instance.isClientMode())
{
rm.apply();
MigrationHelper.flushSchemaCFs();
long now = System.currentTimeMillis();
ByteBuffer buf = serialize();
RowMutation migration = new RowMutation(Table.SYSTEM_TABLE, MIGRATIONS_KEY);
ColumnFamily cf = ColumnFamily.create(Table.SYSTEM_TABLE, MIGRATIONS_CF);
column = new Column(ByteBuffer.wrap(UUIDGen.decompose(newVersion)), buf, now);
cf.addColumn(column);
migration.add(cf);
migration.apply();
// note that we're storing this in the system table, which is not replicated
logger.info("Applying migration {} {}", newVersion.toString(), toString());
migration = new RowMutation(Table.SYSTEM_TABLE, LAST_MIGRATION_KEY);
migration.add(new QueryPath(SCHEMA_CF, null, LAST_MIGRATION_KEY), ByteBuffer.wrap(UUIDGen.decompose(newVersion)), now);
migration.apply();
// if we fail here, there will be schema changes in the CL that will get replayed *AFTER* the schema is loaded.
// CassandraDaemon checks for this condition (the stored version will be greater than the loaded version)
// and calls MigrationManager.applyMigrations(loaded version, stored version).
// flush changes out of memtables so we don't need to rely on the commit log.
ColumnFamilyStore[] schemaStores = new ColumnFamilyStore[] {
Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(Migration.MIGRATIONS_CF),
Table.open(Table.SYSTEM_TABLE).getColumnFamilyStore(Migration.SCHEMA_CF)
};
List<Future<?>> flushes = new ArrayList<Future<?>>();
for (ColumnFamilyStore cfs : schemaStores)
flushes.add(cfs.forceFlush());
for (Future<?> f : flushes)
{
if (f == null)
// applying the migration triggered a flush independently
continue;
try
{
f.get();
}
catch (ExecutionException e)
{
throw new IOException(e);
}
catch (InterruptedException e)
{
throw new IOException(e);
}
}
}
applyModels();
Schema.instance.updateVersion();
}
/** send this migration immediately to existing nodes in the cluster. apply() must be called first. */
/**
* Class specific apply implementation where schema migration logic should be put
*
* @throws IOException on any I/O related error.
* @throws ConfigurationException if there is object misconfiguration.
*/
protected abstract void applyImpl() throws ConfigurationException, IOException;
/** Send schema update (in form of row mutations) to alive nodes in the cluster. apply() must be called first. */
public final void announce()
{
assert !StorageService.instance.isClientMode();
assert column != null;
MigrationManager.announce(column);
MigrationManager.announce(SystemTable.serializeSchema());
passiveAnnounce(); // keeps gossip in sync w/ what we just told everyone
}
/** Announce new schema version over Gossip */
public final void passiveAnnounce()
{
MigrationManager.passiveAnnounce(newVersion);
MigrationManager.passiveAnnounce(Schema.instance.getVersion());
}
/**
* Used only in case node has old style migration schema (newly updated)
* @return the UUID identifying version of the last applied migration
*/
@Deprecated
public static UUID getLastMigrationId()
{
DecoratedKey<?> dkey = StorageService.getPartitioner().decorateKey(LAST_MIGRATION_KEY);
@ -183,137 +111,6 @@ public abstract class Migration
return UUIDGen.getUUID(cf.getColumn(LAST_MIGRATION_KEY).value());
}
/** keep in mind that applyLive might happen on another machine */
abstract void applyModels() throws IOException;
/** Deflate this Migration into an Avro object. */
public abstract void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi);
/** Inflate this Migration from an Avro object: called after the required no-arg constructor. */
public abstract void subinflate(org.apache.cassandra.db.migration.avro.Migration mi);
public UUID getVersion()
{
return newVersion;
}
/**
* Definitions are serialized as a row with a UUID key, with a magical column named DEFINITION_SCHEMA_COLUMN_NAME
* (containing the Avro Schema) and a column per keyspace. Each keyspace column contains a avro.KsDef object
* encoded with the Avro schema.
*/
final RowMutation makeDefinitionMutation(KSMetaData add, KSMetaData remove, UUID versionId) throws IOException
{
// collect all keyspaces, while removing 'remove' and adding 'add'
List<KSMetaData> ksms = new ArrayList<KSMetaData>();
for (String tableName : schema.getNonSystemTables())
{
if (remove != null && remove.name.equals(tableName) || add != null && add.name.equals(tableName))
continue;
ksms.add(schema.getTableDefinition(tableName));
}
if (add != null)
ksms.add(add);
// wrap in mutation
RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, toUTF8Bytes(versionId));
long now = System.currentTimeMillis();
// add a column for each keyspace
for (KSMetaData ksm : ksms)
rm.add(new QueryPath(SCHEMA_CF, null, ByteBufferUtil.bytes(ksm.name)), SerDeUtils.serialize(ksm.toAvro()), now);
// add the schema
rm.add(new QueryPath(SCHEMA_CF,
null,
DefsTable.DEFINITION_SCHEMA_COLUMN_NAME),
ByteBufferUtil.bytes(org.apache.cassandra.db.migration.avro.KsDef.SCHEMA$.toString()),
now);
return rm;
}
public ByteBuffer serialize() throws IOException
{
// super deflate
org.apache.cassandra.db.migration.avro.Migration mi = new org.apache.cassandra.db.migration.avro.Migration();
mi.old_version = new org.apache.cassandra.utils.avro.UUID();
mi.old_version.bytes(UUIDGen.decompose(lastVersion));
mi.new_version = new org.apache.cassandra.utils.avro.UUID();
mi.new_version.bytes(UUIDGen.decompose(newVersion));
mi.classname = new org.apache.avro.util.Utf8(this.getClass().getName());
// TODO: Avro RowMutation serialization?
DataOutputBuffer dob = new DataOutputBuffer();
try
{
RowMutation.serializer().serialize(rm, dob, MessagingService.version_);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
mi.row_mutation = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
// sub deflate
this.subdeflate(mi);
// serialize
return SerDeUtils.serializeWithSchema(mi);
}
public static Migration deserialize(ByteBuffer bytes, int version) throws IOException
{
// deserialize
org.apache.cassandra.db.migration.avro.Migration mi = SerDeUtils.deserializeWithSchema(bytes, new org.apache.cassandra.db.migration.avro.Migration());
// create an instance of the migration subclass
Migration migration;
try
{
Class<?> migrationClass = Class.forName(mi.classname.toString());
Constructor<?> migrationConstructor = migrationClass.getDeclaredConstructor();
migrationConstructor.setAccessible(true);
migration = (Migration)migrationConstructor.newInstance();
}
catch (Exception e)
{
throw new RuntimeException("Invalid migration class: " + mi.classname.toString(), e);
}
// super inflate
migration.lastVersion = UUIDGen.getUUID(ByteBuffer.wrap(mi.old_version.bytes()));
migration.newVersion = UUIDGen.getUUID(ByteBuffer.wrap(mi.new_version.bytes()));
try
{
migration.rm = RowMutation.serializer().deserialize(SerDeUtils.createDataInputStream(mi.row_mutation), version);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
// sub inflate
migration.subinflate(mi);
return migration;
}
/** load serialized migrations. */
public static Collection<IColumn> getLocalMigrations(UUID start, UUID end)
{
DecoratedKey<?> dkey = StorageService.getPartitioner().decorateKey(MIGRATIONS_KEY);
Table defs = Table.open(Table.SYSTEM_TABLE);
ColumnFamilyStore cfStore = defs.getColumnFamilyStore(Migration.MIGRATIONS_CF);
QueryFilter filter = QueryFilter.getSliceFilter(dkey,
new QueryPath(MIGRATIONS_CF),
ByteBuffer.wrap(UUIDGen.decompose(start)),
ByteBuffer.wrap(UUIDGen.decompose(end)),
false,
100);
ColumnFamily cf = cfStore.getColumnFamily(filter);
return cf.getSortedColumns();
}
public static ByteBuffer toUTF8Bytes(UUID version)
{
return ByteBufferUtil.bytes(version.toString());
}
public static boolean isLegalName(String s)
{
return s.matches(Migration.NAME_VALIDATOR_REGEX);

371
src/java/org/apache/cassandra/db/migration/MigrationHelper.java Normal file
View File

@ -0,0 +1,371 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.migration;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Future;
import com.google.common.collect.Iterables;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.RowMutation;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.thrift.KsDef;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.codehaus.jackson.map.ObjectMapper;
public class MigrationHelper
{
private static final ObjectMapper jsonMapper = new ObjectMapper();
public static ByteBuffer readableColumnName(ByteBuffer columnName, AbstractType comparator)
{
return ByteBufferUtil.bytes(comparator.getString(columnName));
}
public static ByteBuffer valueAsBytes(Object value)
{
try
{
return ByteBuffer.wrap(jsonMapper.writeValueAsBytes(value));
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public static Object deserializeValue(ByteBuffer value, Class<?> valueClass)
{
try
{
// because jackson serialized ByteBuffer as byte[] and needs help with deserialization later
if (valueClass.equals(ByteBuffer.class))
return ByteBuffer.wrap((byte[]) deserializeValue(value, byte[].class));
return jsonMapper.readValue(ByteBufferUtil.getArray(value), valueClass);
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
public static Class<?> getValueClass(Class<?> klass, String name)
{
try
{
return klass.getField(name).getType();
}
catch (NoSuchFieldException e)
{
throw new RuntimeException(e); // never happens
}
}
public static ByteBuffer searchComposite(String comp1, boolean start)
{
return compositeNameFor(comp1, !start, null, false, null, false);
}
public static ByteBuffer compositeNameFor(String comp1, String comp2)
{
return compositeNameFor(comp1, ByteBufferUtil.bytes(comp2), null);
}
public static ByteBuffer compositeNameFor(String comp1, ByteBuffer comp2, String comp3)
{
return compositeNameFor(comp1, false, comp2, false, comp3, false);
}
public static ByteBuffer compositeNameFor(String comp1, boolean limit1, ByteBuffer comp2, boolean limit2, String comp3, boolean limit3)
{
int totalSize = 0;
if (comp1 != null)
totalSize += 2 + comp1.length() + 1;
if (comp2 != null)
totalSize += 2 + comp2.remaining() + 1;
if (comp3 != null)
totalSize += 2 + comp3.length() + 1;
ByteBuffer bytes = ByteBuffer.allocate(totalSize);
if (comp1 != null)
{
bytes.putShort((short) comp1.length());
bytes.put(comp1.getBytes());
bytes.put((byte) (limit1 ? 1 : 0));
}
if (comp2 != null)
{
int pos = comp2.position(), limit = comp2.limit();
bytes.putShort((short) comp2.remaining());
bytes.put(comp2);
bytes.put((byte) (limit2 ? 1 : 0));
// restore original range
comp2.position(pos).limit(limit);
}
if (comp3 != null)
{
bytes.putShort((short) comp3.length());
bytes.put(comp3.getBytes());
bytes.put((byte) (limit3 ? 1 : 0));
}
bytes.rewind();
return bytes;
}
public static void flushSchemaCFs()
{
flushSchemaCF(SystemTable.SCHEMA_KEYSPACES_CF);
flushSchemaCF(SystemTable.SCHEMA_COLUMNFAMILIES_CF);
flushSchemaCF(SystemTable.SCHEMA_COLUMNS_CF);
}
public static void flushSchemaCF(String cfName)
{
Future<?> flush = SystemTable.schemaCFS(cfName).forceFlush();
if (flush != null)
FBUtilities.waitOnFuture(flush);
}
/* Schema Mutation Helpers */
public static void addKeyspace(KSMetaData ksm, long timestamp) throws ConfigurationException, IOException
{
addKeyspace(ksm, timestamp, true);
}
public static void addKeyspace(KSMetaData ksDef) throws ConfigurationException, IOException
{
addKeyspace(ksDef, -1, false);
}
public static void addColumnFamily(CFMetaData cfm, long timestamp) throws ConfigurationException, IOException
{
addColumnFamily(cfm, timestamp, true);
}
public static void addColumnFamily(CfDef cfDef) throws ConfigurationException, IOException
{
try
{
addColumnFamily(CFMetaData.fromThrift(cfDef), -1, false);
}
catch (InvalidRequestException e)
{
throw new ConfigurationException(e.getMessage(), e);
}
}
public static void updateKeyspace(KsDef newState) throws ConfigurationException, IOException
{
updateKeyspace(newState, -1, false);
}
public static void updateKeyspace(KsDef newState, long timestamp) throws ConfigurationException, IOException
{
updateKeyspace(newState, timestamp, true);
}
public static void updateColumnFamily(CfDef newState) throws ConfigurationException, IOException
{
updateColumnFamily(newState, -1, false);
}
public static void updateColumnFamily(CfDef newState, long timestamp) throws ConfigurationException, IOException
{
updateColumnFamily(newState, timestamp, true);
}
public static void dropColumnFamily(String ksName, String cfName) throws IOException
{
dropColumnFamily(ksName, cfName, -1, false);
}
public static void dropColumnFamily(String ksName, String cfName, long timestamp) throws IOException
{
dropColumnFamily(ksName, cfName, timestamp, true);
}
public static void dropKeyspace(String ksName) throws IOException
{
dropKeyspace(ksName, -1, false);
}
public static void dropKeyspace(String ksName, long timestamp) throws IOException
{
dropKeyspace(ksName, timestamp, true);
}
/* Migration Helper implementations */
private static void addKeyspace(KSMetaData ksm, long timestamp, boolean withSchemaRecord) throws ConfigurationException, IOException
{
RowMutation keyspaceDef = ksm.toSchema(timestamp);
if (withSchemaRecord)
keyspaceDef.apply();
Schema.instance.load(ksm);
if (!StorageService.instance.isClientMode())
Table.open(ksm.name);
}
private static void addColumnFamily(CFMetaData cfm, long timestamp, boolean withSchemaRecord) throws ConfigurationException, IOException
{
KSMetaData ksm = Schema.instance.getTableDefinition(cfm.ksName);
ksm = KSMetaData.cloneWith(ksm, Iterables.concat(ksm.cfMetaData().values(), Collections.singleton(cfm)));
Schema.instance.load(cfm);
if (withSchemaRecord)
cfm.toSchema(timestamp).apply();
// make sure it's init-ed w/ the old definitions first,
// since we're going to call initCf on the new one manually
Table.open(cfm.ksName);
Schema.instance.setTableDefinition(ksm);
if (!StorageService.instance.isClientMode())
Table.open(ksm.name).initCf(cfm.cfId, cfm.cfName);
}
private static void updateKeyspace(KsDef newState, long timestamp, boolean withSchemaRecord) throws ConfigurationException, IOException
{
KSMetaData oldKsm = Schema.instance.getKSMetaData(newState.name);
if (withSchemaRecord)
{
RowMutation schemaUpdate = oldKsm.diff(newState, timestamp);
schemaUpdate.apply();
}
KSMetaData newKsm = KSMetaData.cloneWith(oldKsm.reloadAttributes(), oldKsm.cfMetaData().values());
Schema.instance.setTableDefinition(newKsm);
if (!StorageService.instance.isClientMode())
Table.open(newState.name).createReplicationStrategy(newKsm);
}
private static void updateColumnFamily(CfDef newState, long timestamp, boolean withSchemaRecord) throws ConfigurationException, IOException
{
CFMetaData cfm = Schema.instance.getCFMetaData(newState.keyspace, newState.name);
if (withSchemaRecord)
{
RowMutation schemaUpdate = cfm.diff(newState, timestamp);
schemaUpdate.apply();
}
cfm.reload();
if (!StorageService.instance.isClientMode())
{
Table table = Table.open(cfm.ksName);
table.getColumnFamilyStore(cfm.cfName).reload();
}
}
private static void dropKeyspace(String ksName, long timestamp, boolean withSchemaRecord) throws IOException
{
KSMetaData ksm = Schema.instance.getTableDefinition(ksName);
String snapshotName = Table.getTimestampedSnapshotName(ksName);
// remove all cfs from the table instance.
for (CFMetaData cfm : ksm.cfMetaData().values())
{
ColumnFamilyStore cfs = Table.open(ksm.name).getColumnFamilyStore(cfm.cfName);
Schema.instance.purge(cfm);
if (!StorageService.instance.isClientMode())
{
cfs.snapshot(snapshotName);
Table.open(ksm.name).dropCf(cfm.cfId);
}
}
if (withSchemaRecord)
{
for (RowMutation m : ksm.dropFromSchema(timestamp))
m.apply();
}
// remove the table from the static instances.
Table.clear(ksm.name);
Schema.instance.clearTableDefinition(ksm);
}
private static void dropColumnFamily(String ksName, String cfName, long timestamp, boolean withSchemaRecord) throws IOException
{
KSMetaData ksm = Schema.instance.getTableDefinition(ksName);
ColumnFamilyStore cfs = Table.open(ksName).getColumnFamilyStore(cfName);
// reinitialize the table.
CFMetaData cfm = ksm.cfMetaData().get(cfName);
Schema.instance.purge(cfm);
Schema.instance.setTableDefinition(makeNewKeyspaceDefinition(ksm, cfm));
if (withSchemaRecord)
cfm.dropFromSchema(timestamp).apply();
if (!StorageService.instance.isClientMode())
{
cfs.snapshot(Table.getTimestampedSnapshotName(cfs.columnFamily));
Table.open(ksm.name).dropCf(cfm.cfId);
}
}
private static KSMetaData makeNewKeyspaceDefinition(KSMetaData ksm, CFMetaData toExclude)
{
// clone ksm but do not include the new def
List<CFMetaData> newCfs = new ArrayList<CFMetaData>(ksm.cfMetaData().values());
newCfs.remove(toExclude);
assert newCfs.size() == ksm.cfMetaData().size() - 1;
return KSMetaData.cloneWith(ksm, newCfs);
}
}

95
src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java
View File

@ -1,15 +1,3 @@
package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.migration.avro.ColumnDef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -27,81 +15,36 @@ import org.apache.cassandra.utils.UUIDGen;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.thrift.CfDef;
public class UpdateColumnFamily extends Migration
{
// does not point to a CFM stored in DatabaseDescriptor.
private CFMetaData metadata;
protected UpdateColumnFamily() { }
/** assumes validation has already happened. That is, replacing oldCfm with newCfm is neither illegal or totally whackass. */
public UpdateColumnFamily(org.apache.cassandra.db.migration.avro.CfDef cf_def) throws ConfigurationException, IOException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
KSMetaData ksm = schema.getTableDefinition(cf_def.keyspace.toString());
if (ksm == null)
throw new ConfigurationException("No such keyspace: " + cf_def.keyspace.toString());
if (cf_def.column_metadata != null)
{
for (ColumnDef entry : cf_def.column_metadata)
{
if (entry.index_name != null && !Migration.isLegalName(entry.index_name.toString()))
throw new ConfigurationException("Invalid index name: " + entry.index_name);
}
}
private final CfDef newState;
CFMetaData oldCfm = schema.getCFMetaData(cf_def.keyspace.toString(), cf_def.name.toString());
// create a copy of the old CF meta data. Apply new settings on top of it.
this.metadata = CFMetaData.fromAvro(oldCfm.toAvro());
this.metadata.apply(cf_def);
// create a copy of the old KS meta data. Use it to create a RowMutation that gets applied to schema and migrations.
KSMetaData newKsMeta = KSMetaData.fromAvro(ksm.toAvro());
newKsMeta.cfMetaData().get(cf_def.name.toString()).apply(cf_def);
rm = makeDefinitionMutation(newKsMeta, null, newVersion);
public UpdateColumnFamily(CfDef newState) throws ConfigurationException
{
super(System.nanoTime());
if (Schema.instance.getCFMetaData(newState.keyspace, newState.name) == null)
throw new ConfigurationException(String.format("(ks=%s, cf=%s) cannot be updated because it doesn't exist.", newState.keyspace, newState.name));
this.newState = newState;
}
void applyModels() throws IOException
protected void applyImpl() throws ConfigurationException, IOException
{
logger.debug("Updating " + schema.getCFMetaData(metadata.cfId) + " to " + metadata);
// apply the meta update.
try
{
schema.getCFMetaData(metadata.cfId).apply(metadata.toAvro());
}
catch (ConfigurationException ex)
{
throw new IOException(ex);
}
schema.setTableDefinition(null, newVersion);
if (!StorageService.instance.isClientMode())
{
Table table = Table.open(metadata.ksName, schema);
ColumnFamilyStore oldCfs = table.getColumnFamilyStore(metadata.cfName);
oldCfs.reload();
}
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.UpdateColumnFamily update = new org.apache.cassandra.db.migration.avro.UpdateColumnFamily();
update.metadata = metadata.toAvro();
mi.migration = update;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.UpdateColumnFamily update = (org.apache.cassandra.db.migration.avro.UpdateColumnFamily)mi.migration;
metadata = CFMetaData.fromAvro(update.metadata);
MigrationHelper.updateColumnFamily(newState, timestamp);
}
@Override
public String toString()
{
return String.format("Update column family to %s", metadata.toString());
return String.format("Update column family with %s", newState);
}
}

87
src/java/org/apache/cassandra/db/migration/UpdateKeyspace.java
View File

@ -1,12 +1,3 @@
package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.*;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -15,80 +6,50 @@ import org.apache.cassandra.utils.UUIDGen;
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db.migration;
import java.io.IOException;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.thrift.KsDef;
public class UpdateKeyspace extends Migration
{
private KSMetaData newKsm;
private KSMetaData oldKsm;
/** Required no-arg constructor */
protected UpdateKeyspace() { }
/** create migration based on thrift parameters */
public UpdateKeyspace(KSMetaData ksm) throws ConfigurationException, IOException
{
super(UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()), Schema.instance.getVersion());
assert ksm != null;
assert ksm.cfMetaData() != null;
if (ksm.cfMetaData().size() > 0)
throw new ConfigurationException("Updated keyspace must not contain any column families");
// create the new ksm by merging the one passed in with the cf defs from the exisitng ksm.
oldKsm = schema.getKSMetaData(ksm.name);
if (oldKsm == null)
throw new ConfigurationException(ksm.name + " cannot be updated because it doesn't exist.");
private final KsDef newState;
this.newKsm = KSMetaData.cloneWith(ksm, oldKsm.cfMetaData().values());
rm = makeDefinitionMutation(newKsm, oldKsm, newVersion);
public UpdateKeyspace(KsDef newState) throws ConfigurationException
{
super(System.nanoTime());
if (newState.isSetCf_defs() && newState.getCf_defs().size() > 0)
throw new ConfigurationException("Updated keyspace must not contain any column families.");
if (Schema.instance.getKSMetaData(newState.name) == null)
throw new ConfigurationException(newState.name + " cannot be updated because it doesn't exist.");
this.newState = newState;
}
void applyModels() throws IOException
{
schema.clearTableDefinition(oldKsm, newVersion);
schema.setTableDefinition(newKsm, newVersion);
Table table = Table.open(newKsm.name, schema);
try
{
table.createReplicationStrategy(newKsm);
}
catch (ConfigurationException e)
{
throw new IOException(e);
}
protected void applyImpl() throws ConfigurationException, IOException
{
MigrationHelper.updateKeyspace(newState, timestamp);
logger.info("Keyspace updated. Please perform any manual operations.");
}
public void subdeflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.UpdateKeyspace uks = new org.apache.cassandra.db.migration.avro.UpdateKeyspace();
uks.newKs = newKsm.toAvro();
uks.oldKs = oldKsm.toAvro();
mi.migration = uks;
}
public void subinflate(org.apache.cassandra.db.migration.avro.Migration mi)
{
org.apache.cassandra.db.migration.avro.UpdateKeyspace uks = (org.apache.cassandra.db.migration.avro.UpdateKeyspace)mi.migration;
newKsm = KSMetaData.fromAvro(uks.newKs);
oldKsm = KSMetaData.fromAvro(uks.oldKs);
}
@Override
public String toString()
{
return String.format("Update keyspace %s to %s", oldKsm.toString(), newKsm.toString());
return String.format("Update keyspace %s with %s", newState.name, newState);
}
}

124
src/java/org/apache/cassandra/io/SerDeUtils.java
View File

@ -1,124 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.ipc.ByteBufferInputStream;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.specific.SpecificRecord;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.io.util.OutputBuffer;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
* Static serialization/deserialization utility functions, intended to eventually replace IVersionedSerializers.
*/
public final class SerDeUtils
{
// unbuffered decoders
private final static DecoderFactory DIRECT_DECODERS = new DecoderFactory().configureDirectDecoder(true);
/**
* Deserializes a single object based on the given Schema.
* @param writer writer's schema
* @param bytes Array to deserialize from
* @param ob An empty object to deserialize into (must not be null).
* @throws IOException
*/
public static <T extends SpecificRecord> T deserialize(Schema writer, ByteBuffer bytes, T ob) throws IOException
{
BinaryDecoder dec = DIRECT_DECODERS.createBinaryDecoder(ByteBufferUtil.getArray(bytes), null);
SpecificDatumReader<T> reader = new SpecificDatumReader<T>(writer);
reader.setExpected(ob.getSchema());
return reader.read(ob, dec);
}
/**
* Serializes a single object.
* @param o Object to serialize
*/
public static <T extends SpecificRecord> ByteBuffer serialize(T o) throws IOException
{
OutputBuffer buff = new OutputBuffer();
BinaryEncoder enc = new BinaryEncoder(buff);
SpecificDatumWriter<T> writer = new SpecificDatumWriter<T>(o.getSchema());
writer.write(o, enc);
enc.flush();
return ByteBuffer.wrap(buff.asByteArray());
}
/**
* Deserializes a single object as stored along with its Schema by serialize(T). NB: See warnings on serialize(T).
* @param ob An empty object to deserialize into (must not be null).
* @param bytes Array to deserialize from
* @throws IOException
*/
public static <T extends SpecificRecord> T deserializeWithSchema(ByteBuffer bytes, T ob) throws IOException
{
BinaryDecoder dec = DIRECT_DECODERS.createBinaryDecoder(ByteBufferUtil.getArray(bytes), null);
Schema writer = Schema.parse(dec.readString(new Utf8()).toString());
SpecificDatumReader<T> reader = new SpecificDatumReader<T>(writer);
reader.setExpected(ob.getSchema());
return reader.read(ob, dec);
}
/**
* Serializes a single object along with its Schema. NB: For performance critical areas, it is <b>much</b>
* more efficient to store the Schema independently.
* @param o Object to serialize
*/
public static <T extends SpecificRecord> ByteBuffer serializeWithSchema(T o) throws IOException
{
OutputBuffer buff = new OutputBuffer();
BinaryEncoder enc = new BinaryEncoder(buff);
enc.writeString(new Utf8(o.getSchema().toString()));
SpecificDatumWriter<T> writer = new SpecificDatumWriter<T>(o.getSchema());
writer.write(o, enc);
enc.flush();
return ByteBuffer.wrap(buff.asByteArray());
}
/**
* @return a DataInputStream wrapping the given buffer.
*/
public static DataInputStream createDataInputStream(ByteBuffer buff)
{
ByteBufferInputStream bbis = new ByteBufferInputStream(Collections.singletonList(buff));
return new DataInputStream(bbis);
}
/**
* Create a generic array of the given type and size. Mostly to minimize imports.
*/
public static <T> GenericArray<T> createArray(int size, Schema schema)
{
return new GenericData.Array<T>(size, Schema.createArray(schema));
}
}

14
src/java/org/apache/cassandra/service/AbstractCassandraDaemon.java
View File

@ -24,7 +24,6 @@ import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.UUID;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ -46,8 +45,6 @@ import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.SystemTable;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.utils.CLibrary;
import org.apache.cassandra.utils.Mx4jTool;
@ -225,17 +222,6 @@ public abstract class AbstractCassandraDaemon implements CassandraDaemon
// replay the log if necessary
CommitLog.instance.recover();
// check to see if CL.recovery modified the lastMigrationId. if it did, we need to re apply migrations. this isn't
// the same as merely reloading the schema (which wouldn't perform file deletion after a DROP). The solution
// is to read those migrations from disk and apply them.
UUID currentMigration = Schema.instance.getVersion();
UUID lastMigration = Migration.getLastMigrationId();
if ((lastMigration != null) && (lastMigration.timestamp() > currentMigration.timestamp()))
{
Gossiper.instance.maybeInitializeLocalState(SystemTable.incrementAndGetGeneration());
MigrationManager.applyMigrations(currentMigration, lastMigration);
}
SystemTable.finishStartup();
// start server internals

304
src/java/org/apache/cassandra/service/MigrationManager.java
View File

@ -18,120 +18,130 @@
package org.apache.cassandra.service;
import java.io.*;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.UUID;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.collect.Iterables;
import com.google.common.collect.MapMaker;
import org.apache.cassandra.config.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.Column;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.db.marshal.TimeUUIDType;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.db.DefsTable;
import org.apache.cassandra.db.RowMutation;
import org.apache.cassandra.gms.*;
import org.apache.cassandra.io.util.FastByteArrayInputStream;
import org.apache.cassandra.io.util.FastByteArrayOutputStream;
import org.apache.cassandra.net.IAsyncResult;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.commons.lang.ArrayUtils;
public class MigrationManager implements IEndpointStateChangeSubscriber
{
private static final Logger logger = LoggerFactory.getLogger(MigrationManager.class);
// avoids re-pushing migrations that we're waiting on target to apply already
private static Map<InetAddress,UUID> lastPushed = new MapMaker().expiration(1, TimeUnit.MINUTES).makeMap();
private static UUID highestKnown;
// try that many times to send migration request to the node before giving up
private static final int MIGRATION_REQUEST_RETRIES = 3;
public void onJoin(InetAddress endpoint, EndpointState epState) {
VersionedValue value = epState.getApplicationState(ApplicationState.SCHEMA);
if (value != null)
{
UUID theirVersion = UUID.fromString(value.value);
rectify(theirVersion, endpoint);
}
}
public void onJoin(InetAddress endpoint, EndpointState epState)
{}
public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)
{
if (state != ApplicationState.SCHEMA)
if (state != ApplicationState.SCHEMA || endpoint.equals(FBUtilities.getBroadcastAddress()))
return;
UUID theirVersion = UUID.fromString(value.value);
rectify(theirVersion, endpoint);
rectifySchema(UUID.fromString(value.value), endpoint);
}
/** gets called after a this node joins a cluster */
public void onAlive(InetAddress endpoint, EndpointState state)
{
{
VersionedValue value = state.getApplicationState(ApplicationState.SCHEMA);
if (value != null)
{
UUID theirVersion = UUID.fromString(value.value);
rectify(theirVersion, endpoint);
}
rectifySchema(UUID.fromString(value.value), endpoint);
}
public void onDead(InetAddress endpoint, EndpointState state) { }
public void onDead(InetAddress endpoint, EndpointState state)
{}
public void onRestart(InetAddress endpoint, EndpointState state) { }
public void onRestart(InetAddress endpoint, EndpointState state)
{}
public void onRemove(InetAddress endpoint) { }
/**
* will either push or pull an updating depending on who is behind.
* fat clients should never push their schemas (since they have no local storage).
*/
public static void rectify(UUID theirVersion, InetAddress endpoint)
public void onRemove(InetAddress endpoint)
{}
public static void rectifySchema(UUID theirVersion, final InetAddress endpoint)
{
updateHighestKnown(theirVersion);
UUID myVersion = Schema.instance.getVersion();
if (theirVersion.timestamp() < myVersion.timestamp()
&& !StorageService.instance.isClientMode())
if (Schema.instance.getVersion().equals(theirVersion))
return;
/**
* if versions differ this node sends request with local migration list to the endpoint
* and expecting to receive a list of migrations to apply locally
*/
Future f = StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
{
if (lastPushed.get(endpoint) == null || theirVersion.timestamp() >= lastPushed.get(endpoint).timestamp())
public void runMayThrow() throws Exception
{
logger.debug("Schema on {} is old. Sending updates since {}", endpoint, theirVersion);
Collection<IColumn> migrations = Migration.getLocalMigrations(theirVersion, myVersion);
pushMigrations(endpoint, migrations);
lastPushed.put(endpoint, TimeUUIDType.instance.compose(Iterables.getLast(migrations).name()));
Message message = new Message(FBUtilities.getBroadcastAddress(),
StorageService.Verb.MIGRATION_REQUEST,
ArrayUtils.EMPTY_BYTE_ARRAY,
Gossiper.instance.getVersion(endpoint));
int retries = 0;
while (retries < MIGRATION_REQUEST_RETRIES)
{
if (!FailureDetector.instance.isAlive(endpoint))
{
logger.error("Can't send migration request: node {} is down.", endpoint);
return;
}
IAsyncResult iar = MessagingService.instance().sendRR(message, endpoint);
try
{
byte[] reply = iar.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS);
DefsTable.mergeRemoteSchema(reply, message.getVersion());
return;
}
catch(TimeoutException e)
{
retries++;
}
}
}
else
{
logger.debug("Waiting for {} to process migrations up to {} before sending more",
endpoint, lastPushed.get(endpoint));
}
}
}
private static void updateHighestKnown(UUID theirversion)
{
if (highestKnown == null || theirversion.timestamp() > highestKnown.timestamp())
highestKnown = theirversion;
});
FBUtilities.waitOnFuture(f);
}
public static boolean isReadyForBootstrap()
{
return highestKnown.compareTo(Schema.instance.getVersion()) == 0;
return StageManager.getStage(Stage.MIGRATION).getActiveCount() == 0;
}
private static void pushMigrations(InetAddress endpoint, Collection<IColumn> migrations)
private static void pushSchemaMutation(InetAddress endpoint, Collection<RowMutation> schema)
{
try
{
Message msg = makeMigrationMessage(migrations, Gossiper.instance.getVersion(endpoint));
Message msg = makeMigrationMessage(schema, Gossiper.instance.getVersion(endpoint));
MessagingService.instance().sendOneWay(msg, endpoint);
}
catch (IOException ex)
@ -140,118 +150,96 @@ public class MigrationManager implements IEndpointStateChangeSubscriber
}
}
/** actively announce a new version to active hosts via rpc */
public static void announce(IColumn column)
/**
* actively announce a new version to active hosts via rpc
* @param schema The list of schema mutations to be applied on the recipient
*/
public static void announce(Collection<RowMutation> schema)
{
Collection<IColumn> migrations = Collections.singleton(column);
for (InetAddress endpoint : Gossiper.instance.getLiveMembers())
pushMigrations(endpoint, migrations);
{
if (endpoint.equals(FBUtilities.getBroadcastAddress()))
continue; // don't push schema mutation to self
pushSchemaMutation(endpoint, schema);
}
}
/** announce my version passively over gossip **/
/**
* Announce my version passively over gossip.
* Used to notify nodes as they arrive in the cluster.
*
* @param version The schema version to announce
*/
public static void passiveAnnounce(UUID version)
{
// this is for notifying nodes as they arrive in the cluster.
Gossiper.instance.addLocalApplicationState(ApplicationState.SCHEMA, StorageService.instance.valueFactory.migration(version));
logger.debug("Gossiping my schema version " + version);
}
/**
* gets called during startup if we notice a mismatch between the current migration version and the one saved. This
* can only happen as a result of the commit log recovering schema updates, which overwrites lastVersionId.
*
* This method silently eats IOExceptions thrown by Migration.apply() as a result of applying a migration out of
* order.
* Serialize given row mutations into raw bytes and make a migration message
* (other half of transformation is in DefinitionsUpdateResponseVerbHandler.)
*
* @param schema The row mutations to send to remote nodes
* @param version The version to use for message
*
* @return Serialized migration containing schema mutations
*
* @throws IOException on failed serialization
*/
public static void applyMigrations(final UUID from, final UUID to) throws IOException
private static Message makeMigrationMessage(Collection<RowMutation> schema, int version) throws IOException
{
List<Future<?>> updates = new ArrayList<Future<?>>();
Collection<IColumn> migrations = Migration.getLocalMigrations(from, to);
for (IColumn col : migrations)
{
// assuming MessagingService.version_ is a bit of a risk, but you're playing with fire if you purposefully
// take down a node to upgrade it during the middle of a schema update.
final Migration migration = Migration.deserialize(col.value(), MessagingService.version_);
Future<?> update = StageManager.getStage(Stage.MIGRATION).submit(new Runnable()
{
public void run()
{
try
{
migration.apply();
}
catch (ConfigurationException ex)
{
// this happens if we try to apply something that's already been applied. ignore and proceed.
logger.debug("Migration not applied " + ex.getMessage());
}
catch (IOException ex)
{
throw new RuntimeException(ex);
}
}
});
updates.add(update);
}
// wait on all the updates before proceeding.
for (Future<?> f : updates)
{
try
{
f.get();
}
catch (InterruptedException e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
}
passiveAnnounce(to); // we don't need to send rpcs, but we need to update gossip
return new Message(FBUtilities.getBroadcastAddress(), StorageService.Verb.DEFINITIONS_UPDATE, serializeSchema(schema, version), version);
}
// other half of transformation is in DefinitionsUpdateResponseVerbHandler.
private static Message makeMigrationMessage(Collection<IColumn> migrations, int version) throws IOException
/**
* Serialize given row mutations into raw bytes
*
* @param schema The row mutations to serialize
* @param version The version of the message service to use for serialization
*
* @return serialized mutations
*
* @throws IOException on failed serialization
*/
public static byte[] serializeSchema(Collection<RowMutation> schema, int version) throws IOException
{
FastByteArrayOutputStream bout = new FastByteArrayOutputStream();
FastByteArrayOutputStream bout = new FastByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
dout.writeInt(migrations.size());
// riddle me this: how do we know that these binary values (which contained serialized row mutations) are compatible
// with the destination? Further, since these migrations may be old, how do we know if they are compatible with
// the current version? The bottom line is that we don't. For this reason, running migrations from a new node
// to an old node will be a crap shoot. Pushing migrations from an old node to a new node should work, so long
// as the oldest migrations are only one version old. We need a way of flattening schemas so that this isn't a
// problem during upgrades.
for (IColumn col : migrations)
{
assert col instanceof Column;
ByteBufferUtil.writeWithLength(col.name(), dout);
ByteBufferUtil.writeWithLength(col.value(), dout);
}
dout.writeInt(schema.size());
for (RowMutation mutation : schema)
RowMutation.serializer().serialize(mutation, dout, version);
dout.close();
byte[] body = bout.toByteArray();
return new Message(FBUtilities.getBroadcastAddress(), StorageService.Verb.DEFINITIONS_UPDATE, body, version);
return bout.toByteArray();
}
// other half of this transformation is in MigrationManager.
public static Collection<Column> makeColumns(Message msg) throws IOException
/**
* Deserialize migration message considering data compatibility starting from version 1.1
*
* @param data The data of the message from coordinator which hold schema mutations to apply
* @param version The version of the message
*
* @return The collection of the row mutations to apply on the node (aka schema)
*
* @throws IOException if message is of incompatible version or data is corrupted
*/
public static Collection<RowMutation> deserializeMigrationMessage(byte[] data, int version) throws IOException
{
Collection<Column> cols = new ArrayList<Column>();
DataInputStream in = new DataInputStream(new FastByteArrayInputStream(msg.getMessageBody()));
if (version < MessagingService.VERSION_11)
throw new IOException("Can't accept schema migrations from Cassandra versions previous to 1.1, please update first.");
Collection<RowMutation> schema = new ArrayList<RowMutation>();
DataInputStream in = new DataInputStream(new FastByteArrayInputStream(data));
int count = in.readInt();
for (int i = 0; i < count; i++)
{
byte[] name = new byte[in.readInt()];
in.readFully(name);
byte[] value = new byte[in.readInt()];
in.readFully(value);
cols.add(new Column(ByteBuffer.wrap(name), ByteBuffer.wrap(value)));
}
in.close();
return cols;
schema.add(RowMutation.serializer().deserialize(in, version));
return schema;
}
}

14
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -103,6 +103,7 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
GOSSIP_DIGEST_ACK2,
DEFINITIONS_ANNOUNCE, // Deprecated
DEFINITIONS_UPDATE,
MIGRATION_REQUEST,
TRUNCATE,
SCHEMA_CHECK,
INDEX_SCAN, // Deprecated
@ -125,6 +126,7 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
put(Verb.MUTATION, Stage.MUTATION);
put(Verb.BINARY, Stage.MUTATION);
put(Verb.READ_REPAIR, Stage.MUTATION);
put(Verb.TRUNCATE, Stage.MUTATION);
put(Verb.READ, Stage.READ);
put(Verb.REQUEST_RESPONSE, Stage.REQUEST_RESPONSE);
put(Verb.STREAM_REPLY, Stage.MISC); // TODO does this really belong on misc? I've just copied old behavior here
@ -138,9 +140,9 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
put(Verb.GOSSIP_DIGEST_ACK, Stage.GOSSIP);
put(Verb.GOSSIP_DIGEST_ACK2, Stage.GOSSIP);
put(Verb.GOSSIP_DIGEST_SYN, Stage.GOSSIP);
put(Verb.DEFINITIONS_UPDATE, Stage.READ);
put(Verb.TRUNCATE, Stage.MUTATION);
put(Verb.DEFINITIONS_UPDATE, Stage.MIGRATION);
put(Verb.SCHEMA_CHECK, Stage.MIGRATION);
put(Verb.MIGRATION_REQUEST, Stage.MIGRATION);
put(Verb.INDEX_SCAN, Stage.READ);
put(Verb.REPLICATION_FINISHED, Stage.MISC);
put(Verb.INTERNAL_RESPONSE, Stage.INTERNAL_RESPONSE);
@ -265,6 +267,8 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
MessagingService.instance().registerVerbHandlers(Verb.RANGE_SLICE, new RangeSliceVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.INDEX_SCAN, new IndexScanVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.COUNTER_MUTATION, new CounterMutationVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.TRUNCATE, new TruncateVerbHandler());
// see BootStrapper for a summary of how the bootstrap verbs interact
MessagingService.instance().registerVerbHandlers(Verb.BOOTSTRAP_TOKEN, new BootStrapper.BootstrapTokenVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.STREAM_REQUEST, new StreamRequestVerbHandler());
@ -282,8 +286,8 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
MessagingService.instance().registerVerbHandlers(Verb.GOSSIP_DIGEST_ACK2, new GossipDigestAck2VerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.DEFINITIONS_UPDATE, new DefinitionsUpdateVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.TRUNCATE, new TruncateVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.SCHEMA_CHECK, new SchemaCheckVerbHandler());
MessagingService.instance().registerVerbHandlers(Verb.MIGRATION_REQUEST, new MigrationRequestVerbHandler());
// spin up the streaming service so it is available for jmx tools.
if (StreamingService.instance == null)
@ -391,6 +395,8 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
{
throw new IOError(ex);
}
Schema.instance.updateVersion();
MigrationManager.passiveAnnounce(Schema.instance.getVersion());
}
@ -499,6 +505,8 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe
logger_.info("Starting up server gossip");
joined = true;
Schema.instance.updateVersion();
// have to start the gossip service before we can see any info on other nodes. this is necessary
// for bootstrap to get the load info it needs.
// (we won't be part of the storage ring though until we add a nodeId to our state, below.)

72
src/java/org/apache/cassandra/thrift/CassandraServer.java
View File

@ -24,8 +24,6 @@ import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import java.util.zip.DataFormatException;
@ -57,6 +55,8 @@ import org.apache.cassandra.service.SocketSessionManagementService;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
import org.apache.thrift.TException;
public class CassandraServer implements Cassandra.Iface
@ -850,27 +850,16 @@ public class CassandraServer implements Cassandra.Iface
// InvalidRequestException. atypical failures will throw a RuntimeException.
private static void applyMigrationOnStage(final Migration m)
{
Future f = StageManager.getStage(Stage.MIGRATION).submit(new Callable()
Future f = StageManager.getStage(Stage.MIGRATION).submit(new WrappedRunnable()
{
public Object call() throws Exception
public void runMayThrow() throws Exception
{
m.apply();
m.announce();
return null;
}
});
try
{
f.get();
}
catch (InterruptedException e)
{
throw new AssertionError(e);
}
catch (ExecutionException e)
{
throw new RuntimeException(e);
}
FBUtilities.waitOnFuture(f);
}
public synchronized String system_add_column_family(CfDef cf_def)
@ -894,12 +883,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
public synchronized String system_drop_column_family(String column_family)
@ -920,12 +903,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
public synchronized String system_add_keyspace(KsDef ks_def)
@ -967,12 +944,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
public synchronized String system_drop_keyspace(String keyspace)
@ -994,12 +965,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
/** update an existing keyspace, but do not allow column family modifications.
@ -1019,7 +984,7 @@ public class CassandraServer implements Cassandra.Iface
try
{
ThriftValidation.validateKsDef(ks_def);
applyMigrationOnStage(new UpdateKeyspace(KSMetaData.fromThrift(ks_def)));
applyMigrationOnStage(new UpdateKeyspace(ks_def));
return Schema.instance.getVersion().toString();
}
catch (ConfigurationException e)
@ -1028,12 +993,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
public synchronized String system_update_column_family(CfDef cf_def)
@ -1054,16 +1013,7 @@ public class CassandraServer implements Cassandra.Iface
{
// ideally, apply() would happen on the stage with the
CFMetaData.applyImplicitDefaults(cf_def);
org.apache.cassandra.db.migration.avro.CfDef result;
try
{
result = CFMetaData.fromThrift(cf_def).toAvro();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
UpdateColumnFamily update = new UpdateColumnFamily(result);
UpdateColumnFamily update = new UpdateColumnFamily(cf_def);
applyMigrationOnStage(update);
return Schema.instance.getVersion().toString();
}
@ -1073,12 +1023,6 @@ public class CassandraServer implements Cassandra.Iface
ex.initCause(e);
throw ex;
}
catch (IOException e)
{
InvalidRequestException ex = new InvalidRequestException(e.getMessage());
ex.initCause(e);
throw ex;
}
}
private void validateSchemaAgreement() throws SchemaDisagreementException

27
src/java/org/apache/cassandra/utils/FBUtilities.java
View File

@ -354,19 +354,22 @@ public class FBUtilities
public static void waitOnFutures(Iterable<Future<?>> futures)
{
for (Future f : futures)
waitOnFuture(f);
}
public static void waitOnFuture(Future<?> future)
{
try
{
try
{
f.get();
}
catch (ExecutionException ee)
{
throw new RuntimeException(ee);
}
catch (InterruptedException ie)
{
throw new AssertionError(ie);
}
future.get();
}
catch (ExecutionException ee)
{
throw new RuntimeException(ee);
}
catch (InterruptedException ie)
{
throw new AssertionError(ie);
}
}

1
test/data/serialization/0.7/db.migration.Keyspace1.bin
View File

File diff suppressed because one or more lines are too long

1
test/data/serialization/0.7/db.migration.Keyspace2.bin
View File

File diff suppressed because one or more lines are too long

BIN
test/data/serialization/0.7/db.migration.Keyspace3.bin
View File

Binary file not shown.

1
test/data/serialization/0.7/db.migration.Keyspace4.bin
View File

File diff suppressed because one or more lines are too long

1
test/data/serialization/0.7/db.migration.Keyspace5.bin
View File

File diff suppressed because one or more lines are too long

2
test/unit/org/apache/cassandra/SchemaLoader.java
View File

@ -58,7 +58,7 @@ public class SchemaLoader
try
{
Schema.instance.load(schemaDefinition(), Schema.instance.getVersion());
Schema.instance.load(schemaDefinition());
}
catch (ConfigurationException e)
{

35
test/unit/org/apache/cassandra/config/CFMetaDataTest.java
View File

@ -21,7 +21,6 @@ package org.apache.cassandra.config;
import java.util.ArrayList;
import java.util.List;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.thrift.CfDef;
@ -64,28 +63,28 @@ public class CFMetaDataTest
CFMetaData cfMetaData = CFMetaData.fromThrift(cfDef);
// make a correct Avro object
org.apache.cassandra.db.migration.avro.CfDef avroCfDef = new org.apache.cassandra.db.migration.avro.CfDef();
avroCfDef.keyspace = new Utf8(KEYSPACE);
avroCfDef.name = new Utf8(COLUMN_FAMILY);
avroCfDef.default_validation_class = new Utf8(cfDef.default_validation_class);
avroCfDef.comment = new Utf8(cfDef.comment);
avroCfDef.column_metadata = new ArrayList<org.apache.cassandra.db.migration.avro.ColumnDef>();
CfDef thriftCfDef = new CfDef();
thriftCfDef.keyspace = KEYSPACE;
thriftCfDef.name = COLUMN_FAMILY;
thriftCfDef.default_validation_class = cfDef.default_validation_class;
thriftCfDef.comment = cfDef.comment;
thriftCfDef.column_metadata = new ArrayList<ColumnDef>();
for (ColumnDef columnDef : columnDefs)
{
org.apache.cassandra.db.migration.avro.ColumnDef c = new org.apache.cassandra.db.migration.avro.ColumnDef();
ColumnDef c = new ColumnDef();
c.name = ByteBufferUtil.clone(columnDef.name);
c.validation_class = new Utf8(columnDef.getValidation_class());
c.index_name = new Utf8(columnDef.getIndex_name());
c.index_type = org.apache.cassandra.db.migration.avro.IndexType.KEYS;
avroCfDef.column_metadata.add(c);
c.validation_class = columnDef.getValidation_class();
c.index_name = columnDef.getIndex_name();
c.index_type = IndexType.KEYS;
thriftCfDef.column_metadata.add(c);
}
org.apache.cassandra.db.migration.avro.CfDef converted = cfMetaData.toAvro();
CfDef converted = cfMetaData.toThrift();
assertEquals(avroCfDef.keyspace, converted.keyspace);
assertEquals(avroCfDef.name, converted.name);
assertEquals(avroCfDef.default_validation_class, converted.default_validation_class);
assertEquals(avroCfDef.comment, converted.comment);
assertEquals(avroCfDef.column_metadata, converted.column_metadata);
assertEquals(thriftCfDef.keyspace, converted.keyspace);
assertEquals(thriftCfDef.name, converted.name);
assertEquals(thriftCfDef.default_validation_class, converted.default_validation_class);
assertEquals(thriftCfDef.comment, converted.comment);
assertEquals(thriftCfDef.column_metadata, converted.column_metadata);
}
}

2
test/unit/org/apache/cassandra/config/ColumnDefinitionTest.java
View File

@ -51,7 +51,7 @@ public class ColumnDefinitionTest
protected void testSerializeDeserialize(ColumnDefinition cd) throws Exception
{
ColumnDefinition newCd = ColumnDefinition.fromAvro(cd.toAvro());
ColumnDefinition newCd = ColumnDefinition.fromThrift(cd.toThrift());
assert cd != newCd;
assert cd.hashCode() == newCd.hashCode();
assert cd.equals(newCd);

26
test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java
View File

@ -18,40 +18,26 @@
*/
package org.apache.cassandra.config;
import static org.junit.Assert.assertNotNull;
import org.apache.avro.specific.SpecificRecord;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.db.migration.AddKeyspace;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.junit.Test;
import java.io.IOException;
import java.util.UUID;
public class DatabaseDescriptorTest
{
protected <D extends SpecificRecord> D serDe(D record, D newInstance) throws IOException
{
D actual = SerDeUtils.deserialize(record.getSchema(),
SerDeUtils.serialize(record),
newInstance);
assert actual.equals(record) : actual + " != " + record;
return actual;
}
@Test
public void testCFMetaDataSerialization() throws IOException, ConfigurationException
public void testCFMetaDataSerialization() throws IOException, ConfigurationException, InvalidRequestException
{
// test serialization of all defined test CFs.
for (String table : Schema.instance.getNonSystemTables())
{
for (CFMetaData cfm : Schema.instance.getTableMetaData(table).values())
{
CFMetaData cfmDupe = CFMetaData.fromAvro(serDe(cfm.toAvro(), new org.apache.cassandra.db.migration.avro.CfDef()));
CFMetaData cfmDupe = CFMetaData.fromThrift(cfm.toThrift());
assert cfmDupe != null;
assert cfmDupe.equals(cfm);
}
@ -65,7 +51,7 @@ public class DatabaseDescriptorTest
{
// Not testing round-trip on the KsDef via serDe() because maps
// cannot be compared in avro.
KSMetaData ksmDupe = KSMetaData.fromAvro(ksm.toAvro());
KSMetaData ksmDupe = KSMetaData.fromThrift(ksm.toThrift());
assert ksmDupe != null;
assert ksmDupe.equals(ksm);
}
@ -88,8 +74,8 @@ public class DatabaseDescriptorTest
assert Schema.instance.getTableDefinition("ks0") != null;
assert Schema.instance.getTableDefinition("ks1") != null;
Schema.instance.clearTableDefinition(Schema.instance.getTableDefinition("ks0"), new UUID(4096, 0));
Schema.instance.clearTableDefinition(Schema.instance.getTableDefinition("ks1"), new UUID(4096, 0));
Schema.instance.clearTableDefinition(Schema.instance.getTableDefinition("ks0"));
Schema.instance.clearTableDefinition(Schema.instance.getTableDefinition("ks1"));
assert Schema.instance.getTableDefinition("ks0") == null;
assert Schema.instance.getTableDefinition("ks1") == null;

112
test/unit/org/apache/cassandra/db/DefsTest.java
View File

@ -18,15 +18,12 @@
package org.apache.cassandra.db;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
import org.apache.avro.util.Utf8;
import org.apache.cassandra.CleanupHelper;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.*;
@ -41,38 +38,20 @@ import org.apache.cassandra.db.migration.DropKeyspace;
import org.apache.cassandra.db.migration.Migration;
import org.apache.cassandra.db.migration.UpdateColumnFamily;
import org.apache.cassandra.db.migration.UpdateKeyspace;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.locator.OldNetworkTopologyStrategy;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.thrift.CfDef;
import org.apache.cassandra.thrift.ColumnDef;
import org.apache.cassandra.thrift.IndexType;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
import org.junit.Test;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.junit.Test;
public class DefsTest extends CleanupHelper
{
@Test
public void testZeroInjection() throws IOException
{
org.apache.cassandra.db.migration.avro.CfDef cd = new org.apache.cassandra.db.migration.avro.CfDef();
// populate only fields that must be non-null.
cd.keyspace = new Utf8("Lest Ks");
cd.name = new Utf8("Mest Cf");
org.apache.cassandra.db.migration.avro.CfDef cd2 = SerDeUtils.deserializeWithSchema(SerDeUtils.serializeWithSchema(cd), new org.apache.cassandra.db.migration.avro.CfDef());
assert cd.equals(cd2);
// make sure some of the fields didn't get unexpected zeros put in during [de]serialize operations.
assert cd.min_compaction_threshold == null;
assert cd2.min_compaction_threshold == null;
assert cd.compaction_strategy == null;
}
@Test
public void ensureStaticCFMIdsAreLessThan1000()
{
@ -109,22 +88,22 @@ public class DefsTest extends CleanupHelper
// we'll be adding this one later. make sure it's not already there.
assert cfm.getColumn_metadata().get(ByteBuffer.wrap(new byte[] { 5 })) == null;
org.apache.cassandra.db.migration.avro.CfDef cfDef = cfm.toAvro();
CfDef cfDef = cfm.toThrift();
// add one.
org.apache.cassandra.db.migration.avro.ColumnDef addIndexDef = new org.apache.cassandra.db.migration.avro.ColumnDef();
ColumnDef addIndexDef = new ColumnDef();
addIndexDef.index_name = "5";
addIndexDef.index_type = org.apache.cassandra.db.migration.avro.IndexType.KEYS;
addIndexDef.index_type = IndexType.KEYS;
addIndexDef.name = ByteBuffer.wrap(new byte[] { 5 });
addIndexDef.validation_class = BytesType.class.getName();
cfDef.column_metadata.add(addIndexDef);
// remove one.
org.apache.cassandra.db.migration.avro.ColumnDef removeIndexDef = new org.apache.cassandra.db.migration.avro.ColumnDef();
removeIndexDef.index_name = new Utf8("0");
removeIndexDef.index_type = org.apache.cassandra.db.migration.avro.IndexType.KEYS;
ColumnDef removeIndexDef = new ColumnDef();
removeIndexDef.index_name = "0";
removeIndexDef.index_type = IndexType.KEYS;
removeIndexDef.name = ByteBuffer.wrap(new byte[] { 0 });
removeIndexDef.validation_class = new Utf8(BytesType.class.getName());
removeIndexDef.validation_class = BytesType.class.getName();
assert cfDef.column_metadata.remove(removeIndexDef);
cfm.apply(cfDef);
@ -146,10 +125,11 @@ public class DefsTest extends CleanupHelper
for (String s : invalid)
assert !Migration.isLegalName(s);
}
@Test
public void saveAndRestore() throws IOException
{
/*
// verify dump and reload.
UUID first = UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress());
DefsTable.dumpToStorage(first);
@ -162,8 +142,9 @@ public class DefsTest extends CleanupHelper
KSMetaData defined = Schema.instance.getTableDefinition(loaded.name);
assert defined.equals(loaded) : String.format("%s != %s", loaded, defined);
}
*/
}
@Test
public void addNewCfToBogusTable() throws InterruptedException
{
@ -181,52 +162,6 @@ public class DefsTest extends CleanupHelper
throw new AssertionError("Unexpected exception.");
}
}
@Test
public void testMigrations() throws IOException, ConfigurationException
{
// do a save. make sure it doesn't mess with the defs version.
UUID prior = Schema.instance.getVersion();
UUID ver0 = UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress());
DefsTable.dumpToStorage(ver0);
assert Schema.instance.getVersion().equals(prior);
// add a cf.
CFMetaData newCf1 = addTestCF("Keyspace1", "MigrationCf_1", "Migration CF");
Migration m1 = new AddColumnFamily(newCf1);
m1.apply();
UUID ver1 = m1.getVersion();
assert Schema.instance.getVersion().equals(ver1);
// drop it.
Migration m3 = new DropColumnFamily("Keyspace1", "MigrationCf_1");
m3.apply();
UUID ver3 = m3.getVersion();
assert Schema.instance.getVersion().equals(ver3);
// now lets load the older migrations to see if that code works.
Collection<IColumn> serializedMigrations = Migration.getLocalMigrations(ver1, ver3);
assert serializedMigrations.size() == 2;
// test deserialization of the migrations.
Migration[] reconstituded = new Migration[2];
int i = 0;
for (IColumn col : serializedMigrations)
{
UUID version = UUIDGen.getUUID(col.name());
reconstituded[i] = Migration.deserialize(col.value(), MessagingService.version_);
assert version.equals(reconstituded[i].getVersion());
i++;
}
assert m1.getClass().equals(reconstituded[0].getClass());
assert m3.getClass().equals(reconstituded[1].getClass());
// verify that the row mutations are the same. rather than exposing the private fields, serialize and verify.
assert m1.serialize().equals(reconstituded[0].serialize());
assert m3.serialize().equals(reconstituded[1].serialize());
}
@Test
public void addNewCfWithNullComment() throws ConfigurationException, IOException, ExecutionException, InterruptedException
@ -473,7 +408,7 @@ public class DefsTest extends CleanupHelper
KSMetaData newBadKs = KSMetaData.testMetadata(cf.ksName, SimpleStrategy.class, KSMetaData.optsWithRF(4), cf2);
try
{
new UpdateKeyspace(newBadKs).apply();
new UpdateKeyspace(newBadKs.toThrift()).apply();
throw new AssertionError("Should not have been able to update a KS with a KS that described column families.");
}
catch (ConfigurationException ex)
@ -485,7 +420,7 @@ public class DefsTest extends CleanupHelper
KSMetaData newBadKs2 = KSMetaData.testMetadata(cf.ksName + "trash", SimpleStrategy.class, KSMetaData.optsWithRF(4));
try
{
new UpdateKeyspace(newBadKs2).apply();
new UpdateKeyspace(newBadKs2.toThrift()).apply();
throw new AssertionError("Should not have been able to update a KS with an invalid KS name.");
}
catch (ConfigurationException ex)
@ -494,7 +429,7 @@ public class DefsTest extends CleanupHelper
}
KSMetaData newKs = KSMetaData.testMetadata(cf.ksName, OldNetworkTopologyStrategy.class, KSMetaData.optsWithRF(1));
new UpdateKeyspace(newKs).apply();
new UpdateKeyspace(newKs.toThrift()).apply();
KSMetaData newFetchedKs = Schema.instance.getKSMetaData(newKs.name);
assert newFetchedKs.strategyClass.equals(newKs.strategyClass);
@ -514,8 +449,8 @@ public class DefsTest extends CleanupHelper
assert Schema.instance.getCFMetaData(cf.ksName, cf.cfName) != null;
// updating certain fields should fail.
org.apache.cassandra.db.migration.avro.CfDef cf_def = cf.toAvro();
cf_def.column_metadata = new ArrayList<org.apache.cassandra.db.migration.avro.ColumnDef>();
CfDef cf_def = cf.toThrift();
cf_def.column_metadata = new ArrayList<ColumnDef>();
cf_def.default_validation_class ="BytesType";
cf_def.min_compaction_threshold = 5;
cf_def.max_compaction_threshold = 31;
@ -561,7 +496,7 @@ public class DefsTest extends CleanupHelper
cf_def.id = oldId;
}
CharSequence oldStr = cf_def.name;
String oldStr = cf_def.name;
try
{
cf_def.name = cf_def.name + "_renamed";
@ -634,6 +569,9 @@ public class DefsTest extends CleanupHelper
@Test
public void testDropIndex() throws IOException, ExecutionException, InterruptedException, ConfigurationException
{
// persist keyspace definition in the system table
Schema.instance.getKSMetaData("Keyspace6").toSchema(System.currentTimeMillis()).apply();
// insert some data. save the sstable descriptor so we can make sure it's marked for delete after the drop
RowMutation rm = new RowMutation("Keyspace6", ByteBufferUtil.bytes("k1"));
rm.add(new QueryPath("Indexed1", null, ByteBufferUtil.bytes("notbirthdate")), ByteBufferUtil.bytes(1L), 0);
@ -649,7 +587,7 @@ public class DefsTest extends CleanupHelper
ColumnDefinition cdOld = meta.getColumn_metadata().values().iterator().next();
ColumnDefinition cdNew = new ColumnDefinition(cdOld.name, cdOld.getValidator(), null, null, null);
meta.columnMetadata(Collections.singletonMap(cdOld.name, cdNew));
UpdateColumnFamily update = new UpdateColumnFamily(meta.toAvro());
UpdateColumnFamily update = new UpdateColumnFamily(meta.toThrift());
update.apply();
// check
@ -667,4 +605,4 @@ public class DefsTest extends CleanupHelper
return newCFMD;
}
}
}

77
test/unit/org/apache/cassandra/db/migration/SerializationsTest.java
View File

@ -1,77 +0,0 @@
package org.apache.cassandra.db.migration;
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import org.apache.cassandra.AbstractSerializationsTester;
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.io.SerDeUtils;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.commons.codec.binary.Base64;
import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.UUID;
public class SerializationsTest extends AbstractSerializationsTester
{
private static final int ksCount = 5;
private void testWrite() throws IOException, ConfigurationException
{
for (int i = 0; i < ksCount; i++)
{
String tableName = "Keyspace" + (i + 1);
KSMetaData ksm = Schema.instance.getKSMetaData(tableName);
UUID uuid = UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress());
Schema.instance.clearTableDefinition(ksm, uuid);
Migration m = new AddKeyspace(ksm);
ByteBuffer bytes = m.serialize();
DataOutputStream out = getOutput("db.migration." + tableName + ".bin");
out.writeUTF(new String(Base64.encodeBase64(bytes.array())));
out.close();
}
}
@Test
public void testRead() throws IOException, ConfigurationException
{
if (AbstractSerializationsTester.EXECUTE_WRITES)
testWrite();
for (int i = 0; i < ksCount; i++)
{
String tableName = "Keyspace" + (i + 1);
DataInputStream in = getInput("db.migration." + tableName + ".bin");
byte[] raw = Base64.decodeBase64(in.readUTF().getBytes());
org.apache.cassandra.db.migration.avro.Migration obj = new org.apache.cassandra.db.migration.avro.Migration();
SerDeUtils.deserializeWithSchema(ByteBuffer.wrap(raw), obj);
in.close();
}
}
}