From fbda616f682362053f3a11b9f4bbdd725c9b38fa Mon Sep 17 00:00:00 2001 From: Gary Dusbabek Date: Tue, 23 Feb 2010 21:48:31 +0000 Subject: [PATCH] Refactor DatabaseDescriptor to isolate the code that establishes the keyspaces and column families. Patch by Gary Dusbabek, reviewed by Eric Evans. CASSANDRA-819 git-svn-id: https://svn.apache.org/repos/asf/incubator/cassandra/trunk@915533 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/cassandra/config/CFMetaData.java | 101 +++- .../cassandra/config/DatabaseDescriptor.java | 499 ++++++++++-------- .../apache/cassandra/config/KSMetaData.java | 136 +++++ .../cassandra/db/marshal/AbstractType.java | 8 + .../cassandra/service/StorageService.java | 4 +- .../apache/cassandra/utils/FBUtilities.java | 12 + .../config/DatabaseDescriptorTest.java | 36 +- .../service/AntiEntropyServiceTest.java | 4 +- 8 files changed, 566 insertions(+), 234 deletions(-) create mode 100644 src/java/org/apache/cassandra/config/KSMetaData.java diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index 86e617224a..778b7bb5e4 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -19,15 +19,36 @@ package org.apache.cassandra.config; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.utils.FBUtilities; -public class CFMetaData +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public final class CFMetaData { - public String tableName; // name of table which has this column family - public String cfName; // name of the column family - public String columnType; // type: super, standard, etc. - public AbstractType comparator; // name sorted, time stamp sorted etc. - public AbstractType subcolumnComparator; // like comparator, for supercolumns - public String comment; // for humans only + public final String tableName; // name of table which has this column family + public final String cfName; // name of the column family + public final String columnType; // type: super, standard, etc. + public final AbstractType comparator; // name sorted, time stamp sorted etc. + public final AbstractType subcolumnComparator; // like comparator, for supercolumns + public final String comment; // for humans only + public final double rowCacheSize; // default 0 + public final double keysCachedFraction; // default 0.01 + + CFMetaData(String tableName, String cfName, String columnType, AbstractType comparator, AbstractType subcolumnComparator, String comment, double rowCacheSize, double keysCachedFraction) + { + this.tableName = tableName; + this.cfName = cfName; + this.columnType = columnType; + this.comparator = comparator; + this.subcolumnComparator = subcolumnComparator; + this.comment = comment; + this.rowCacheSize = rowCacheSize; + this.keysCachedFraction = keysCachedFraction; + } // a quick and dirty pretty printer for describing the column family... public String pretty() @@ -36,4 +57,70 @@ public class CFMetaData + "Column Family Type: " + columnType + "\n" + "Columns Sorted By: " + comparator + "\n"; } + + public static byte[] serialize(CFMetaData cfm) throws IOException + { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + DataOutputStream dout = new DataOutputStream(bout); + dout.writeUTF(cfm.tableName); + dout.writeUTF(cfm.cfName); + dout.writeUTF(cfm.columnType); + dout.writeUTF(cfm.comparator.getClass().getName()); + dout.writeBoolean(cfm.subcolumnComparator != null); + if (cfm.subcolumnComparator != null) + dout.writeUTF(cfm.subcolumnComparator.getClass().getName()); + dout.writeBoolean(cfm.comment != null); + if (cfm.comment != null) + dout.writeUTF(cfm.comment); + dout.writeDouble(cfm.rowCacheSize); + dout.writeDouble(cfm.keysCachedFraction); + dout.close(); + return bout.toByteArray(); + } + + public static CFMetaData deserialize(InputStream in) throws IOException + { + + DataInputStream din = new DataInputStream(in); + String tableName = din.readUTF(); + String cfName = din.readUTF(); + String columnType = din.readUTF(); + AbstractType comparator = null; + try + { + comparator = (AbstractType)Class.forName(din.readUTF()).newInstance(); + } + catch (Exception ex) + { + throw new IOException(ex); + } + AbstractType subcolumnComparator = null; + try + { + subcolumnComparator = din.readBoolean() ? (AbstractType)Class.forName(din.readUTF()).newInstance() : null; + } + catch (Exception ex) + { + + } + String comment = din.readBoolean() ? din.readUTF() : null; + double rowCacheSize = din.readDouble(); + double keysCachedFraction = din.readDouble(); + return new CFMetaData(tableName, cfName, columnType, comparator, subcolumnComparator, comment, rowCacheSize, keysCachedFraction); + } + + public boolean equals(Object obj) + { + if (!(obj instanceof CFMetaData)) + return false; + CFMetaData other = (CFMetaData)obj; + return other.tableName.equals(tableName) + && other.cfName.equals(cfName) + && other.columnType.equals(columnType) + && other.comparator.equals(comparator) + && FBUtilities.equals(other.subcolumnComparator, subcolumnComparator) + && FBUtilities.equals(other.comment, comment) + && other.rowCacheSize == rowCacheSize + && other.keysCachedFraction == keysCachedFraction; + } } diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 011cb1bd8c..657792d8c1 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -26,16 +26,19 @@ import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.BytesType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.locator.EndPointSnitch; import org.apache.cassandra.locator.IEndPointSnitch; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.io.util.FileUtils; -import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.XMLUtils; import org.apache.log4j.Logger; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; +import javax.xml.xpath.XPathExpressionException; import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -84,29 +87,13 @@ public class DatabaseDescriptor private static double flushDataBufferSizeInMB_ = 32; private static double flushIndexBufferSizeInMB_ = 8; private static int slicedReadBufferSizeInKB_ = 64; - private static Set tables_ = new HashSet(); + + static Map tables_ = new HashMap(); private static int bmtThreshold_ = 256; - private static Map, Double> tableKeysCachedFractions_ = new HashMap, Double>(); - private static Map, Double> tableRowCacheSizes = new HashMap, Double>(); - - /* - * A map from table names to the set of column families for the table and the - * corresponding meta data for that column family. - */ - private static Map> tableToCFMetaDataMap_; - - // map tables to replication strategies. - private static Map> replicationStrategyClasses_; - - // map tables to replication factors. - private static Map replicationFactors_; - /* Hashing strategy Random or OPHF */ private static IPartitioner partitioner_; - private static Map endPointSnitches_; - /* if the size of columns or super-columns are more than this, indexing will kick in */ private static int columnIndexSizeInKB_; /* Number of minutes to keep a memtable in memory */ @@ -457,170 +444,30 @@ public class DatabaseDescriptor if ( value != null) CommitLog.setSegmentSize(Integer.parseInt(value) * 1024 * 1024); - tableToCFMetaDataMap_ = new HashMap>(); - replicationFactors_ = new HashMap(); - replicationStrategyClasses_ = new HashMap>(); - endPointSnitches_ = new HashMap(); - - /* Read the table related stuff from config */ - NodeList tables = xmlUtils.getRequestedNodeList("/Storage/Keyspaces/Keyspace"); - int size = tables.getLength(); - for ( int i = 0; i < size; ++i ) - { - Node table = tables.item(i); - - /* parsing out the table name */ - String tName = XMLUtils.getAttributeValue(table, "Name"); - if (tName == null) - { - throw new ConfigurationException("Table name attribute is required"); - } - if (tName.equalsIgnoreCase(Table.SYSTEM_TABLE)) - { - throw new ConfigurationException("'system' is a reserved table name for Cassandra internals"); - } - tables_.add(tName); - tableToCFMetaDataMap_.put(tName, new HashMap()); - - /* See which replica placement strategy to use */ - String replicaPlacementStrategyClassName = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/ReplicaPlacementStrategy"); - if (replicaPlacementStrategyClassName == null) - { - throw new ConfigurationException("Missing replicaplacementstrategy directive for " + tName); - } - try - { - Class cls = (Class) Class.forName(replicaPlacementStrategyClassName); - replicationStrategyClasses_.put(tName, cls); - } - catch (ClassNotFoundException e) - { - throw new ConfigurationException("Invalid replicaplacementstrategy class " + replicaPlacementStrategyClassName); - } - - /* Data replication factor */ - String replicationFactor = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/ReplicationFactor"); - if (replicationFactor == null) - throw new ConfigurationException("Missing replicationfactor directory for keyspace " + tName); - else - replicationFactors_.put(tName, Integer.parseInt(replicationFactor)); - - /* end point snitch */ - String endPointSnitchClassName = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/EndPointSnitch"); - if (endPointSnitchClassName == null) - { - throw new ConfigurationException("Missing endpointsnitch directive for keyspace " + tName); - } - try - { - Class cls = Class.forName(endPointSnitchClassName); - endPointSnitches_.put(tName, (IEndPointSnitch)cls.getConstructor().newInstance()); - } - catch (ClassNotFoundException e) - { - throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName); - } - - String xqlTable = "/Storage/Keyspaces/Keyspace[@Name='" + tName + "']/"; - NodeList columnFamilies = xmlUtils.getRequestedNodeList(xqlTable + "ColumnFamily"); - - //NodeList columnFamilies = xmlUtils.getRequestedNodeList(table, "ColumnFamily"); - int size2 = columnFamilies.getLength(); - - for ( int j = 0; j < size2; ++j ) - { - Node columnFamily = columnFamilies.item(j); - String cfName = XMLUtils.getAttributeValue(columnFamily, "Name"); - if (cfName == null) - { - throw new ConfigurationException("ColumnFamily name attribute is required"); - } - String xqlCF = xqlTable + "ColumnFamily[@Name='" + cfName + "']/"; - - // Parse out the column type - String rawColumnType = XMLUtils.getAttributeValue(columnFamily, "ColumnType"); - String columnType = ColumnFamily.getColumnType(rawColumnType); - if (columnType == null) - { - throw new ConfigurationException("ColumnFamily " + cfName + " has invalid type " + rawColumnType); - } - - if (XMLUtils.getAttributeValue(columnFamily, "ColumnSort") != null) - { - throw new ConfigurationException("ColumnSort is no longer an accepted attribute. Use CompareWith instead."); - } - - // Parse out the column comparator - AbstractType columnComparator = getComparator(columnFamily, "CompareWith"); - AbstractType subcolumnComparator = null; - if (columnType.equals("Super")) - { - subcolumnComparator = getComparator(columnFamily, "CompareSubcolumnsWith"); - } - else if (XMLUtils.getAttributeValue(columnFamily, "CompareSubcolumnsWith") != null) - { - throw new ConfigurationException("CompareSubcolumnsWith is only a valid attribute on super columnfamilies (not regular columnfamily " + cfName + ")"); - } - - if ((value = XMLUtils.getAttributeValue(columnFamily, "KeysCachedFraction")) != null) - { - tableKeysCachedFractions_.put(Pair.create(tName, cfName), Double.valueOf(value)); - } - - if ((value = XMLUtils.getAttributeValue(columnFamily, "RowsCached")) != null) - { - if (value.endsWith("%")) - { - tableRowCacheSizes.put(Pair.create(tName, cfName), Double.valueOf(value.substring(0, value.length() - 1)) / 100); - } - else - { - tableRowCacheSizes.put(Pair.create(tName, cfName), Double.valueOf(value)); - } - } - - // Parse out user-specified logical names for the various dimensions - // of a the column family from the config. - String cfComment = xmlUtils.getNodeValue(xqlCF + "Comment"); - - // now populate the column family meta data and - // insert it into the table dictionary. - CFMetaData cfMetaData = new CFMetaData(); - - cfMetaData.tableName = tName; - cfMetaData.cfName = cfName; - cfMetaData.comment = cfComment; - - cfMetaData.columnType = columnType; - cfMetaData.comparator = columnComparator; - cfMetaData.subcolumnComparator = subcolumnComparator; - - tableToCFMetaDataMap_.get(tName).put(cfName, cfMetaData); - } - } + readTablesFromXml(); if (tables_.isEmpty()) throw new ConfigurationException("No keyspaces configured"); // Hardcoded system tables - tables_.add(Table.SYSTEM_TABLE); - Map systemMetadata = new HashMap(); + KSMetaData systemMeta = new KSMetaData(Table.SYSTEM_TABLE, null, -1, null); + tables_.put(Table.SYSTEM_TABLE, systemMeta); + systemMeta.cfMetaData.put(SystemTable.STATUS_CF, new CFMetaData(Table.SYSTEM_TABLE, + SystemTable.STATUS_CF, + "Standard", + new UTF8Type(), + null, + "persistent metadata for the local node", + 0d, + 0.01d)); - CFMetaData data = new CFMetaData(); - data.cfName = SystemTable.STATUS_CF; - data.columnType = "Standard"; - data.comparator = new UTF8Type(); - data.comment = "persistent metadata for the local node"; - systemMetadata.put(data.cfName, data); - - data = new CFMetaData(); - data.cfName = HintedHandOffManager.HINTS_CF; - data.columnType = "Super"; - data.comparator = new UTF8Type(); - data.subcolumnComparator = new BytesType(); - data.comment = "hinted handoff data"; - systemMetadata.put(data.cfName, data); - - tableToCFMetaDataMap_.put(Table.SYSTEM_TABLE, systemMetadata); + systemMeta.cfMetaData.put(HintedHandOffManager.HINTS_CF, new CFMetaData(Table.SYSTEM_TABLE, + HintedHandOffManager.HINTS_CF, + "Super", + new UTF8Type(), + new BytesType(), + "hinted handoff data", + 0d, + 0.01d)); /* Load the seeds for node contact points */ String[] seeds = xmlUtils.getNodeValues("/Storage/Seeds/Seed"); @@ -645,6 +492,200 @@ public class DatabaseDescriptor } } + private static void readTablesFromXml() throws ConfigurationException + { + XMLUtils xmlUtils = null; + try + { + xmlUtils = new XMLUtils(configFileName_); + } + catch (ParserConfigurationException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (SAXException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (IOException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + + /* Read the table related stuff from config */ + try + { + NodeList tables = xmlUtils.getRequestedNodeList("/Storage/Keyspaces/Keyspace"); + int size = tables.getLength(); + for ( int i = 0; i < size; ++i ) + { + String value = null; + Node table = tables.item(i); + + /* parsing out the table ksName */ + String ksName = XMLUtils.getAttributeValue(table, "Name"); + if (ksName == null) + { + throw new ConfigurationException("Table name attribute is required"); + } + if (ksName.equalsIgnoreCase(Table.SYSTEM_TABLE)) + { + throw new ConfigurationException("'system' is a reserved table name for Cassandra internals"); + } + + /* See which replica placement strategy to use */ + String replicaPlacementStrategyClassName = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + ksName + "']/ReplicaPlacementStrategy"); + if (replicaPlacementStrategyClassName == null) + { + throw new ConfigurationException("Missing replicaplacementstrategy directive for " + ksName); + } + Class repStratClass = null; + try + { + repStratClass = (Class) Class.forName(replicaPlacementStrategyClassName); + } + catch (ClassNotFoundException e) + { + throw new ConfigurationException("Invalid replicaplacementstrategy class " + replicaPlacementStrategyClassName); + } + + /* Data replication factor */ + String replicationFactor = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + ksName + "']/ReplicationFactor"); + int repFact = -1; + if (replicationFactor == null) + throw new ConfigurationException("Missing replicationfactor directory for keyspace " + ksName); + else + { + repFact = Integer.parseInt(replicationFactor); + } + + /* end point snitch */ + String endPointSnitchClassName = xmlUtils.getNodeValue("/Storage/Keyspaces/Keyspace[@Name='" + ksName + "']/EndPointSnitch"); + if (endPointSnitchClassName == null) + { + throw new ConfigurationException("Missing endpointsnitch directive for keyspace " + ksName); + } + IEndPointSnitch epSnitch = null; + try + { + Class cls = Class.forName(endPointSnitchClassName); + epSnitch = (IEndPointSnitch)cls.getConstructor().newInstance(); + } + catch (ClassNotFoundException e) + { + throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName); + } + catch (NoSuchMethodException e) + { + throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName + " " + e.getMessage()); + } + catch (InstantiationException e) + { + throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName + " " + e.getMessage()); + } + catch (IllegalAccessException e) + { + throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName + " " + e.getMessage()); + } + catch (InvocationTargetException e) + { + throw new ConfigurationException("Invalid endpointsnitch class " + endPointSnitchClassName + " " + e.getMessage()); + } + + String xqlTable = "/Storage/Keyspaces/Keyspace[@Name='" + ksName + "']/"; + NodeList columnFamilies = xmlUtils.getRequestedNodeList(xqlTable + "ColumnFamily"); + + KSMetaData meta = new KSMetaData(ksName, repStratClass, repFact, epSnitch); + + //NodeList columnFamilies = xmlUtils.getRequestedNodeList(table, "ColumnFamily"); + int size2 = columnFamilies.getLength(); + + for ( int j = 0; j < size2; ++j ) + { + Node columnFamily = columnFamilies.item(j); + String tableName = ksName; + String cfName = XMLUtils.getAttributeValue(columnFamily, "Name"); + if (cfName == null) + { + throw new ConfigurationException("ColumnFamily name attribute is required"); + } + String xqlCF = xqlTable + "ColumnFamily[@Name='" + cfName + "']/"; + + // Parse out the column type + String rawColumnType = XMLUtils.getAttributeValue(columnFamily, "ColumnType"); + String columnType = ColumnFamily.getColumnType(rawColumnType); + if (columnType == null) + { + throw new ConfigurationException("ColumnFamily " + cfName + " has invalid type " + rawColumnType); + } + + if (XMLUtils.getAttributeValue(columnFamily, "ColumnSort") != null) + { + throw new ConfigurationException("ColumnSort is no longer an accepted attribute. Use CompareWith instead."); + } + + // Parse out the column comparator + AbstractType comparator = getComparator(columnFamily, "CompareWith"); + AbstractType subcolumnComparator = null; + if (columnType.equals("Super")) + { + subcolumnComparator = getComparator(columnFamily, "CompareSubcolumnsWith"); + } + else if (XMLUtils.getAttributeValue(columnFamily, "CompareSubcolumnsWith") != null) + { + throw new ConfigurationException("CompareSubcolumnsWith is only a valid attribute on super columnfamilies (not regular columnfamily " + cfName + ")"); + } + + double keysCachedFraction = 0.01d; + if ((value = XMLUtils.getAttributeValue(columnFamily, "KeysCachedFraction")) != null) + { + keysCachedFraction = Double.valueOf(value); + } + + double rowCacheSize = 0; + if ((value = XMLUtils.getAttributeValue(columnFamily, "RowsCached")) != null) + { + if (value.endsWith("%")) + { + rowCacheSize = Double.valueOf(value.substring(0, value.length() - 1)) / 100; + } + else + { + rowCacheSize = Double.valueOf(value); + } + } + + // Parse out user-specified logical names for the various dimensions + // of a the column family from the config. + String comment = xmlUtils.getNodeValue(xqlCF + "Comment"); + + // insert it into the table dictionary. + meta.cfMetaData.put(cfName, new CFMetaData(tableName, cfName, columnType, comparator, subcolumnComparator, comment, rowCacheSize, keysCachedFraction)); + } + + tables_.put(meta.name, meta); + } + } + catch (XPathExpressionException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (TransformerException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + } + public static IAuthenticator getAuthenticator() { return authenticator; @@ -655,11 +696,21 @@ public class DatabaseDescriptor return thriftFramed_; } - private static AbstractType getComparator(Node columnFamily, String attr) - throws ConfigurationException, TransformerException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, InstantiationException + private static AbstractType getComparator(Node columnFamily, String attr) throws ConfigurationException +// throws ConfigurationException, TransformerException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, InstantiationException { Class typeClass; - String compareWith = XMLUtils.getAttributeValue(columnFamily, attr); + String compareWith = null; + try + { + compareWith = XMLUtils.getAttributeValue(columnFamily, attr); + } + catch (TransformerException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } if (compareWith == null) { typeClass = BytesType.class; @@ -676,7 +727,34 @@ public class DatabaseDescriptor throw new ConfigurationException("Unable to load class " + className + " for " + attr + " attribute"); } } - return typeClass.getConstructor().newInstance(); + try + { + return typeClass.getConstructor().newInstance(); + } + catch (InstantiationException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (IllegalAccessException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (InvocationTargetException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } + catch (NoSuchMethodException e) + { + ConfigurationException ex = new ConfigurationException(e.getMessage()); + ex.initCause(e); + throw ex; + } } /** @@ -707,7 +785,7 @@ public class DatabaseDescriptor for (String dataFile : dataFileDirectories_) { FileUtils.createDirectory(dataFile + File.separator + Table.SYSTEM_TABLE); - for (String table : tables_) + for (String table : tables_.keySet()) { String oneDir = dataFile + File.separator + table; FileUtils.createDirectory(oneDir); @@ -729,7 +807,7 @@ public class DatabaseDescriptor public static void storeMetadata() throws IOException { int cfId = 0; - Set tables = tableToCFMetaDataMap_.keySet(); + Set tables = tables_.keySet(); for (String table : tables) { @@ -738,7 +816,7 @@ public class DatabaseDescriptor { tmetadata = Table.TableMetadata.instance(table); /* Column families associated with this table */ - Map columnFamilies = tableToCFMetaDataMap_.get(table); + Map columnFamilies = tables_.get(table).cfMetaData; for (String columnFamily : columnFamilies.keySet()) { @@ -760,12 +838,12 @@ public class DatabaseDescriptor public static IEndPointSnitch getEndPointSnitch(String table) { - return endPointSnitches_.get(table); + return tables_.get(table).epSnitch; } - public static Class getReplicaPlacementStrategyClass(String table) + public static Class getReplicaPlacementStrategyClass(String table) { - return replicationStrategyClasses_.get(table); + return tables_.get(table).repStratClass; } public static String getJobTrackerAddress() @@ -820,7 +898,9 @@ public class DatabaseDescriptor public static Map getTableMetaData(String tableName) { assert tableName != null; - return tableToCFMetaDataMap_.get(tableName); + KSMetaData ksm = tables_.get(tableName); + assert ksm != null; + return Collections.unmodifiableMap(ksm.cfMetaData); } /* @@ -831,11 +911,10 @@ public class DatabaseDescriptor public static CFMetaData getCFMetaData(String tableName, String cfName) { assert tableName != null; - Map cfInfo = tableToCFMetaDataMap_.get(tableName); - if (cfInfo == null) + KSMetaData ksm = tables_.get(tableName); + if (ksm == null) return null; - - return cfInfo.get(cfName); + return ksm.cfMetaData.get(cfName); } public static String getColumnType(String tableName, String cfName) @@ -850,12 +929,12 @@ public class DatabaseDescriptor public static Set getTables() { - return tables_; + return tables_.keySet(); } public static List getNonSystemTables() { - List tables = new ArrayList(tables_); + List tables = new ArrayList(tables_.keySet()); tables.remove(Table.SYSTEM_TABLE); return Collections.unmodifiableList(tables); } @@ -877,12 +956,12 @@ public class DatabaseDescriptor public static int getReplicationFactor(String table) { - return replicationFactors_.get(table); + return tables_.get(table).replicationFactor; } public static int getQuorum(String table) { - return (replicationFactors_.get(table) / 2) + 1; + return (tables_.get(table).replicationFactor / 2) + 1; } public static long getRpcTimeout() @@ -1006,21 +1085,20 @@ public class DatabaseDescriptor return getCFMetaData(tableName, cfName).subcolumnComparator; } - public static Map> getTableToColumnFamilyMap() - { - return tableToCFMetaDataMap_; - } - public static double getKeysCachedFraction(String tableName, String columnFamilyName) { - Double v = tableKeysCachedFractions_.get(Pair.create(tableName, columnFamilyName)); - return v == null ? 0.01 : v; + CFMetaData cfm = getCFMetaData(tableName, columnFamilyName); + if (cfm == null) + return 0.01d; + return cfm.keysCachedFraction; } public static double getRowsCachedFraction(String tableName, String columnFamilyName) { - Double v = tableRowCacheSizes.get(Pair.create(tableName, columnFamilyName)); - return v == null ? 0 : v; + CFMetaData cfm = getCFMetaData(tableName, columnFamilyName); + if (cfm == null) + return 0.01d; + return cfm.rowCacheSize; } private static class ConfigurationException extends Exception @@ -1099,13 +1177,4 @@ public class DatabaseDescriptor { return autoBootstrap_; } - - /** - * For testing purposes. - */ - static void setReplicationFactorUnsafe(String table, int factor) - { - replicationFactors_.remove(table); - replicationFactors_.put(table, factor); - } } diff --git a/src/java/org/apache/cassandra/config/KSMetaData.java b/src/java/org/apache/cassandra/config/KSMetaData.java new file mode 100644 index 0000000000..2ed56364f6 --- /dev/null +++ b/src/java/org/apache/cassandra/config/KSMetaData.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.config; + +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.IEndPointSnitch; +import org.apache.cassandra.utils.FBUtilities; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +public final class KSMetaData +{ + public final String name; + public final Class repStratClass; + public final int replicationFactor; + public final IEndPointSnitch epSnitch; + public final Map cfMetaData = new HashMap(); + + KSMetaData(String name, Class repStratClass, int replicationFactor, IEndPointSnitch epSnitch) + { + this.name = name; + this.repStratClass = repStratClass; + this.replicationFactor = replicationFactor; + this.epSnitch = epSnitch; + } + + public boolean equals(Object obj) + { + if (obj == null) + return false; + if (!(obj instanceof KSMetaData)) + return false; + KSMetaData other = (KSMetaData)obj; + return other.name.equals(name) + && FBUtilities.equals(other.repStratClass, repStratClass) + && other.replicationFactor == replicationFactor + && sameEpSnitch(other, this) + && other.cfMetaData.size() == cfMetaData.size() + && other.cfMetaData.equals(cfMetaData); + } + + // epsnitches generally have no state, so comparing class names is sufficient. + private static boolean sameEpSnitch(KSMetaData a, KSMetaData b) + { + if (a.epSnitch == null && b.epSnitch == null) + return true; + else if (a.epSnitch == null && b.epSnitch != null) + return false; + else if (a.epSnitch != null && b.epSnitch == null) + return false; + else + return a.epSnitch.getClass().getName().equals(b.epSnitch.getClass().getName()); + } + + public static byte[] serialize(KSMetaData ksm) throws IOException + { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + DataOutputStream dout = new DataOutputStream(bout); + dout.writeUTF(ksm.name); + dout.writeBoolean(ksm.repStratClass != null); + if (ksm.repStratClass != null) + dout.writeUTF(ksm.repStratClass.getName()); + dout.writeInt(ksm.replicationFactor); + dout.writeBoolean(ksm.epSnitch != null); + if (ksm.epSnitch != null) + dout.writeUTF(ksm.epSnitch.getClass().getName()); + dout.writeInt(ksm.cfMetaData.size()); + for (CFMetaData cfm : ksm.cfMetaData.values()) + dout.write(CFMetaData.serialize(cfm)); + dout.close(); + return bout.toByteArray(); + } + + public static KSMetaData deserialize(InputStream in) throws IOException + { + DataInputStream din = new DataInputStream(in); + String name = din.readUTF(); + Class repStratClass = null; + try + { + repStratClass = din.readBoolean() ? (Class)Class.forName(din.readUTF()) : null; + } + catch (Exception ex) + { + throw new IOException(ex); + } + int replicationFactor = din.readInt(); + IEndPointSnitch epSnitch = null; + try + { + epSnitch = din.readBoolean() ? (IEndPointSnitch)Class.forName(din.readUTF()).newInstance() : null; + } + catch (Exception ex) + { + throw new IOException(ex); + } + int cfsz = din.readInt(); + KSMetaData ksm = new KSMetaData(name, repStratClass, replicationFactor, epSnitch); + for (int i = 0; i < cfsz; i++) + { + try + { + CFMetaData cfm = CFMetaData.deserialize(din); + ksm.cfMetaData.put(cfm.cfName, cfm); + } + catch (IOException ex) + { + System.err.println(ksm.name); + throw ex; + } + } + return ksm; + } +} diff --git a/src/java/org/apache/cassandra/db/marshal/AbstractType.java b/src/java/org/apache/cassandra/db/marshal/AbstractType.java index b3cf7b1b14..651fdac187 100644 --- a/src/java/org/apache/cassandra/db/marshal/AbstractType.java +++ b/src/java/org/apache/cassandra/db/marshal/AbstractType.java @@ -86,4 +86,12 @@ public abstract class AbstractType implements Comparator } return builder.toString(); } + + public final boolean equals(Object obj) + { + if (obj == null) + return false; + else + return obj.getClass().getName().equals(getClass().getName()); + } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index e4229fc0b0..60b7df8754 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -250,13 +250,13 @@ public class StorageService implements IEndPointStateChangeSubscriber, StorageSe public static AbstractReplicationStrategy getReplicationStrategy(TokenMetadata tokenMetadata, String table) { AbstractReplicationStrategy replicationStrategy = null; - Class cls = DatabaseDescriptor.getReplicaPlacementStrategyClass(table); + Class cls = DatabaseDescriptor.getReplicaPlacementStrategyClass(table); if (cls == null) throw new RuntimeException(String.format("No replica strategy configured for %s", table)); Class [] parameterTypes = new Class[] { TokenMetadata.class, IEndPointSnitch.class}; try { - Constructor constructor = cls.getConstructor(parameterTypes); + Constructor constructor = cls.getConstructor(parameterTypes); replicationStrategy = constructor.newInstance(tokenMetadata, DatabaseDescriptor.getEndPointSnitch(table)); } catch (Exception e) diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index e3b06ba5e0..dd7b56f734 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -415,4 +415,16 @@ public class FBUtilities Collections.sort(keys); } } + + public static boolean equals(Object a, Object b) + { + if (a == null && b == null) + return true; + else if (a != null && b == null) + return false; + else if (a == null && b != null) + return false; + else + return a.equals(b); + } } diff --git a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java index e7ba4c74f4..985046a2d6 100644 --- a/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java +++ b/test/unit/org/apache/cassandra/config/DatabaseDescriptorTest.java @@ -21,6 +21,9 @@ package org.apache.cassandra.config; import static org.junit.Assert.assertNotNull; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.IOException; + public class DatabaseDescriptorTest { @Test @@ -29,13 +32,32 @@ public class DatabaseDescriptorTest assertNotNull(DatabaseDescriptor.getConfigFileName(), "DatabaseDescriptor should always be able to return the file name of the config file"); } - /** - * Allow modification of replicationFactor for testing purposes. - * TODO: A more general method of property modification would be useful, but - * will probably have to wait for a refactor away from all the static fields. - */ - public static void setReplicationFactor(String table, int factor) + @Test + public void testCFMetaDataSerialization() throws IOException { - DatabaseDescriptor.setReplicationFactorUnsafe(table, factor); + // test serialization of all defined test CFs. + for (String table : DatabaseDescriptor.getNonSystemTables()) + { + for (CFMetaData cfm : DatabaseDescriptor.getTableMetaData(table).values()) + { + byte[] ser = CFMetaData.serialize(cfm); + CFMetaData cfmDupe = CFMetaData.deserialize(new ByteArrayInputStream(ser)); + assert cfmDupe != null; + assert cfmDupe.equals(cfm); + } + } + + } + + @Test + public void testKSMetaDataSerialization() throws IOException + { + for (KSMetaData ksm : DatabaseDescriptor.tables_.values()) + { + byte[] ser = KSMetaData.serialize(ksm); + KSMetaData ksmDupe = KSMetaData.deserialize(new ByteArrayInputStream(ser)); + assert ksmDupe != null; + assert ksmDupe.equals(ksm); + } } } diff --git a/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java b/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java index de1d1b2d09..bd4d644829 100644 --- a/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java +++ b/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java @@ -61,9 +61,7 @@ public class AntiEntropyServiceTest extends CleanupHelper if (!initialized) { LOCAL = FBUtilities.getLocalAddress(); - tablename = DatabaseDescriptor.getTables().iterator().next(); - // bump the replication factor so that local overlaps with REMOTE below - DatabaseDescriptorTest.setReplicationFactor(tablename, 2); + tablename = "Keyspace4"; StorageService.instance.initServer(); // generate a fake endpoint for which we can spoof receiving/sending trees