diff --git a/CHANGES.txt b/CHANGES.txt index 0a44d21306..292f494eec 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042) * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087) * fix KEYS index from skipping results (CASSANDRA-3996) + * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076) 1.1-beta2 @@ -56,6 +57,8 @@ Merged from 1.0: * ensure that directory is selected for compaction for user-defined tasks and upgradesstables (CASSANDRA-3985) * fix NPE on invalid CQL delete command (CASSANDRA-3755) + * allow custom types in CLI's assume command (CASSANDRA-4081) + * Fix totalBytes count for parallel compactions (CASSANDRA-3758) 1.1-beta1 diff --git a/NEWS.txt b/NEWS.txt index 4d64428895..0b5b82bb9c 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -48,7 +48,8 @@ Upgrading + Prior to 1.1, you could use KEY as the primary key name in some select statements, even if the PK was actually given a different name. In 1.1+ you must use the defined PK name. - + - The sliced_buffer_size_in_kb option has been removed from the + cassandra.yaml config file (this option was a no-op since 1.0). Features -------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 866319cc19..1874f320b7 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -223,10 +223,6 @@ concurrent_writes: 32 # the maximum number of secondary indexes created on a single CF. memtable_flush_queue_size: 4 -# Buffer size to use when performing contiguous column slices. -# Increase this to the size of the column slices you typically perform -sliced_buffer_size_in_kb: 64 - # Whether to, when doing sequential writing, fsync() at intervals in # order to force the operating system to flush the dirty # buffers. Enable this to avoid sudden dirty buffer flushing from diff --git a/examples/client_only/conf/cassandra.yaml b/examples/client_only/conf/cassandra.yaml index 2d92794faf..701d1911df 100644 --- a/examples/client_only/conf/cassandra.yaml +++ b/examples/client_only/conf/cassandra.yaml @@ -150,10 +150,6 @@ concurrent_writes: 32 # By default this will be set to the amount of data directories defined. #memtable_flush_writers: 1 -# Buffer size to use when performing contiguous column slices. -# Increase this to the size of the column slices you typically perform -sliced_buffer_size_in_kb: 64 - # TCP port, for commands and data storage_port: 7000 diff --git a/src/java/org/apache/cassandra/cli/Cli.g b/src/java/org/apache/cassandra/cli/Cli.g index 7bf1ac2bd2..a8a369c872 100644 --- a/src/java/org/apache/cassandra/cli/Cli.g +++ b/src/java/org/apache/cassandra/cli/Cli.g @@ -303,8 +303,8 @@ truncateStatement ; assumeStatement - : ASSUME columnFamily assumptionElement=Identifier 'AS' defaultType=Identifier - -> ^(NODE_ASSUME columnFamily $assumptionElement $defaultType) + : ASSUME columnFamily assumptionElement=Identifier 'AS' entityName + -> ^(NODE_ASSUME columnFamily $assumptionElement entityName) ; consistencyLevelStatement diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java index 9a286fc032..72fb40dab0 100644 --- a/src/java/org/apache/cassandra/cli/CliClient.java +++ b/src/java/org/apache/cassandra/cli/CliClient.java @@ -1506,17 +1506,25 @@ public class CliClient AbstractType comparator; // Could be UTF8Type, IntegerType, LexicalUUIDType etc. - String defaultType = statement.getChild(2).getText(); + String defaultType = CliUtils.unescapeSQLString(statement.getChild(2).getText()); try { - comparator = Function.valueOf(defaultType.toUpperCase()).getValidator(); + comparator = TypeParser.parse(defaultType); } - catch (Exception e) + catch (ConfigurationException e) { - String functions = Function.getFunctionNames(); - sessionState.out.println("Type '" + defaultType + "' was not found. Available: " + functions); - return; + try + { + comparator = Function.valueOf(defaultType.toUpperCase()).getValidator(); + } + catch (Exception ne) + { + String functions = Function.getFunctionNames(); + sessionState.out.println("Type '" + defaultType + "' was not found. Available: " + functions + + " Or any class which extends o.a.c.db.marshal.AbstractType."); + return; + } } // making string representation look property e.g. o.a.c.db.marshal.UTF8Type diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 2bc34dce77..91b96f16d8 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -61,8 +61,6 @@ public class Config public Integer memtable_flush_writers = null; // will get set to the length of data dirs in DatabaseDescriptor public Integer memtable_total_space_in_mb; - public Integer sliced_buffer_size_in_kb = 64; - public Integer storage_port = 7000; public Integer ssl_storage_port = 7001; public String listen_address; diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 8c5997f658..792dc186a1 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -864,16 +864,6 @@ public class DatabaseDescriptor return indexAccessMode; } - public static int getIndexedReadBufferSizeInKB() - { - return conf.column_index_size_in_kb; - } - - public static int getSlicedReadBufferSizeInKB() - { - return conf.sliced_buffer_size_in_kb; - } - public static boolean isSnapshotBeforeCompaction() { return conf.snapshot_before_compaction; diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java index ac8938c8db..d5d999fbb9 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java @@ -58,7 +58,7 @@ public class SSTableNamesIterator extends SimpleAbstractColumnIterator implement this.columns = columns; this.key = key; - FileDataInput file = sstable.getFileDataInput(key, DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024); + FileDataInput file = sstable.getFileDataInput(key); if (file == null) return; diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java index 5c97729ad7..8e4fc0692b 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java @@ -45,7 +45,7 @@ public class SSTableSliceIterator implements IColumnIterator public SSTableSliceIterator(SSTableReader sstable, DecoratedKey key, ByteBuffer startColumn, ByteBuffer finishColumn, boolean reversed) { this.key = key; - fileToClose = sstable.getFileDataInput(this.key, DatabaseDescriptor.getSlicedReadBufferSizeInKB() * 1024); + fileToClose = sstable.getFileDataInput(this.key); if (fileToClose == null) return; diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java index 5e7b0888e9..e05a64c323 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java @@ -41,15 +41,24 @@ public abstract class AbstractCompactionIterable extends CompactionInfo.Holder i protected final OperationType type; protected final CompactionController controller; - protected long totalBytes; + protected final long totalBytes; protected volatile long bytesRead; + protected final List scanners; protected final Throttle throttle; - public AbstractCompactionIterable(CompactionController controller, OperationType type) + public AbstractCompactionIterable(CompactionController controller, OperationType type, List scanners) { this.controller = controller; this.type = type; + this.scanners = scanners; + this.bytesRead = 0; + + long bytes = 0; + for (SSTableScanner scanner : scanners) + bytes += scanner.getFileLength(); + this.totalBytes = bytes; + this.throttle = new Throttle(toString(), new Throttle.ThroughputFunction() { /** @return Instantaneous throughput target in bytes per millisecond. */ diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java index 2fd0240535..eb88489cb2 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterable.java @@ -41,7 +41,6 @@ public class CompactionIterable extends AbstractCompactionIterable private static Logger logger = LoggerFactory.getLogger(CompactionIterable.class); private long row; - private final List scanners; private static final Comparator comparator = new Comparator() { @@ -58,12 +57,8 @@ public class CompactionIterable extends AbstractCompactionIterable protected CompactionIterable(OperationType type, List scanners, CompactionController controller) { - super(controller, type); - this.scanners = scanners; + super(controller, type, scanners); row = 0; - totalBytes = bytesRead = 0; - for (SSTableScanner scanner : scanners) - totalBytes += scanner.getFileLength(); } protected static List getScanners(Iterable sstables) throws IOException diff --git a/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java b/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java index 7a901cc36c..764a54909c 100644 --- a/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java +++ b/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java @@ -59,7 +59,6 @@ public class ParallelCompactionIterable extends AbstractCompactionIterable { private static Logger logger = LoggerFactory.getLogger(ParallelCompactionIterable.class); - private final List scanners; private final int maxInMemorySize; public ParallelCompactionIterable(OperationType type, Iterable sstables, CompactionController controller) throws IOException @@ -74,8 +73,7 @@ public class ParallelCompactionIterable extends AbstractCompactionIterable protected ParallelCompactionIterable(OperationType type, List scanners, CompactionController controller, int maxInMemorySize) { - super(controller, type); - this.scanners = scanners; + super(controller, type, scanners); this.maxInMemorySize = maxInMemorySize; } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index 2715613ef7..8c0719e253 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -849,7 +849,7 @@ public class SSTableReader extends SSTable return new SSTableBoundedScanner(this, true, range); } - public FileDataInput getFileDataInput(DecoratedKey decoratedKey, int bufferSize) + public FileDataInput getFileDataInput(DecoratedKey decoratedKey) { long position = getPosition(decoratedKey, Operator.EQ); if (position < 0) diff --git a/src/java/org/apache/cassandra/net/IncomingTcpConnection.java b/src/java/org/apache/cassandra/net/IncomingTcpConnection.java index edae8767d3..f329636db6 100644 --- a/src/java/org/apache/cassandra/net/IncomingTcpConnection.java +++ b/src/java/org/apache/cassandra/net/IncomingTcpConnection.java @@ -46,7 +46,6 @@ public class IncomingTcpConnection extends Thread { assert socket != null; this.socket = socket; - from = socket.getInetAddress(); // maximize chance of this not being nulled by disconnect } /** @@ -92,6 +91,7 @@ public class IncomingTcpConnection extends Thread input = new DataInputStream(new BufferedInputStream(socket.getInputStream(), 4096)); // Receive the first message to set the version. Message msg = receiveMessage(input, version); + from = msg.getFrom(); // why? see => CASSANDRA-4099 if (version > MessagingService.version_) { // save the endpoint so gossip will reconnect to it @@ -100,7 +100,7 @@ public class IncomingTcpConnection extends Thread } else if (msg != null) { - Gossiper.instance.setVersion(msg.getFrom(), version); + Gossiper.instance.setVersion(from, version); logger.debug("set version for {} to {}", from, version); } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java index 102fccd45b..004dac812c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java @@ -125,7 +125,7 @@ public class SSTableReaderTest extends SchemaLoader for (int j = 0; j < 100; j += 2) { DecoratedKey dk = Util.dk(String.valueOf(j)); - FileDataInput file = sstable.getFileDataInput(dk, DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024); + FileDataInput file = sstable.getFileDataInput(dk); DecoratedKey keyInDisk = SSTableReader.decodeKey(sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));