Merge branch 'cassandra-1.1.0' into cassandra-1.1

CHANGES.txt

@@ -9,6 +9,7 @@
  * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042)
  * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087)
  * fix KEYS index from skipping results (CASSANDRA-3996)
+ * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076)
 
 
 1.1-beta2
@@ -56,6 +57,8 @@ Merged from 1.0:
  * ensure that directory is selected for compaction for user-defined
    tasks and upgradesstables (CASSANDRA-3985)
  * fix NPE on invalid CQL delete command (CASSANDRA-3755)
+ * allow custom types in CLI's assume command (CASSANDRA-4081)
+ * Fix totalBytes count for parallel compactions (CASSANDRA-3758)
 
 
 1.1-beta1

NEWS.txt

@@ -48,7 +48,8 @@ Upgrading
       + Prior to 1.1, you could use KEY as the primary key name in some
         select statements, even if the PK was actually given a different
         name.  In 1.1+ you must use the defined PK name.
-        
+    - The sliced_buffer_size_in_kb option has been removed from the
+      cassandra.yaml config file (this option was a no-op since 1.0).
 
 Features
 --------

conf/cassandra.yaml

@@ -223,10 +223,6 @@ concurrent_writes: 32
 # the maximum number of secondary indexes created on a single CF.
 memtable_flush_queue_size: 4
 
-# Buffer size to use when performing contiguous column slices. 
-# Increase this to the size of the column slices you typically perform
-sliced_buffer_size_in_kb: 64
-
 # Whether to, when doing sequential writing, fsync() at intervals in
 # order to force the operating system to flush the dirty
 # buffers. Enable this to avoid sudden dirty buffer flushing from

examples/client_only/conf/cassandra.yaml

@@ -150,10 +150,6 @@ concurrent_writes: 32
 # By default this will be set to the amount of data directories defined.
 #memtable_flush_writers: 1
 
-# Buffer size to use when performing contiguous column slices. 
-# Increase this to the size of the column slices you typically perform
-sliced_buffer_size_in_kb: 64
-
 # TCP port, for commands and data
 storage_port: 7000
 

src/java/org/apache/cassandra/cli/Cli.g

@@ -303,8 +303,8 @@ truncateStatement
     ;
 
 assumeStatement
-    : ASSUME columnFamily assumptionElement=Identifier 'AS' defaultType=Identifier
-        -> ^(NODE_ASSUME columnFamily $assumptionElement $defaultType)
+    : ASSUME columnFamily assumptionElement=Identifier 'AS' entityName
+        -> ^(NODE_ASSUME columnFamily $assumptionElement entityName)
     ;
 
 consistencyLevelStatement

src/java/org/apache/cassandra/cli/CliClient.java

@@ -1506,17 +1506,25 @@ public class CliClient
         AbstractType<?> comparator;
 
         // Could be UTF8Type, IntegerType, LexicalUUIDType etc.
-        String defaultType = statement.getChild(2).getText();
+        String defaultType = CliUtils.unescapeSQLString(statement.getChild(2).getText());
 
         try
         {
-            comparator = Function.valueOf(defaultType.toUpperCase()).getValidator();
+            comparator = TypeParser.parse(defaultType);
         }
-        catch (Exception e)
+        catch (ConfigurationException e)
         {
-            String functions = Function.getFunctionNames();
-            sessionState.out.println("Type '" + defaultType + "' was not found. Available: " + functions);
-            return;
+            try
+            {
+                comparator = Function.valueOf(defaultType.toUpperCase()).getValidator();
+            }
+            catch (Exception ne)
+            {
+                String functions = Function.getFunctionNames();
+                sessionState.out.println("Type '" + defaultType + "' was not found. Available: " + functions
+                                         + " Or any class which extends o.a.c.db.marshal.AbstractType.");
+                return;
+            }
         }
 
         // making string representation look property e.g. o.a.c.db.marshal.UTF8Type

src/java/org/apache/cassandra/config/Config.java

@@ -61,8 +61,6 @@ public class Config
     public Integer memtable_flush_writers = null; // will get set to the length of data dirs in DatabaseDescriptor
     public Integer memtable_total_space_in_mb;
 
-    public Integer sliced_buffer_size_in_kb = 64;
-
     public Integer storage_port = 7000;
     public Integer ssl_storage_port = 7001;
     public String listen_address;

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -864,16 +864,6 @@ public class DatabaseDescriptor
         return indexAccessMode;
     }
 
-    public static int getIndexedReadBufferSizeInKB()
-    {
-        return conf.column_index_size_in_kb;
-    }
-
-    public static int getSlicedReadBufferSizeInKB()
-    {
-        return conf.sliced_buffer_size_in_kb;
-    }
-
     public static boolean isSnapshotBeforeCompaction()
     {
         return conf.snapshot_before_compaction;

src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java

@@ -58,7 +58,7 @@ public class SSTableNamesIterator extends SimpleAbstractColumnIterator implement
         this.columns = columns;
         this.key = key;
 
-        FileDataInput file = sstable.getFileDataInput(key, DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024);
+        FileDataInput file = sstable.getFileDataInput(key);
         if (file == null)
             return;
 

src/java/org/apache/cassandra/db/columniterator/SSTableSliceIterator.java

@@ -45,7 +45,7 @@ public class SSTableSliceIterator implements IColumnIterator
     public SSTableSliceIterator(SSTableReader sstable, DecoratedKey key, ByteBuffer startColumn, ByteBuffer finishColumn, boolean reversed)
     {
         this.key = key;
-        fileToClose = sstable.getFileDataInput(this.key, DatabaseDescriptor.getSlicedReadBufferSizeInKB() * 1024);
+        fileToClose = sstable.getFileDataInput(this.key);
         if (fileToClose == null)
             return;
 

src/java/org/apache/cassandra/db/compaction/AbstractCompactionIterable.java

@@ -41,15 +41,24 @@ public abstract class AbstractCompactionIterable extends CompactionInfo.Holder i
 
     protected final OperationType type;
     protected final CompactionController controller;
-    protected long totalBytes;
+    protected final long totalBytes;
     protected volatile long bytesRead;
+    protected final List<SSTableScanner> scanners;
 
     protected final Throttle throttle;
 
-    public AbstractCompactionIterable(CompactionController controller, OperationType type)
+    public AbstractCompactionIterable(CompactionController controller, OperationType type, List<SSTableScanner> scanners)
     {
         this.controller = controller;
         this.type = type;
+        this.scanners = scanners;
+        this.bytesRead = 0;
+
+        long bytes = 0;
+        for (SSTableScanner scanner : scanners)
+            bytes += scanner.getFileLength();
+        this.totalBytes = bytes;
+
         this.throttle = new Throttle(toString(), new Throttle.ThroughputFunction()
         {
             /** @return Instantaneous throughput target in bytes per millisecond. */

src/java/org/apache/cassandra/db/compaction/CompactionIterable.java

@@ -41,7 +41,6 @@ public class CompactionIterable extends AbstractCompactionIterable
     private static Logger logger = LoggerFactory.getLogger(CompactionIterable.class);
 
     private long row;
-    private final List<SSTableScanner> scanners;
 
     private static final Comparator<IColumnIterator> comparator = new Comparator<IColumnIterator>()
     {
@@ -58,12 +57,8 @@ public class CompactionIterable extends AbstractCompactionIterable
 
     protected CompactionIterable(OperationType type, List<SSTableScanner> scanners, CompactionController controller)
     {
-        super(controller, type);
-        this.scanners = scanners;
+        super(controller, type, scanners);
         row = 0;
-        totalBytes = bytesRead = 0;
-        for (SSTableScanner scanner : scanners)
-            totalBytes += scanner.getFileLength();
     }
 
     protected static List<SSTableScanner> getScanners(Iterable<SSTableReader> sstables) throws IOException

src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java

@@ -59,7 +59,6 @@ public class ParallelCompactionIterable extends AbstractCompactionIterable
 {
     private static Logger logger = LoggerFactory.getLogger(ParallelCompactionIterable.class);
 
-    private final List<SSTableScanner> scanners;
     private final int maxInMemorySize;
 
     public ParallelCompactionIterable(OperationType type, Iterable<SSTableReader> sstables, CompactionController controller) throws IOException
@@ -74,8 +73,7 @@ public class ParallelCompactionIterable extends AbstractCompactionIterable
 
     protected ParallelCompactionIterable(OperationType type, List<SSTableScanner> scanners, CompactionController controller, int maxInMemorySize)
     {
-        super(controller, type);
-        this.scanners = scanners;
+        super(controller, type, scanners);
         this.maxInMemorySize = maxInMemorySize;
     }
 

src/java/org/apache/cassandra/io/sstable/SSTableReader.java

@@ -849,7 +849,7 @@ public class SSTableReader extends SSTable
         return new SSTableBoundedScanner(this, true, range);
     }
 
-    public FileDataInput getFileDataInput(DecoratedKey decoratedKey, int bufferSize)
+    public FileDataInput getFileDataInput(DecoratedKey decoratedKey)
     {
         long position = getPosition(decoratedKey, Operator.EQ);
         if (position < 0)

src/java/org/apache/cassandra/net/IncomingTcpConnection.java

@@ -46,7 +46,6 @@ public class IncomingTcpConnection extends Thread
     {
         assert socket != null;
         this.socket = socket;
-        from = socket.getInetAddress(); // maximize chance of this not being nulled by disconnect
     }
 
     /**
@@ -92,6 +91,7 @@ public class IncomingTcpConnection extends Thread
             input = new DataInputStream(new BufferedInputStream(socket.getInputStream(), 4096));
             // Receive the first message to set the version.
             Message msg = receiveMessage(input, version);
+            from = msg.getFrom(); // why? see => CASSANDRA-4099
             if (version > MessagingService.version_)
             {
                 // save the endpoint so gossip will reconnect to it
@@ -100,7 +100,7 @@ public class IncomingTcpConnection extends Thread
             }
             else if (msg != null)
             {
-                Gossiper.instance.setVersion(msg.getFrom(), version);
+                Gossiper.instance.setVersion(from, version);
                 logger.debug("set version for {} to {}", from, version);
             }
 

test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java

@@ -125,7 +125,7 @@ public class SSTableReaderTest extends SchemaLoader
         for (int j = 0; j < 100; j += 2)
         {
             DecoratedKey dk = Util.dk(String.valueOf(j));
-            FileDataInput file = sstable.getFileDataInput(dk, DatabaseDescriptor.getIndexedReadBufferSizeInKB() * 1024);
+            FileDataInput file = sstable.getFileDataInput(dk);
             DecoratedKey keyInDisk = SSTableReader.decodeKey(sstable.partitioner,
                                                              sstable.descriptor,
                                                              ByteBufferUtil.readWithShortLength(file));