diff --git a/CHANGES.txt b/CHANGES.txt index 46bee529ba..316007b6a6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -34,6 +34,8 @@ charset is not UTF8 (CASSANDRA-2367) * fix potential infinite loop in ByteBufferUtil.inputStream (CASSANDRA-2365) * allow negative numbers in the cli (CASSANDRA-2358) + * fix incorrect truncation of long to int when reading columns via block + index (CASSANDRA-2376) 0.7.4 diff --git a/conf/schema-sample.txt b/conf/schema-sample.txt index c108cf6c3d..df2b32038d 100644 --- a/conf/schema-sample.txt +++ b/conf/schema-sample.txt @@ -53,7 +53,8 @@ create column family Super3 and comment = 'A column family with supercolumns, whose column names are Longs (8 bytes)'; create column family Indexed1 - with default_validation_class = LongType + with comparator = UTF8Type + and default_validation_class = LongType and column_metadata = [{ column_name : birthdate, validation_class : LongType, diff --git a/src/java/org/apache/cassandra/cli/Cli.g b/src/java/org/apache/cassandra/cli/Cli.g index 9774960212..1941ae12e1 100644 --- a/src/java/org/apache/cassandra/cli/Cli.g +++ b/src/java/org/apache/cassandra/cli/Cli.g @@ -591,7 +591,6 @@ Alnum // syntactic Elements IntegerPositiveLiteral : Digit+ - | '-' Digit+ ; IntegerNegativeLiteral diff --git a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java index 105703625e..b264c6957d 100644 --- a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java +++ b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java @@ -39,6 +39,7 @@ import org.apache.cassandra.io.sstable.IndexHelper; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileMark; import org.apache.cassandra.io.sstable.SSTableReader; +import org.apache.cassandra.io.util.FileUtils; /** * This is a reader that finds the block for a starting column and returns @@ -172,8 +173,7 @@ class IndexedSliceReader extends AbstractIterator implements IColumnIte boolean outOfBounds = false; file.reset(mark); - long curOffset = file.skipBytes((int) curColPosition.offset); - assert curOffset == curColPosition.offset; + FileUtils.skipBytesFully(file, curColPosition.offset); while (file.bytesPastMark(mark) < curColPosition.offset + curColPosition.width && !outOfBounds) { IColumn column = emptyColumnFamily.getColumnSerializer().deserialize(file); diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java index 93e6a96ed0..6cbc64bae0 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java @@ -172,8 +172,7 @@ public class SSTableNamesIterator extends SimpleAbstractColumnIterator implement for (IndexHelper.IndexInfo indexInfo : ranges) { file.reset(mark); - long curOffsert = file.skipBytes((int) indexInfo.offset); - assert curOffsert == indexInfo.offset; + FileUtils.skipBytesFully(file, indexInfo.offset); // TODO only completely deserialize columns we are interested in while (file.bytesPastMark(mark) < indexInfo.offset + indexInfo.width) { diff --git a/src/java/org/apache/cassandra/hadoop/streaming/AvroOutputReader.java b/src/java/org/apache/cassandra/hadoop/streaming/AvroOutputReader.java index 3366886d2a..da903db3f8 100644 --- a/src/java/org/apache/cassandra/hadoop/streaming/AvroOutputReader.java +++ b/src/java/org/apache/cassandra/hadoop/streaming/AvroOutputReader.java @@ -32,6 +32,7 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.specific.SpecificDatumReader; import org.apache.cassandra.hadoop.avro.Mutation; import org.apache.cassandra.hadoop.avro.StreamingMutation; +import org.apache.cassandra.io.util.FileUtils; import org.apache.hadoop.streaming.PipeMapRed; import org.apache.hadoop.streaming.io.OutputReader; @@ -139,15 +140,8 @@ public class AvroOutputReader extends OutputReader> @Override public long skip(long n) throws IOException { - long skipped = 0; - while (n > 0) - { - // skip in batches up to max_int in size - int skip = (int)Math.min(Integer.MAX_VALUE, n); - skipped += in.skipBytes(skip); - n -= skip; - } - return skipped; + FileUtils.skipBytesFully(in, n); + return n; } } } diff --git a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java index 59cc68dad2..3144a68186 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java @@ -28,6 +28,7 @@ import java.util.List; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.utils.*; /** @@ -46,9 +47,7 @@ public class IndexHelper /* size of the bloom filter */ int size = in.readInt(); /* skip the serialized bloom filter */ - int skipped = in.skipBytes(size); - if (skipped != size) - throw new EOFException("attempted to skip " + size + " bytes but only skipped " + skipped); + FileUtils.skipBytesFully(in, size); } /** @@ -61,8 +60,7 @@ public class IndexHelper /* read only the column index list */ int columnIndexSize = file.readInt(); /* skip the column index data */ - if (file.skipBytes(columnIndexSize) != columnIndexSize) - throw new EOFException(); + FileUtils.skipBytesFully(file, columnIndexSize); } /** diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index 6b4d517622..4b56c28137 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -227,7 +227,7 @@ public abstract class SSTable while (ifile.getFilePointer() < BYTES_CAP && keys < SAMPLES_CAP) { ByteBufferUtil.skipShortLength(ifile); - ifile.skipBytes(8); + FileUtils.skipBytesFully(ifile, 8); keys++; } assert keys > 0 && ifile.getFilePointer() > 0 && ifile.length() > 0; diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java index 251d80ff2b..265768c5f5 100644 --- a/src/java/org/apache/cassandra/io/util/FileUtils.java +++ b/src/java/org/apache/cassandra/io/util/FileUtils.java @@ -210,4 +210,29 @@ public class FileUtils // The directory is now empty so now it can be smoked deleteWithConfirm(dir); } + + public static void skipBytesFully(DataInput in, int bytes) throws IOException + { + int n = 0; + while (n < bytes) + { + int skipped = in.skipBytes(bytes - n); + if (skipped == 0) + throw new EOFException("EOF after " + n + " bytes out of " + bytes); + n += skipped; + } + } + + public static void skipBytesFully(DataInput in, long bytes) throws IOException + { + long n = 0; + while (n < bytes) + { + int m = (int) Math.min(Integer.MAX_VALUE, bytes - n); + int skipped = in.skipBytes(m); + if (skipped == 0) + throw new EOFException("EOF after " + n + " bytes out of " + bytes); + n += skipped; + } + } } diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 1ade564541..d44b1d9dc1 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -31,6 +31,8 @@ import java.util.Arrays; import static com.google.common.base.Charsets.UTF_8; import org.apache.cassandra.io.util.FileDataInput; +import org.apache.cassandra.io.util.FileUtils; + import org.apache.commons.lang.ArrayUtils; /** @@ -373,12 +375,7 @@ public class ByteBufferUtil public static ByteBuffer skipShortLength(DataInput in) throws IOException { int skip = readShortLength(in); - while (skip > 0) - { - int skipped = in.skipBytes(skip); - if (skipped == 0) throw new EOFException(); - skip -= skipped; - } + FileUtils.skipBytesFully(in, skip); return null; }