diff --git a/build.xml b/build.xml index c34bcfc109..bc2bbfd99f 100644 --- a/build.xml +++ b/build.xml @@ -62,7 +62,6 @@ - @@ -402,6 +401,7 @@ + @@ -433,6 +433,7 @@ + + diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh index eb0553b61b..1655438a53 100644 --- a/conf/cassandra-env.sh +++ b/conf/cassandra-env.sh @@ -212,6 +212,7 @@ JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=1" JVM_OPTS="$JVM_OPTS -XX:CMSInitiatingOccupancyFraction=75" JVM_OPTS="$JVM_OPTS -XX:+UseCMSInitiatingOccupancyOnly" JVM_OPTS="$JVM_OPTS -XX:+UseTLAB" +JVM_OPTS="$JVM_OPTS -XX:CompileCommandFile=$CASSANDRA_CONF/hotspot_compiler" # note: bash evals '1.7.x' as > '1.7' so this is really a >= 1.7 jvm check if [ "$JVM_ARCH" = "64-Bit" ] ; then @@ -242,6 +243,9 @@ fi # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 # JVM_OPTS="$JVM_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1414" +# uncomment to have Cassandra JVM log internal method compilation (developers only) +# JVM_OPTS="$JVM_OPTS -XX:+UnlockDiagnosticVMOptions -XX:+LogCompilation" + # Prefer binding to IPv4 network intefaces (when net.ipv6.bindv6only=1). See # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6342561 (short version: # comment out this entry to enable IPv6 support). diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index f1e5576268..4d97e864d0 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -290,7 +290,7 @@ memtable_cleanup_threshold: 0.4 # heap_buffers: on heap nio buffers # offheap_buffers: off heap (direct) nio buffers # offheap_objects: native memory, eliminating nio buffer heap overhead -memtable_allocation_type: heap_buffers +memtable_allocation_type: offheap_objects # Total space to use for commitlogs. Since commitlog segments are # mmapped, and hence use up address space, the default size is 32 diff --git a/src/java/org/apache/cassandra/db/AbstractNativeCell.java b/src/java/org/apache/cassandra/db/AbstractNativeCell.java index f3128b8760..d79ad19667 100644 --- a/src/java/org/apache/cassandra/db/AbstractNativeCell.java +++ b/src/java/org/apache/cassandra/db/AbstractNativeCell.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.security.MessageDigest; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.cql3.ColumnIdentifier; @@ -30,6 +31,7 @@ import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.FastByteOperations; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.memory.*; @@ -285,6 +287,12 @@ public abstract class AbstractNativeCell extends AbstractCell implements CellNam } public ByteBuffer get(int i) + { + return get(i, null); + } + + @Inline + private ByteBuffer get(int i, AbstractAllocator copy) { // remember to take dense/sparse into account, and only return EOC when not dense int size = size(); @@ -292,7 +300,12 @@ public abstract class AbstractNativeCell extends AbstractCell implements CellNam int cellNamesOffset = nameDeltaOffset(size); int startDelta = i == 0 ? 0 : getShort(nameDeltaOffset(i)); int endDelta = i < size - 1 ? getShort(nameDeltaOffset(i + 1)) : valueStartOffset() - cellNamesOffset; - return getByteBuffer(cellNamesOffset + startDelta, endDelta - startDelta).order(ByteOrder.BIG_ENDIAN); + int length = endDelta - startDelta; + if (copy == null) + return getByteBuffer(cellNamesOffset + startDelta, length).order(ByteOrder.BIG_ENDIAN); + ByteBuffer result = copy.allocate(length); + FastByteOperations.UnsafeOperations.copy(null, peer + cellNamesOffset + startDelta, result, 0, length); + return result; } private static final ThreadLocal BUFFER = new ThreadLocal() @@ -496,12 +509,12 @@ public abstract class AbstractNativeCell extends AbstractCell implements CellNam switch (nametype()) { case SIMPLE_DENSE: - return CellNames.simpleDense(allocator.clone(get(0))); + return CellNames.simpleDense(get(0, allocator)); case COMPOUND_DENSE: r = new ByteBuffer[size()]; for (int i = 0; i < r.length; i++) - r[i] = allocator.clone(get(i)); + r[i] = get(i, allocator); return CellNames.compositeDense(r); case COMPOUND_SPARSE_STATIC: @@ -509,7 +522,7 @@ public abstract class AbstractNativeCell extends AbstractCell implements CellNam int clusteringSize = clusteringSize(); r = clusteringSize == 0 ? EMPTY : new ByteBuffer[clusteringSize()]; for (int i = 0; i < clusteringSize; i++) - r[i] = allocator.clone(get(i)); + r[i] = get(i, allocator); ByteBuffer nameBuffer = get(r.length); ColumnIdentifier name; @@ -645,4 +658,38 @@ public abstract class AbstractNativeCell extends AbstractCell implements CellNam checkPosition(offset, length); return MemoryUtil.getByteBuffer(peer + offset, length); } + + // requires isByteOrderComparable to be true. Compares the name components only; ; may need to compare EOC etc still + public final int compareTo(final Composite that) + { + int size = size(); + int size2 = that.size(); + int minSize = Math.min(size, size2); + int startDelta = 0; + int cellNamesOffset = nameDeltaOffset(size); + for (int i = 0 ; i < minSize ; i++) + { + int endDelta = i < size - 1 ? getShort(nameDeltaOffset(i + 1)) : valueStartOffset() - cellNamesOffset; + long offset = peer + cellNamesOffset + startDelta; + int length = endDelta - startDelta; + int cmp = FastByteOperations.UnsafeOperations.compareTo(null, offset, length, that.get(i)); + if (cmp != 0) + return cmp; + startDelta = endDelta; + } + + EOC eoc = that.eoc(); + if (size == size2) + return this.eoc().compareTo(eoc); + + return size < size2 ? this.eoc().prefixComparisonResult : -eoc.prefixComparisonResult; + } + + public final int compareToSimple(final Composite that) + { + assert size() == 1 && that.size() == 1; + int length = valueStartOffset() - nameDeltaOffset(1); + long offset = peer + nameDeltaOffset(1); + return FastByteOperations.UnsafeOperations.compareTo(null, offset, length, that.get(0)); + } } diff --git a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java index 8a2d43ebca..e92d5a3c40 100644 --- a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java +++ b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java @@ -27,6 +27,7 @@ import com.google.common.base.Function; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterables; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.composites.CellName; import org.apache.cassandra.db.composites.Composite; @@ -122,6 +123,7 @@ public class ArrayBackedSortedColumns extends ColumnFamily /** * synchronized so that concurrent (read-only) accessors don't mess the internal state. */ + @Inline private synchronized void sortCells() { if (isSorted) @@ -129,7 +131,7 @@ public class ArrayBackedSortedColumns extends ColumnFamily Comparator comparator = reversed ? getComparator().columnReverseComparator() - : getComparator().columnComparator(); + : getComparator().columnComparator(false); // Sort the unsorted segment - will still potentially contain duplicate (non-reconciled) cells Arrays.sort(cells, sortedSize, size, comparator); diff --git a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java index 03cd0c56d0..559e75988c 100644 --- a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java +++ b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java @@ -40,6 +40,8 @@ import org.apache.cassandra.utils.btree.UpdateFunction; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.memory.HeapAllocator; import org.apache.cassandra.utils.memory.MemtableAllocator; +import org.apache.cassandra.utils.memory.NativeAllocator; +import org.apache.cassandra.utils.memory.NativePool; import static org.apache.cassandra.db.index.SecondaryIndexManager.Updater; @@ -186,7 +188,7 @@ public class AtomicBTreeColumns extends ColumnFamily deletionInfo = current.deletionInfo; } - Object[] tree = BTree.update(current.tree, metadata.comparator.columnComparator(), cm, cm.getColumnCount(), true, updater); + Object[] tree = BTree.update(current.tree, metadata.comparator.columnComparator(Memtable.MEMORY_POOL instanceof NativePool), cm, cm.getColumnCount(), true, updater); if (tree != null && refUpdater.compareAndSet(this, current, new Holder(tree, deletionInfo))) { @@ -226,14 +228,7 @@ public class AtomicBTreeColumns extends ColumnFamily private Comparator asymmetricComparator() { - final Comparator cmp = metadata.comparator; - return new Comparator() - { - public int compare(Object o1, Object o2) - { - return cmp.compare((Composite) o1, ((Cell) o2).name()); - } - }; + return metadata.comparator.asymmetricColumnComparator(Memtable.MEMORY_POOL instanceof NativePool); } public Iterable getColumnNames() diff --git a/src/java/org/apache/cassandra/db/BufferDecoratedKey.java b/src/java/org/apache/cassandra/db/BufferDecoratedKey.java index d375162240..8a1ad596ed 100644 --- a/src/java/org/apache/cassandra/db/BufferDecoratedKey.java +++ b/src/java/org/apache/cassandra/db/BufferDecoratedKey.java @@ -20,6 +20,8 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.utils.FastByteOperations; +import org.apache.cassandra.utils.memory.MemoryUtil; public class BufferDecoratedKey extends DecoratedKey { diff --git a/src/java/org/apache/cassandra/db/CollationController.java b/src/java/org/apache/cassandra/db/CollationController.java index 061537b325..c9cbf69a2a 100644 --- a/src/java/org/apache/cassandra/db/CollationController.java +++ b/src/java/org/apache/cassandra/db/CollationController.java @@ -28,6 +28,7 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy; import org.apache.cassandra.db.composites.CellName; @@ -67,6 +68,7 @@ public class CollationController * Once we have data for all requests columns that is newer than the newest remaining maxtimestamp, * we stop. */ + @Inline private ColumnFamily collectTimeOrderedData(boolean copyOnHeap) { final ColumnFamily container = ArrayBackedSortedColumns.factory.create(cfs.metadata, filter.filter.isReversed()); diff --git a/src/java/org/apache/cassandra/db/ConsistencyLevel.java b/src/java/org/apache/cassandra/db/ConsistencyLevel.java index 7583a90fa3..986a2aadfa 100644 --- a/src/java/org/apache/cassandra/db/ConsistencyLevel.java +++ b/src/java/org/apache/cassandra/db/ConsistencyLevel.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import com.google.common.collect.Iterables; +import net.nicoulaj.compilecommand.annotations.Inline; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -244,6 +245,7 @@ public enum ConsistencyLevel } } + @Inline public void assureSufficientLiveNodes(Keyspace keyspace, Iterable liveEndpoints) throws UnavailableException { int blockFor = blockFor(keyspace); diff --git a/src/java/org/apache/cassandra/db/DecoratedKey.java b/src/java/org/apache/cassandra/db/DecoratedKey.java index fb3d7ab768..604cbb7154 100644 --- a/src/java/org/apache/cassandra/db/DecoratedKey.java +++ b/src/java/org/apache/cassandra/db/DecoratedKey.java @@ -20,10 +20,13 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; import java.util.Comparator; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FastByteOperations; +import org.apache.cassandra.utils.memory.MemoryUtil; /** * Represents a decorated key, handy for certain operations diff --git a/src/java/org/apache/cassandra/db/NativeDecoratedKey.java b/src/java/org/apache/cassandra/db/NativeDecoratedKey.java index 52aa50c7dc..79dc53f765 100644 --- a/src/java/org/apache/cassandra/db/NativeDecoratedKey.java +++ b/src/java/org/apache/cassandra/db/NativeDecoratedKey.java @@ -20,13 +20,14 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.utils.FastByteOperations; import org.apache.cassandra.utils.concurrent.OpOrder; import org.apache.cassandra.utils.memory.MemoryUtil; import org.apache.cassandra.utils.memory.NativeAllocator; public class NativeDecoratedKey extends DecoratedKey { - private final long peer; + final long peer; public NativeDecoratedKey(Token token, NativeAllocator allocator, OpOrder.Group writeOp, ByteBuffer key) { diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCType.java b/src/java/org/apache/cassandra/db/composites/AbstractCType.java index e299e42936..491a9d13ac 100644 --- a/src/java/org/apache/cassandra/db/composites/AbstractCType.java +++ b/src/java/org/apache/cassandra/db/composites/AbstractCType.java @@ -22,7 +22,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Comparator; +import org.apache.cassandra.db.Cell; import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.NativeCell; import org.apache.cassandra.db.RangeTombstone; import org.apache.cassandra.db.RowIndexEntry; import org.apache.cassandra.db.TypeSizes; @@ -39,6 +41,39 @@ import static org.apache.cassandra.io.sstable.IndexHelper.IndexInfo; public abstract class AbstractCType implements CType { + static final Comparator rightNativeCell = new Comparator() + { + public int compare(Cell o1, Cell o2) + { + return -((NativeCell) o2).compareTo(o1.name()); + } + }; + + static final Comparator neitherNativeCell = new Comparator() + { + public int compare(Cell o1, Cell o2) + { + return compareUnsigned(o1.name(), o2.name()); + } + }; + + // only one or the other of these will ever be used + static final Comparator asymmetricRightNativeCell = new Comparator() + { + public int compare(Object o1, Object o2) + { + return -((NativeCell) o2).compareTo((Composite) o1); + } + }; + + static final Comparator asymmetricNeitherNativeCell = new Comparator() + { + public int compare(Object o1, Object o2) + { + return compareUnsigned((Composite) o1, ((Cell) o2).name()); + } + }; + private final Comparator reverseComparator; private final Comparator indexComparator; private final Comparator indexReverseComparator; @@ -60,11 +95,6 @@ public abstract class AbstractCType implements CType { public int compare(Composite c1, Composite c2) { - if (c1.isEmpty()) - return c2.isEmpty() ? 0 : -1; - if (c2.isEmpty()) - return 1; - return AbstractCType.this.compare(c2, c1); } }; @@ -102,30 +132,37 @@ public abstract class AbstractCType implements CType return isByteOrderComparable; } + static int compareUnsigned(Composite c1, Composite c2) + { + int s1 = c1.size(); + int s2 = c2.size(); + int minSize = Math.min(s1, s2); + + for (int i = 0; i < minSize; i++) + { + int cmp = ByteBufferUtil.compareUnsigned(c1.get(i), c2.get(i)); + if (cmp != 0) + return cmp; + } + + if (s1 == s2) + return c1.eoc().compareTo(c2.eoc()); + return s1 < s2 ? c1.eoc().prefixComparisonResult : -c2.eoc().prefixComparisonResult; + } + public int compare(Composite c1, Composite c2) { int s1 = c1.size(); int s2 = c2.size(); int minSize = Math.min(s1, s2); - if (isByteOrderComparable) + for (int i = 0; i < minSize; i++) { - for (int i = 0; i < minSize; i++) - { - int cmp = ByteBufferUtil.compareUnsigned(c1.get(i), c2.get(i)); - if (cmp != 0) - return cmp; - } - } - else - { - for (int i = 0; i < minSize; i++) - { - AbstractType comparator = subtype(i); - int cmp = comparator.compare(c1.get(i), c2.get(i)); - if (cmp != 0) - return cmp; - } + int cmp = isByteOrderComparable + ? ByteBufferUtil.compareUnsigned(c1.get(i), c2.get(i)) + : subtype(i).compare(c1.get(i), c2.get(i)); + if (cmp != 0) + return cmp; } if (s1 == s2) @@ -133,6 +170,20 @@ public abstract class AbstractCType implements CType return s1 < s2 ? c1.eoc().prefixComparisonResult : -c2.eoc().prefixComparisonResult; } + protected Comparator getByteOrderColumnComparator(boolean isRightNative) + { + if (isRightNative) + return rightNativeCell; + return neitherNativeCell; + } + + protected Comparator getByteOrderAsymmetricColumnComparator(boolean isRightNative) + { + if (isRightNative) + return asymmetricRightNativeCell; + return asymmetricNeitherNativeCell; + } + public void validate(Composite name) { ByteBuffer previous = null; diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java index 56be8cfc79..7570d2c0c8 100644 --- a/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/AbstractCellNameType.java @@ -40,8 +40,9 @@ import org.apache.cassandra.utils.ByteBufferUtil; public abstract class AbstractCellNameType extends AbstractCType implements CellNameType { - private final Comparator columnComparator; + final Comparator columnComparator; private final Comparator columnReverseComparator; + final Comparator asymmetricComparator; private final Comparator onDiskAtomComparator; private final ISerializer cellSerializer; @@ -60,6 +61,13 @@ public abstract class AbstractCellNameType extends AbstractCType implements Cell return AbstractCellNameType.this.compare(c1.name(), c2.name()); } }; + asymmetricComparator = new Comparator() + { + public int compare(Object c1, Object c2) + { + return AbstractCellNameType.this.compare((Composite) c1, ((Cell) c2).name()); + } + }; columnReverseComparator = new Comparator() { public int compare(Cell c1, Cell c2) @@ -125,9 +133,18 @@ public abstract class AbstractCellNameType extends AbstractCType implements Cell diskAtomFilterSerializer = new IDiskAtomFilter.Serializer(this); } - public Comparator columnComparator() + public final Comparator columnComparator(boolean isRightNative) { - return columnComparator; + if (!isByteOrderComparable) + return columnComparator; + return getByteOrderColumnComparator(isRightNative); + } + + public final Comparator asymmetricColumnComparator(boolean isRightNative) + { + if (!isByteOrderComparable) + return asymmetricComparator; + return getByteOrderAsymmetricColumnComparator(isRightNative); } public Comparator columnReverseComparator() diff --git a/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java index 641f14d851..b7482127be 100644 --- a/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/AbstractCompoundCellNameType.java @@ -20,7 +20,9 @@ package org.apache.cassandra.db.composites; import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Comparator; +import org.apache.cassandra.db.Cell; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.CompositeType; diff --git a/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java b/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java index 95386fdbaa..f9c9641d2c 100644 --- a/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/AbstractSimpleCellNameType.java @@ -20,7 +20,11 @@ package org.apache.cassandra.db.composites; import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Comparator; +import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.db.Cell; +import org.apache.cassandra.db.NativeCell; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.utils.ByteBufferUtil; @@ -28,6 +32,39 @@ public abstract class AbstractSimpleCellNameType extends AbstractCellNameType { protected final AbstractType type; + static final Comparator rightNativeCell = new Comparator() + { + public int compare(Cell o1, Cell o2) + { + return -((NativeCell) o2).compareToSimple(o1.name()); + } + }; + + static final Comparator neitherNativeCell = new Comparator() + { + public int compare(Cell o1, Cell o2) + { + return compareUnsigned(o1.name(), o2.name()); + } + }; + + // only one or the other of these will ever be used + static final Comparator asymmetricRightNativeCell = new Comparator() + { + public int compare(Object o1, Object o2) + { + return -((NativeCell) o2).compareToSimple((Composite) o1); + } + }; + + static final Comparator asymmetricNeitherNativeCell = new Comparator() + { + public int compare(Object o1, Object o2) + { + return compareUnsigned((Composite) o1, ((Cell) o2).name()); + } + }; + protected AbstractSimpleCellNameType(AbstractType type) { super(type.isByteOrderComparable()); @@ -44,31 +81,38 @@ public abstract class AbstractSimpleCellNameType extends AbstractCellNameType return 1; } + @Inline + static int compareUnsigned(Composite c1, Composite c2) + { + ByteBuffer b1 = c1.toByteBuffer(); + ByteBuffer b2 = c2.toByteBuffer(); + assert b1.hasRemaining() & b2.hasRemaining(); + return ByteBufferUtil.compareUnsigned(b1, b2); + } + public int compare(Composite c1, Composite c2) { - // This method assumes that simple composites never have an EOC != NONE. This assumption - // stands in particular on the fact that a Composites.EMPTY never has a non-NONE EOC. If - // this ever change, we'll need to update this. - if (isByteOrderComparable) - { - // toByteBuffer is always cheap for simple types, and we keep virtual method calls to a minimum: - // hasRemaining will always be inlined, as will most of the call-stack for BBU.compareUnsigned - ByteBuffer b1 = c1.toByteBuffer(); - ByteBuffer b2 = c2.toByteBuffer(); - if (!b1.hasRemaining() || !b2.hasRemaining()) - return b1.hasRemaining() ? 1 : (b2.hasRemaining() ? -1 : 0); - return ByteBufferUtil.compareUnsigned(b1, b2); - } - - boolean c1isEmpty = c1.isEmpty(); - boolean c2isEmpty = c2.isEmpty(); - if (c1isEmpty || c2isEmpty) - return !c1isEmpty ? 1 : (!c2isEmpty ? -1 : 0); + return compareUnsigned(c1, c2); + assert !(c1.isEmpty() | c2.isEmpty()); return type.compare(c1.get(0), c2.get(0)); } + protected Comparator getByteOrderColumnComparator(boolean isRightNative) + { + if (isRightNative) + return rightNativeCell; + return neitherNativeCell; + } + + protected Comparator getByteOrderAsymmetricColumnComparator(boolean isRightNative) + { + if (isRightNative) + return asymmetricRightNativeCell; + return asymmetricNeitherNativeCell; + } + public AbstractType subtype(int i) { if (i != 0) diff --git a/src/java/org/apache/cassandra/db/composites/CType.java b/src/java/org/apache/cassandra/db/composites/CType.java index c2516f5117..280f7af69b 100644 --- a/src/java/org/apache/cassandra/db/composites/CType.java +++ b/src/java/org/apache/cassandra/db/composites/CType.java @@ -55,6 +55,8 @@ public interface CType extends Comparator */ public int size(); + int compare(Composite o1, Composite o2); + /** * Gets a subtype of this CType. */ diff --git a/src/java/org/apache/cassandra/db/composites/CellNameType.java b/src/java/org/apache/cassandra/db/composites/CellNameType.java index acc052764c..4f45d41949 100644 --- a/src/java/org/apache/cassandra/db/composites/CellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/CellNameType.java @@ -174,7 +174,8 @@ public interface CellNameType extends CType // Ultimately, those might be split into an IVersionedSerializer and an ISSTableSerializer public ISerializer cellSerializer(); - public Comparator columnComparator(); + public Comparator columnComparator(boolean isRightNative); + public Comparator asymmetricColumnComparator(boolean isRightNative); public Comparator columnReverseComparator(); public Comparator onDiskAtomComparator(); diff --git a/src/java/org/apache/cassandra/db/composites/Composites.java b/src/java/org/apache/cassandra/db/composites/Composites.java index 154e9f7fb1..f6626e0843 100644 --- a/src/java/org/apache/cassandra/db/composites/Composites.java +++ b/src/java/org/apache/cassandra/db/composites/Composites.java @@ -57,7 +57,10 @@ public abstract class Composites public ByteBuffer get(int i) { - throw new IndexOutOfBoundsException(); + if (i > 0) + throw new IndexOutOfBoundsException(); + + return ByteBufferUtil.EMPTY_BYTE_BUFFER; } public EOC eoc() diff --git a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java index 29e1617553..57eb5fd339 100644 --- a/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java +++ b/src/java/org/apache/cassandra/db/composites/CompoundSparseCellNameType.java @@ -41,7 +41,6 @@ public class CompoundSparseCellNameType extends AbstractCompoundCellNameType protected final Map internedIds; private final Composite staticPrefix; - private final boolean isByteOrderComparable; public CompoundSparseCellNameType(List> types) { @@ -64,7 +63,6 @@ public class CompoundSparseCellNameType extends AbstractCompoundCellNameType this.columnNameType = columnNameType; this.internedIds = internedIds; this.staticPrefix = makeStaticPrefix(clusteringType.size()); - this.isByteOrderComparable = isByteOrderComparable(fullType.types); } private static Composite makeStaticPrefix(int size) diff --git a/src/java/org/apache/cassandra/db/composites/SimpleCType.java b/src/java/org/apache/cassandra/db/composites/SimpleCType.java index 229d53828f..c824179994 100644 --- a/src/java/org/apache/cassandra/db/composites/SimpleCType.java +++ b/src/java/org/apache/cassandra/db/composites/SimpleCType.java @@ -18,6 +18,7 @@ package org.apache.cassandra.db.composites; import java.nio.ByteBuffer; +import java.util.Comparator; import java.util.List; import org.apache.cassandra.db.marshal.AbstractType; @@ -48,26 +49,13 @@ public class SimpleCType extends AbstractCType public int compare(Composite c1, Composite c2) { + if (isByteOrderComparable) + return AbstractSimpleCellNameType.compareUnsigned(c1, c2); + + assert !(c1.isEmpty() | c2.isEmpty()); // This method assumes that simple composites never have an EOC != NONE. This assumption // stands in particular on the fact that a Composites.EMPTY never has a non-NONE EOC. If // this ever change, we'll need to update this. - - if (isByteOrderComparable) - { - // toByteBuffer is always cheap for simple types, and we keep virtual method calls to a minimum: - // hasRemaining will always be inlined, as will most of the call-stack for BBU.compareUnsigned - ByteBuffer b1 = c1.toByteBuffer(); - ByteBuffer b2 = c2.toByteBuffer(); - if (!b1.hasRemaining() || !b2.hasRemaining()) - return b1.hasRemaining() ? 1 : (b2.hasRemaining() ? -1 : 0); - return ByteBufferUtil.compareUnsigned(b1, b2); - } - - boolean c1isEmpty = c1.isEmpty(); - boolean c2isEmpty = c2.isEmpty(); - if (c1isEmpty || c2isEmpty) - return !c1isEmpty ? 1 : (!c2isEmpty ? -1 : 0); - return type.compare(c1.get(0), c2.get(0)); } diff --git a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java b/src/java/org/apache/cassandra/db/filter/ColumnSlice.java index bca4743d24..7d470b300a 100644 --- a/src/java/org/apache/cassandra/db/filter/ColumnSlice.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnSlice.java @@ -55,7 +55,7 @@ public class ColumnSlice public boolean includes(Comparator cmp, Composite name) { - return cmp.compare(start, name) <= 0 && (finish.isEmpty() || cmp.compare(finish, name) >= 0); + return (start.isEmpty() || cmp.compare(start, name) <= 0) && (finish.isEmpty() || cmp.compare(finish, name) >= 0); } public boolean isBefore(Comparator cmp, Composite name) @@ -114,26 +114,32 @@ public class ColumnSlice * Validates that the provided slice array contains only non-overlapped slices valid for a query {@code reversed} * or not on a table using {@code comparator}. */ - public static boolean validateSlices(ColumnSlice[] slices, CellNameType comparator, boolean reversed) + public static boolean validateSlices(ColumnSlice[] slices, CellNameType type, boolean reversed) { - return validateSlices(slices, reversed ? comparator.reverseComparator() : comparator); - } + Comparator comparator = reversed ? type.reverseComparator() : type; - /** - * Validates that the provided slice array contains only non-overlapped slices in {@code comparator} order. - */ - public static boolean validateSlices(ColumnSlice[] slices, Comparator comparator) - { for (int i = 0; i < slices.length; i++) { - if (i > 0 && comparator.compare(slices[i-1].finish, slices[i].start) >= 0) - return false; + Composite start = slices[i].start; + Composite finish = slices[i].finish; - if (slices[i].finish.isEmpty()) - return i == slices.length - 1; + if (start.isEmpty() || finish.isEmpty()) + { + if (start.isEmpty() && i > 0) + return false; - if (comparator.compare(slices[i].start, slices[i].finish) > 0) - return false; + if (finish.isEmpty()) + return i == slices.length - 1; + } + else + { + // !finish.isEmpty() is imposed by prior loop + if (i > 0 && comparator.compare(slices[i - 1].finish, start) >= 0) + return false; + + if (comparator.compare(start, finish) > 0) + return false; + } } return true; } @@ -159,13 +165,21 @@ public class ColumnSlice @Override public int compare(ColumnSlice s1, ColumnSlice s2) { - int c = comparator.compare(s1.start, s2.start); - if (c != 0) - return c; + if (s1.start.isEmpty() || s2.start.isEmpty()) + { + if (s1.start.isEmpty() != s2.start.isEmpty()) + return s1.start.isEmpty() ? -1 : 1; + } + else + { + int c = comparator.compare(s1.start, s2.start); + if (c != 0) + return c; + } // For the finish, empty always means greater return s1.finish.isEmpty() || s2.finish.isEmpty() - ? s1.finish.isEmpty() ? 1 : s2.finish.isEmpty() ? -1 : 0 + ? (s1.finish.isEmpty() ? 1 : -1) : comparator.compare(s1.finish, s2.finish); } }); diff --git a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java index 0e0643f02f..77a5dc7460 100644 --- a/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java @@ -103,7 +103,7 @@ public class NamesQueryFilter implements IDiskAtomFilter public Comparator getColumnComparator(CellNameType comparator) { - return comparator.columnComparator(); + return comparator.columnComparator(false); } @Override diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java index e24f68b967..b8c3e2fc55 100644 --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java @@ -186,7 +186,7 @@ public class SliceQueryFilter implements IDiskAtomFilter public Comparator getColumnComparator(CellNameType comparator) { - return reversed ? comparator.columnReverseComparator() : comparator.columnComparator(); + return reversed ? comparator.columnReverseComparator() : comparator.columnComparator(false); } public void collectReducedColumns(ColumnFamily container, Iterator reducedColumns, int gcBefore, long now) diff --git a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java index 7099a151a2..b0bbfc48c7 100644 --- a/src/java/org/apache/cassandra/io/sstable/IndexHelper.java +++ b/src/java/org/apache/cassandra/io/sstable/IndexHelper.java @@ -111,8 +111,8 @@ public class IndexHelper */ public static int indexFor(Composite name, List indexList, CType comparator, boolean reversed, int lastIndex) { - if (name.isEmpty() && reversed) - return indexList.size() - 1; + if (name.isEmpty()) + return lastIndex >= 0 ? lastIndex : reversed ? indexList.size() - 1 : 0; if (lastIndex >= indexList.size()) return -1; diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java index 67dee818db..53064335a9 100644 --- a/src/java/org/apache/cassandra/io/util/Memory.java +++ b/src/java/org/apache/cassandra/io/util/Memory.java @@ -22,6 +22,7 @@ import java.nio.ByteOrder; import com.sun.jna.Native; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.utils.FastByteOperations; import org.apache.cassandra.utils.memory.MemoryUtil; import sun.misc.Unsafe; import sun.nio.ch.DirectBuffer; @@ -273,7 +274,7 @@ public class Memory long end = memoryOffset + count; checkPosition(end - 1); - unsafe.copyMemory(null, peer + memoryOffset, buffer, BYTE_ARRAY_BASE_OFFSET + bufferOffset, count); + FastByteOperations.UnsafeOperations.copy(null, peer + memoryOffset, buffer, bufferOffset, count); } private void checkPosition(long offset) diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 988c623868..8d20af4935 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -31,6 +31,7 @@ import com.google.common.base.Predicate; import com.google.common.cache.CacheLoader; import com.google.common.collect.*; import com.google.common.util.concurrent.Uninterruptibles; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1142,6 +1143,7 @@ public class StorageProxy implements StorageProxyMBean * Performs the actual reading of a row out of the StorageService, fetching * a specific set of column names from a given column family. */ + @Inline public static List read(List commands, ConsistencyLevel consistency_level) throws UnavailableException, IsBootstrappingException, ReadTimeoutException, InvalidRequestException { @@ -1220,6 +1222,7 @@ public class StorageProxy implements StorageProxyMBean * 4. If the digests (if any) match the data return the data * 5. else carry out read repair by getting data from all the nodes. */ + @Inline private static List fetchRows(List initialCommands, ConsistencyLevel consistencyLevel) throws UnavailableException, ReadTimeoutException { diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java index 86b5bee18b..0621db9948 100644 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@ -404,7 +404,7 @@ public class CassandraServer implements Cassandra.Iface if (metadata.isSuper()) { CellNameType columnType = new SimpleDenseCellNameType(metadata.comparator.subtype(parent.isSetSuper_column() ? 1 : 0)); - SortedSet s = new TreeSet(columnType); + SortedSet s = new TreeSet<>(columnType); for (ByteBuffer bb : predicate.column_names) s.add(columnType.cellFromByteBuffer(bb)); filter = SuperColumns.fromSCNamesFilter(metadata.comparator, parent.bufferForSuper_column(), new NamesQueryFilter(s)); @@ -2071,11 +2071,14 @@ public class CassandraServer implements Cassandra.Iface fixOptionalSliceParameters(request.getColumn_slices().get(i)); Composite start = metadata.comparator.fromByteBuffer(request.getColumn_slices().get(i).start); Composite finish = metadata.comparator.fromByteBuffer(request.getColumn_slices().get(i).finish); - int compare = metadata.comparator.compare(start, finish); - if (!request.reversed && compare > 0) - throw new InvalidRequestException(String.format("Column slice at index %d had start greater than finish", i)); - else if (request.reversed && compare < 0) - throw new InvalidRequestException(String.format("Reversed column slice at index %d had start less than finish", i)); + if (!start.isEmpty() && !finish.isEmpty()) + { + int compare = metadata.comparator.compare(start, finish); + if (!request.reversed && compare > 0) + throw new InvalidRequestException(String.format("Column slice at index %d had start greater than finish", i)); + else if (request.reversed && compare < 0) + throw new InvalidRequestException(String.format("Reversed column slice at index %d had start less than finish", i)); + } slices[i] = new ColumnSlice(start, finish); } diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java index 420a7764b1..62addfaad3 100644 --- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java +++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java @@ -32,6 +32,7 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.UUID; +import net.nicoulaj.compilecommand.annotations.Inline; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileUtils; @@ -78,16 +79,19 @@ public class ByteBufferUtil { public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); + @Inline public static int compareUnsigned(ByteBuffer o1, ByteBuffer o2) { return FastByteOperations.compareUnsigned(o1, o2); } + @Inline public static int compare(byte[] o1, ByteBuffer o2) { return FastByteOperations.compareUnsigned(o1, 0, o1.length, o2); } + @Inline public static int compare(ByteBuffer o1, byte[] o2) { return FastByteOperations.compareUnsigned(o1, o2, 0, o2.length); diff --git a/src/java/org/apache/cassandra/utils/FastByteOperations.java b/src/java/org/apache/cassandra/utils/FastByteOperations.java index e60b096872..1431fd3622 100644 --- a/src/java/org/apache/cassandra/utils/FastByteOperations.java +++ b/src/java/org/apache/cassandra/utils/FastByteOperations.java @@ -24,9 +24,9 @@ import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedAction; -import com.google.common.primitives.Longs; -import com.google.common.primitives.UnsignedBytes; +import com.google.common.primitives.*; +import net.nicoulaj.compilecommand.annotations.Inline; import sun.misc.Unsafe; /** @@ -34,39 +34,43 @@ import sun.misc.Unsafe; * This is borrowed and slightly modified from Guava's {@link UnsignedBytes} * class to be able to compare arrays that start at non-zero offsets. */ -class FastByteOperations +public class FastByteOperations { /** * Lexicographically compare two byte arrays. */ + @Inline public static int compareUnsigned(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { return BestHolder.BEST.compare(b1, s1, l1, b2, s2, l2); } + @Inline public static int compareUnsigned(ByteBuffer b1, byte[] b2, int s2, int l2) { return BestHolder.BEST.compare(b1, b2, s2, l2); } + @Inline public static int compareUnsigned(byte[] b1, int s1, int l1, ByteBuffer b2) { return -BestHolder.BEST.compare(b2, b1, s1, l1); } + @Inline public static int compareUnsigned(ByteBuffer b1, ByteBuffer b2) { - if (b1 == b2) - return 0; return BestHolder.BEST.compare(b1, b2); } + @Inline public static void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); } + @Inline public static void copy(ByteBuffer src, int srcPosition, ByteBuffer trg, int trgPosition, int length) { BestHolder.BEST.copy(src, srcPosition, trg, trgPosition, length); @@ -180,122 +184,97 @@ class FastByteOperations } } - static final boolean littleEndian = - ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); - - /** - * Returns true if x1 is less than x2, when both values are treated as - * unsigned. - */ - static boolean lessThanUnsigned(long x1, long x2) - { - return (x1 + Long.MIN_VALUE) < (x2 + Long.MIN_VALUE); - } + static final boolean BIG_ENDIAN = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); public int compare(byte[] buffer1, int offset1, int length1, byte[] buffer2, int offset2, int length2) { - return compareTo(buffer1, BYTE_ARRAY_BASE_OFFSET + offset1, length1, buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2); + return compareTo(buffer1, BYTE_ARRAY_BASE_OFFSET + offset1, length1, + buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2); } public int compare(ByteBuffer buffer1, byte[] buffer2, int offset2, int length2) { - final Object obj1; - final long offset1; - final int length1; + Object obj1; + long offset1; if (buffer1.hasArray()) { obj1 = buffer1.array(); - offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset() + buffer1.position(); - length1 = buffer1.remaining(); + offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset(); } else { obj1 = null; - offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET) + buffer1.position(); - length1 = buffer1.remaining(); + offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET); + } + int length1; + { + int position = buffer1.position(); + int limit = buffer1.limit(); + length1 = limit - position; + offset1 += position; } return compareTo(obj1, offset1, length1, buffer2, BYTE_ARRAY_BASE_OFFSET + offset2, length2); } public int compare(ByteBuffer buffer1, ByteBuffer buffer2) { - final Object obj1, obj2; - final long offset1, offset2; - final int length1, length2; - if (buffer1.hasArray()) - { - obj1 = buffer1.array(); - offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset() + buffer1.position(); - length1 = buffer1.remaining(); - } - else - { - obj1 = null; - offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET) + buffer1.position(); - length1 = buffer1.remaining(); - } - if (buffer2.hasArray()) - { - obj2 = buffer2.array(); - offset2 = BYTE_ARRAY_BASE_OFFSET + buffer2.arrayOffset() + buffer2.position(); - length2 = buffer2.remaining(); - } - else - { - obj2 = null; - offset2 = theUnsafe.getLong(buffer2, DIRECT_BUFFER_ADDRESS_OFFSET) + buffer2.position(); - length2 = buffer2.remaining(); - } - return compareTo(obj1, offset1, length1, obj2, offset2, length2); + return compareTo(buffer1, buffer2); } public void copy(ByteBuffer src, int srcPosition, byte[] trg, int trgPosition, int length) { if (src.hasArray()) - { System.arraycopy(src.array(), src.arrayOffset() + srcPosition, trg, trgPosition, length); - return; - } - long srcOffset = srcPosition + theUnsafe.getLong(src, DIRECT_BUFFER_ADDRESS_OFFSET); - copy(null, srcOffset, trg, BYTE_ARRAY_BASE_OFFSET + trgPosition, length); + else + copy(null, srcPosition + theUnsafe.getLong(src, DIRECT_BUFFER_ADDRESS_OFFSET), trg, trgPosition, length); } public void copy(ByteBuffer srcBuf, int srcPosition, ByteBuffer trgBuf, int trgPosition, int length) { - if (srcBuf.hasArray() && trgBuf.hasArray()) - { - System.arraycopy(srcBuf.array(), srcBuf.arrayOffset() + srcPosition, trgBuf.array(), trgBuf.arrayOffset() + trgPosition, length); - return; - } - Object src, trg; - long srcOffset, trgOffset; - if (srcBuf.isDirect()) - { - srcOffset = srcPosition + theUnsafe.getLong(srcBuf, DIRECT_BUFFER_ADDRESS_OFFSET); - src = null; - } - else + Object src; + long srcOffset; + if (srcBuf.hasArray()) { src = srcBuf.array(); - srcOffset = BYTE_ARRAY_BASE_OFFSET + srcBuf.arrayOffset() + srcPosition; - } - if (trgBuf.isDirect()) - { - trgOffset = trgPosition + theUnsafe.getLong(trgBuf, DIRECT_BUFFER_ADDRESS_OFFSET); - trg = null; + srcOffset = BYTE_ARRAY_BASE_OFFSET + srcBuf.arrayOffset(); } else { - trg = trgBuf.array(); - trgOffset = BYTE_ARRAY_BASE_OFFSET + trgBuf.arrayOffset() + trgPosition; + src = null; + srcOffset = theUnsafe.getLong(srcBuf, DIRECT_BUFFER_ADDRESS_OFFSET); + } + copy(src, srcOffset + srcPosition, trgBuf, trgPosition, length); + } + + @Inline + public static void copy(Object src, long srcOffset, ByteBuffer trgBuf, int trgPosition, int length) + { + if (trgBuf.hasArray()) + copy(src, srcOffset, trgBuf.array(), trgBuf.arrayOffset() + trgPosition, length); + else + copy(src, srcOffset, null, trgPosition + theUnsafe.getLong(trgBuf, DIRECT_BUFFER_ADDRESS_OFFSET), length); + } + + @Inline + public static void copy(Object src, long srcOffset, byte[] trg, int trgPosition, int length) + { + if (length <= MIN_COPY_THRESHOLD) + { + for (int i = 0 ; i < length ; i++) + trg[trgPosition + i] = theUnsafe.getByte(src, srcOffset + i); + } + else + { + copy(src, srcOffset, trg, BYTE_ARRAY_BASE_OFFSET + trgPosition, length); } - copy(src, srcOffset, trg, trgOffset, length); } // 1M, copied from java.nio.Bits (unfortunately a package-private class) private static final long UNSAFE_COPY_THRESHOLD = 1 << 20; + private static final long MIN_COPY_THRESHOLD = 6; - static void copy(Object src, long srcOffset, Object dst, long dstOffset, long length) + @Inline + public static void copy(Object src, long srcOffset, Object dst, long dstOffset, long length) { while (length > 0) { long size = (length > UNSAFE_COPY_THRESHOLD) ? UNSAFE_COPY_THRESHOLD : length; @@ -307,6 +286,50 @@ class FastByteOperations } } + @Inline + public static int compareTo(ByteBuffer buffer1, ByteBuffer buffer2) + { + Object obj1; + long offset1; + int length1; + if (buffer1.hasArray()) + { + obj1 = buffer1.array(); + offset1 = BYTE_ARRAY_BASE_OFFSET + buffer1.arrayOffset(); + } + else + { + obj1 = null; + offset1 = theUnsafe.getLong(buffer1, DIRECT_BUFFER_ADDRESS_OFFSET); + } + offset1 += buffer1.position(); + length1 = buffer1.remaining(); + return compareTo(obj1, offset1, length1, buffer2); + } + + @Inline + public static int compareTo(Object buffer1, long offset1, int length1, ByteBuffer buffer) + { + Object obj2; + long offset2; + + int position = buffer.position(); + int limit = buffer.limit(); + if (buffer.hasArray()) + { + obj2 = buffer.array(); + offset2 = BYTE_ARRAY_BASE_OFFSET + buffer.arrayOffset(); + } + else + { + obj2 = null; + offset2 = theUnsafe.getLong(buffer, DIRECT_BUFFER_ADDRESS_OFFSET); + } + int length2 = limit - position; + offset2 += position; + + return compareTo(buffer1, offset1, length1, obj2, offset2, length2); + } /** * Lexicographically compare two arrays. @@ -319,65 +342,40 @@ class FastByteOperations * @param length2 How much to compare from the right buffer * @return 0 if equal, < 0 if left is less than right, etc. */ - public int compareTo(Object buffer1, long memoryOffset1, int length1, + @Inline + public static int compareTo(Object buffer1, long memoryOffset1, int length1, Object buffer2, long memoryOffset2, int length2) { - // Short circuit equal case - if (buffer1 == buffer2 && memoryOffset1 == memoryOffset2 && length1 == length2) - return 0; - int minLength = Math.min(length1, length2); - int minWords = minLength / Longs.BYTES; /* * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a * time is no slower than comparing 4 bytes at a time even on 32-bit. * On the other hand, it is substantially faster on 64-bit. */ - - int wordComparisons = minWords * Longs.BYTES; + int wordComparisons = minLength & ~7; for (int i = 0; i < wordComparisons ; i += Longs.BYTES) { long lw = theUnsafe.getLong(buffer1, memoryOffset1 + (long) i); long rw = theUnsafe.getLong(buffer2, memoryOffset2 + (long) i); - long diff = lw ^ rw; - if (diff != 0) + if (lw != rw) { - if (!littleEndian) - return lessThanUnsigned(lw, rw) ? -1 : 1; + if (BIG_ENDIAN) + return UnsignedLongs.compare(lw, rw); - // Use binary search - int n = 0; - int y; - int x = (int) diff; - if (x == 0) - { - x = (int) (diff >>> 32); - n = 32; - } - - y = x << 16; - if (y == 0) - n += 16; - else - x = y; - - y = x << 8; - if (y == 0) - n += 8; - return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL)); + return UnsignedLongs.compare(Long.reverseBytes(lw), Long.reverseBytes(rw)); } } - // The epilogue to cover the last (minLength % 8) elements. - for (int i = minWords * Longs.BYTES; i < minLength; i++) + for (int i = wordComparisons ; i < minLength ; i++) { - int result = UnsignedBytes.compare(theUnsafe.getByte(buffer1, memoryOffset1 + i), - theUnsafe.getByte(buffer2, memoryOffset2 + i)); - if (result != 0) - return result; + int b1 = theUnsafe.getByte(buffer1, memoryOffset1 + i) & 0xFF; + int b2 = theUnsafe.getByte(buffer2, memoryOffset2 + i) & 0xFF; + if (b1 != b2) + return b1 - b2; } + return length1 - length2; } diff --git a/src/java/org/apache/cassandra/utils/MurmurHash.java b/src/java/org/apache/cassandra/utils/MurmurHash.java index 9dcde6df39..bf83681858 100644 --- a/src/java/org/apache/cassandra/utils/MurmurHash.java +++ b/src/java/org/apache/cassandra/utils/MurmurHash.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.utils; +import net.nicoulaj.compilecommand.annotations.Inline; + import java.nio.ByteBuffer; /** @@ -170,6 +172,7 @@ public class MurmurHash return k; } + @Inline public static void hash3_x64_128(ByteBuffer key, int offset, int length, long seed, long[] result) { final int nblocks = length >> 4; // Process as 128-bit blocks. diff --git a/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java b/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java index ba6cf2178c..9d57182586 100644 --- a/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java +++ b/src/java/org/apache/cassandra/utils/btree/NodeBuilder.java @@ -25,7 +25,6 @@ import java.util.Comparator; import static org.apache.cassandra.utils.btree.BTree.EMPTY_BRANCH; import static org.apache.cassandra.utils.btree.BTree.FAN_FACTOR; -import static org.apache.cassandra.utils.btree.BTree.POSITIVE_INFINITY; import static org.apache.cassandra.utils.btree.BTree.compare; import static org.apache.cassandra.utils.btree.BTree.find; import static org.apache.cassandra.utils.btree.BTree.getKeyEnd; @@ -145,7 +144,7 @@ final class NodeBuilder // and simply avoids performing a binary search until we've checked the proceeding key; // possibly we should disable this check if we determine that it fails more than a handful of times // during any given builder use to get the best of both worlds - int c = comparator.compare(copyFrom[i], key); + int c = -comparator.compare(key, copyFrom[i]); if (c >= 0) { found = c == 0; @@ -168,7 +167,7 @@ final class NodeBuilder return null; key = next; } - else if (i == copyFromKeyEnd && compare(comparator, upperBound, key) <= 0) + else if (i == copyFromKeyEnd && compare(comparator, key, upperBound) >= 0) owns = false; if (isLeaf(copyFrom)) diff --git a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java index 532d071438..129a60bbb1 100644 --- a/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java +++ b/src/java/org/apache/cassandra/utils/memory/MemoryUtil.java @@ -22,6 +22,7 @@ import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import com.google.common.primitives.*; import sun.misc.Unsafe; public abstract class MemoryUtil diff --git a/test/unit/org/apache/cassandra/db/NativeCellTest.java b/test/unit/org/apache/cassandra/db/NativeCellTest.java index 0d03f55f90..6a2bf7358b 100644 --- a/test/unit/org/apache/cassandra/db/NativeCellTest.java +++ b/test/unit/org/apache/cassandra/db/NativeCellTest.java @@ -133,6 +133,86 @@ public class NativeCellTest } } + + @Test + public void testComparator() + { + + Random rand = ThreadLocalRandom.current(); + for (Name test : TESTS) + { + byte[] bytes = new byte[7]; + byte[] bytes2 = new byte[7]; + rand.nextBytes(bytes); + rand.nextBytes(bytes2); + + // test regular Cell + Cell buf, nat, buf2, nat2; + buf = new BufferCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong()); + nat = buf.localCopy(metadata, nativeAllocator, group); + + buf2 = new BufferCell(test.name, ByteBuffer.wrap(bytes2), rand.nextLong()); + nat2 = buf2.localCopy(metadata, nativeAllocator, group); + + assert test.type.compare(buf.name(), nat.name()) == 0; + assert test.type.compare(buf2.name(), nat2.name()) == 0; + + int val = test.type.compare(buf.name(), buf2.name()); + assert test.type.compare(nat.name(), nat2.name()) == val; + assert test.type.compare(nat.name(), buf2.name()) == val; + assert test.type.compare(buf.name(), nat2.name()) == val; + + + // test DeletedCell + buf = new BufferDeletedCell(test.name, rand.nextInt(100000), rand.nextLong()); + nat = buf.localCopy(metadata, nativeAllocator, group); + buf2 = new BufferDeletedCell(test.name, rand.nextInt(100000), rand.nextLong()); + nat2 = buf2.localCopy(metadata, nativeAllocator, group); + + assert test.type.compare(buf.name(), nat.name()) == 0; + assert test.type.compare(buf2.name(), nat2.name()) == 0; + + val = test.type.compare(buf.name(), buf2.name()); + assert test.type.compare(nat.name(), nat2.name()) == val; + assert test.type.compare(nat.name(), buf2.name()) == val; + assert test.type.compare(buf.name(), nat2.name()) == val; + + + + // test ExpiringCell + buf = new BufferExpiringCell(test.name, ByteBuffer.wrap(bytes), rand.nextLong(), rand.nextInt(100000)); + nat = buf.localCopy(metadata, nativeAllocator, group); + + buf2 = new BufferExpiringCell(test.name, ByteBuffer.wrap(bytes2), rand.nextLong(), rand.nextInt(100000)); + nat2 = buf2.localCopy(metadata, nativeAllocator, group); + + assert test.type.compare(buf.name(), nat.name()) == 0; + assert test.type.compare(buf2.name(), nat2.name()) == 0; + + val = test.type.compare(buf.name(), buf2.name()); + assert test.type.compare(nat.name(), nat2.name()) == val; + assert test.type.compare(nat.name(), buf2.name()) == val; + assert test.type.compare(buf.name(), nat2.name()) == val; + + + // test CounterCell + buf = new BufferCounterCell(test.name, CounterContext.instance().createLocal(rand.nextLong()), rand.nextLong(), rand.nextInt(100000)); + nat = buf.localCopy(metadata, nativeAllocator, group); + + buf2 = new BufferCounterCell(test.name, CounterContext.instance().createLocal(rand.nextLong()), rand.nextLong(), rand.nextInt(100000)); + nat2 = buf2.localCopy(metadata, nativeAllocator, group); + + assert test.type.compare(buf.name(), nat.name()) == 0; + assert test.type.compare(buf2.name(), nat2.name()) == 0; + + val = test.type.compare(buf.name(), buf2.name()); + assert test.type.compare(nat.name(), nat2.name()) == val; + assert test.type.compare(nat.name(), buf2.name()) == val; + assert test.type.compare(buf.name(), nat2.name()) == val; + + } + } + static void test(Name test, Cell buf, Cell nat) throws IOException { Assert.assertTrue(buf.equals(nat)); diff --git a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java index 4bd755be89..496a2dc4f7 100644 --- a/test/unit/org/apache/cassandra/db/composites/CTypeTest.java +++ b/test/unit/org/apache/cassandra/db/composites/CTypeTest.java @@ -91,15 +91,11 @@ public class CTypeTest { CellName a = stype2.makeCellName(UUIDType.instance.fromString("00000000-0000-0000-0000-000000000000")); CellName z = stype2.makeCellName(UUIDType.instance.fromString("ffffffff-ffff-ffff-ffff-ffffffffffff")); - Composite empty = Composites.EMPTY; assert stype2.compare(a,z) < 0; assert stype2.compare(z,a) > 0; assert stype2.compare(a,a) == 0; assert stype2.compare(z,z) == 0; - assert stype2.compare(empty, a) < 0; - assert stype2.compare(a,empty) > 0; - assert stype2.compare(empty, empty) == 0; } @@ -108,16 +104,11 @@ public class CTypeTest { CellName a = stype1.makeCellName(ByteBufferUtil.bytes("a")); CellName z = stype1.makeCellName(ByteBufferUtil.bytes("z")); - Composite empty = Composites.EMPTY; assert stype1.compare(a,z) < 0; assert stype1.compare(z,a) > 0; assert stype1.compare(a,a) == 0; assert stype1.compare(z,z) == 0; - assert stype1.compare(empty, a) < 0; - assert stype1.compare(a,empty) > 0; - assert stype1.compare(empty, empty) == 0; - } } diff --git a/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java b/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java index 2dc37441a4..15f1797e36 100644 --- a/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java +++ b/test/unit/org/apache/cassandra/db/filter/ColumnSliceTest.java @@ -382,12 +382,12 @@ public class ColumnSliceTest private static void assertSlicesValid(ColumnSlice[] slices) { - assertTrue("Slices " + toString(slices) + " should be valid", ColumnSlice.validateSlices(slices, simpleIntType)); + assertTrue("Slices " + toString(slices) + " should be valid", ColumnSlice.validateSlices(slices, simpleIntType, false)); } private static void assertSlicesInvalid(ColumnSlice[] slices) { - assertFalse("Slices " + toString(slices) + " shouldn't be valid", ColumnSlice.validateSlices(slices, simpleIntType)); + assertFalse("Slices " + toString(slices) + " shouldn't be valid", ColumnSlice.validateSlices(slices, simpleIntType, false)); } private static void assertSlicesEquals(ColumnSlice[] expected, ColumnSlice[] actual)