natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add LittleEndianMemoryUtil and NativeEndianMemoryUtil, switch memtable-related off-heap objects and Memory to use them and have Little Endian now. 

Add BE offsets detection on Summary loading.
Add test SSTables in an old format with BE offsets in Summary component to LegacySSTableTest.

Patch by Dmitry Konstantinov; reviewed by Branimir Lambov, Michael Semb Wever for CASSANDRA-20190
This commit is contained in:
Dmitry Konstantinov 2025-03-20 16:28:50 +00:00
parent 07831c9cc7
commit ae82efc013
85 changed files with 852 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0.5
* Switch memtable-related off-heap objects to Native Endian and Memory to Little Endian (CASSANDRA-20190)
* SAI marks an index as non-empty when a partial partition/row modifications is flushed due to repair (CASSANDRA-20567)
* SAI fails queries when multiple columns exist and a non-indexed column is a composite with a map (CASSANDRA-19891)
* Grant permission on keyspaces system_views and system_virtual_schema not possible (CASSANDRA-20171)

25
src/java/org/apache/cassandra/db/NativeClustering.java
View File

@ -28,6 +28,7 @@ import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.HeapCloner;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
public class NativeClustering implements Clustering<ByteBuffer>
@ -50,30 +51,30 @@ public class NativeClustering implements Clustering<ByteBuffer>
peer = allocator.allocate(metadataSize + dataSize + bitmapSize, writeOp);
long bitmapStart = peer + metadataSize;
MemoryUtil.setShort(peer, (short) count);
MemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets
NativeEndianMemoryUtil.setShort(peer, (short) count);
NativeEndianMemoryUtil.setShort(peer + (metadataSize - 2), (short) dataSize); // goes at the end of the other offsets
MemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
NativeEndianMemoryUtil.setByte(bitmapStart, bitmapSize, (byte) 0);
long dataStart = peer + metadataSize + bitmapSize;
int dataOffset = 0;
for (int i = 0 ; i < count ; i++)
{
MemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);
NativeEndianMemoryUtil.setShort(peer + 2 + i * 2, (short) dataOffset);
ByteBuffer value = clustering.bufferAt(i);
if (value == null)
{
long boffset = bitmapStart + (i >>> 3);
int b = MemoryUtil.getByte(boffset);
int b = NativeEndianMemoryUtil.getByte(boffset);
b |= 1 << (i & 7);
MemoryUtil.setByte(boffset, (byte) b);
NativeEndianMemoryUtil.setByte(boffset, (byte) b);
continue;
}
assert value.order() == ByteOrder.BIG_ENDIAN;
int size = value.remaining();
MemoryUtil.setBytes(dataStart + dataOffset, value);
NativeEndianMemoryUtil.setBytes(dataStart + dataOffset, value);
dataOffset += size;
}
}
@ -90,13 +91,13 @@ public class NativeClustering implements Clustering<ByteBuffer>
public int size()
{
return MemoryUtil.getShort(peer);
return NativeEndianMemoryUtil.getUnsignedShort(peer);
}
public int dataSize()
{
int dataSizeOffset = (size() * 2) + 2; // metadataSize - 2
return MemoryUtil.getShort(peer + dataSizeOffset);
return NativeEndianMemoryUtil.getUnsignedShort(peer + dataSizeOffset);
}
public ByteBuffer get(int i)
@ -109,12 +110,12 @@ public class NativeClustering implements Clustering<ByteBuffer>
int metadataSize = (size * 2) + 4;
int bitmapSize = ((size + 7) >>> 3);
long bitmapStart = peer + metadataSize;
int b = MemoryUtil.getByte(bitmapStart + (i >>> 3));
int b = NativeEndianMemoryUtil.getByte(bitmapStart + (i >>> 3));
if ((b & (1 << (i & 7))) != 0)
return null;
int startOffset = MemoryUtil.getShort(peer + 2 + i * 2);
int endOffset = MemoryUtil.getShort(peer + 4 + i * 2);
int startOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 2 + i * 2);
int endOffset = NativeEndianMemoryUtil.getUnsignedShort(peer + 4 + i * 2);
return MemoryUtil.getByteBuffer(bitmapStart + bitmapSize + startOffset,
endOffset - startOffset,
ByteOrder.BIG_ENDIAN);

9
src/java/org/apache/cassandra/db/NativeDecoratedKey.java
View File

@ -26,6 +26,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;
public class NativeDecoratedKey extends DecoratedKey
{
@ -39,7 +40,7 @@ public class NativeDecoratedKey extends DecoratedKey
int size = key.remaining();
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, key);
}
@ -50,14 +51,14 @@ public class NativeDecoratedKey extends DecoratedKey
int size = keyBytes.length;
this.peer = allocator.allocate(4 + size, writeOp);
MemoryUtil.setInt(peer, size);
NativeEndianMemoryUtil.setInt(peer, size);
MemoryUtil.setBytes(peer + 4, keyBytes, 0, size);
}
@Inline
int length()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}
@Inline
@ -75,7 +76,7 @@ public class NativeDecoratedKey extends DecoratedKey
@Override
public int getKeyLength()
{
return MemoryUtil.getInt(peer);
return NativeEndianMemoryUtil.getInt(peer);
}
@Override

33
src/java/org/apache/cassandra/db/rows/NativeCell.java
View File

@ -28,6 +28,7 @@ import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.concurrent.OpOrder;
import org.apache.cassandra.utils.memory.MemoryUtil;
import org.apache.cassandra.utils.memory.NativeAllocator;
import org.apache.cassandra.utils.memory.NativeEndianMemoryUtil;
public class NativeCell extends AbstractCell<ByteBuffer>
{
@ -101,11 +102,11 @@ public class NativeCell extends AbstractCell<ByteBuffer>
// cellpath? : timestamp : ttl : localDeletionTime : length : <data> : [cell path length] : [<cell path data>]
peer = allocator.allocate((int) size, writeOp);
MemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
MemoryUtil.setLong(peer + TIMESTAMP, timestamp);
MemoryUtil.setInt(peer + TTL, ttl);
MemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
MemoryUtil.setInt(peer + LENGTH, value.remaining());
NativeEndianMemoryUtil.setByte(peer + HAS_CELLPATH, (byte)(path == null ? 0 : 1));
NativeEndianMemoryUtil.setLong(peer + TIMESTAMP, timestamp);
NativeEndianMemoryUtil.setInt(peer + TTL, ttl);
NativeEndianMemoryUtil.setInt(peer + DELETION, localDeletionTimeUnsignedInteger);
NativeEndianMemoryUtil.setInt(peer + LENGTH, value.remaining());
MemoryUtil.setBytes(peer + VALUE, value);
if (path != null)
@ -114,7 +115,7 @@ public class NativeCell extends AbstractCell<ByteBuffer>
assert pathbuffer.order() == ByteOrder.BIG_ENDIAN;
long offset = peer + VALUE + value.remaining();
MemoryUtil.setInt(offset, pathbuffer.remaining());
NativeEndianMemoryUtil.setInt(offset, pathbuffer.remaining());
MemoryUtil.setBytes(offset + 4, pathbuffer);
}
}
@ -126,17 +127,17 @@ public class NativeCell extends AbstractCell<ByteBuffer>
public long timestamp()
{
return MemoryUtil.getLong(peer + TIMESTAMP);
return NativeEndianMemoryUtil.getLong(peer + TIMESTAMP);
}
public int ttl()
{
return MemoryUtil.getInt(peer + TTL);
return NativeEndianMemoryUtil.getInt(peer + TTL);
}
public ByteBuffer value()// FIXME: add native accessor
{
int length = MemoryUtil.getInt(peer + LENGTH);
int length = NativeEndianMemoryUtil.getInt(peer + LENGTH);
return MemoryUtil.getByteBuffer(peer + VALUE, length, ByteOrder.BIG_ENDIAN);
}
@ -147,7 +148,7 @@ public class NativeCell extends AbstractCell<ByteBuffer>
public int valueSize()
{
return MemoryUtil.getInt(peer + LENGTH);
return NativeEndianMemoryUtil.getInt(peer + LENGTH);
}
public CellPath path()
@ -155,8 +156,8 @@ public class NativeCell extends AbstractCell<ByteBuffer>
if (!hasPath())
return null;
long offset = peer + VALUE + MemoryUtil.getInt(peer + LENGTH);
int size = MemoryUtil.getInt(offset);
long offset = peer + VALUE + NativeEndianMemoryUtil.getInt(peer + LENGTH);
int size = NativeEndianMemoryUtil.getInt(offset);
return CellPath.create(MemoryUtil.getByteBuffer(offset + 4, size, ByteOrder.BIG_ENDIAN));
}
@ -194,20 +195,20 @@ public class NativeCell extends AbstractCell<ByteBuffer>
public long offHeapSize()
{
long size = offHeapSizeWithoutPath(MemoryUtil.getInt(peer + LENGTH));
long size = offHeapSizeWithoutPath(NativeEndianMemoryUtil.getInt(peer + LENGTH));
if (hasPath())
size += 4 + MemoryUtil.getInt(peer + size);
size += 4 + NativeEndianMemoryUtil.getInt(peer + size);
return size;
}
private boolean hasPath()
{
return MemoryUtil.getByte(peer+ HAS_CELLPATH) != 0;
return NativeEndianMemoryUtil.getByte(peer + HAS_CELLPATH) != 0;
}
@Override
protected int localDeletionTimeAsUnsignedInt()
{
return MemoryUtil.getInt(peer + DELETION);
return NativeEndianMemoryUtil.getInt(peer + DELETION);
}
}

12
src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummary.java
View File

@ -454,6 +454,18 @@ public class IndexSummary extends WrappedSharedCloseable
entries.free();
throw ioe;
}
// Before 5.0 offsets were written using Native Endian, now they are stored as Little Endian,
// so we apply a heuristic here to detect
// if the loading index summary was created on a Big Endian machine using Native Endian format
if (offsets.size() > 0)
{
int offset = offsets.getInt(0);
int offsetReversed = Integer.reverseBytes(offset);
if (offsetReversed > 0 && offset > offsetReversed || offset - offsets.size() < 0)
throw new IOException(String.format("Rebuilding index summary because offset value (%d) at position: %d " +
"is Big Endian while Little Endian is expected", offset, 0));
}
// our on-disk representation treats the offsets and the summary data as one contiguous structure,
// in which the offsets are based from the start of the structure. i.e., if the offsets occupy
// X bytes, the value of the first offset will be X. In memory we split the two regions up, so that

144
src/java/org/apache/cassandra/io/util/Memory.java
View File

@ -22,14 +22,15 @@ import java.nio.ByteBuffer;
import net.nicoulaj.compilecommand.annotations.Inline;
import org.apache.cassandra.utils.Architecture;
import org.apache.cassandra.utils.FastByteOperations;
import org.apache.cassandra.utils.concurrent.Ref;
import org.apache.cassandra.utils.memory.LittleEndianMemoryUtil;
import org.apache.cassandra.utils.memory.MemoryUtil;
import sun.misc.Unsafe;
/**
* An off-heap region of memory that must be manually free'd when no longer needed.
* It uses Little Endian (LE).
*/
public class Memory implements AutoCloseable, ReadableMemory
{
@ -90,7 +91,7 @@ public class Memory implements AutoCloseable, ReadableMemory
public void setByte(long offset, byte b)
{
checkBounds(offset, offset + 1);
unsafe.putByte(peer + offset, b);
LittleEndianMemoryUtil.setByte(peer + offset, b);
}
public void setMemory(long offset, long bytes, byte b)
@ -103,86 +104,13 @@ public class Memory implements AutoCloseable, ReadableMemory
public void setLong(long offset, long l)
{
checkBounds(offset, offset + 8);
if (Architecture.IS_UNALIGNED)
unsafe.putLong(peer + offset, Architecture.BIG_ENDIAN ? Long.reverseBytes(l) : l);
else
putLongByByte(peer + offset, l);
}
private void putLongByByte(long address, long value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address, (byte) (value >> 56));
unsafe.putByte(address + 1, (byte) (value >> 48));
unsafe.putByte(address + 2, (byte) (value >> 40));
unsafe.putByte(address + 3, (byte) (value >> 32));
unsafe.putByte(address + 4, (byte) (value >> 24));
unsafe.putByte(address + 5, (byte) (value >> 16));
unsafe.putByte(address + 6, (byte) (value >> 8));
unsafe.putByte(address + 7, (byte) (value));
}
else
{
unsafe.putByte(address + 7, (byte) (value >> 56));
unsafe.putByte(address + 6, (byte) (value >> 48));
unsafe.putByte(address + 5, (byte) (value >> 40));
unsafe.putByte(address + 4, (byte) (value >> 32));
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address, (byte) (value));
}
LittleEndianMemoryUtil.setLong(peer + offset, l);
}
public void setInt(long offset, int l)
{
checkBounds(offset, offset + 4);
if (Architecture.IS_UNALIGNED)
unsafe.putInt(peer + offset, Architecture.BIG_ENDIAN ? Integer.reverseBytes(l) : l);
else
putIntByByte(peer + offset, l);
}
private void putIntByByte(long address, int value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address, (byte) (value >> 24));
unsafe.putByte(address + 1, (byte) (value >> 16));
unsafe.putByte(address + 2, (byte) (value >> 8));
unsafe.putByte(address + 3, (byte) (value));
}
else
{
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address, (byte) (value));
}
}
public void setShort(long offset, short l)
{
checkBounds(offset, offset + 2);
if (Architecture.IS_UNALIGNED)
unsafe.putShort(peer + offset, Architecture.BIG_ENDIAN ? Short.reverseBytes(l) : l);
else
putShortByByte(peer + offset, l);
}
private void putShortByByte(long address, short value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address, (byte) (value >> 8));
unsafe.putByte(address + 1, (byte) (value));
}
else
{
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address, (byte) (value));
}
LittleEndianMemoryUtil.setInt(peer + offset, l);
}
public void setBytes(long memoryOffset, ByteBuffer buffer)
@ -230,69 +158,19 @@ public class Memory implements AutoCloseable, ReadableMemory
public byte getByte(long offset)
{
checkBounds(offset, offset + 1);
return unsafe.getByte(peer + offset);
return LittleEndianMemoryUtil.getByte(peer + offset);
}
public long getLong(long offset)
{
checkBounds(offset, offset + 8);
if (Architecture.IS_UNALIGNED)
return Architecture.BIG_ENDIAN ? Long.reverseBytes(unsafe.getLong(peer+offset)) : unsafe.getLong(peer+offset);
else
return getLongByByte(peer + offset);
}
private long getLongByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((long) unsafe.getByte(address ) ) << 56) |
(((long) unsafe.getByte(address + 1) & 0xff) << 48) |
(((long) unsafe.getByte(address + 2) & 0xff) << 40) |
(((long) unsafe.getByte(address + 3) & 0xff) << 32) |
(((long) unsafe.getByte(address + 4) & 0xff) << 24) |
(((long) unsafe.getByte(address + 5) & 0xff) << 16) |
(((long) unsafe.getByte(address + 6) & 0xff) << 8) |
(((long) unsafe.getByte(address + 7) & 0xff) );
}
else
{
return (((long) unsafe.getByte(address + 7) ) << 56) |
(((long) unsafe.getByte(address + 6) & 0xff) << 48) |
(((long) unsafe.getByte(address + 5) & 0xff) << 40) |
(((long) unsafe.getByte(address + 4) & 0xff) << 32) |
(((long) unsafe.getByte(address + 3) & 0xff) << 24) |
(((long) unsafe.getByte(address + 2) & 0xff) << 16) |
(((long) unsafe.getByte(address + 1) & 0xff) << 8) |
(((long) unsafe.getByte(address ) & 0xff) );
}
return LittleEndianMemoryUtil.getLong(peer + offset);
}
public int getInt(long offset)
{
checkBounds(offset, offset + 4);
if (Architecture.IS_UNALIGNED)
return Architecture.BIG_ENDIAN ? Integer.reverseBytes(unsafe.getInt(peer+offset)) : unsafe.getInt(peer+offset);
else
return getIntByByte(peer + offset);
}
private int getIntByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return ((unsafe.getByte(address ) ) << 24) |
((unsafe.getByte(address + 1) & 0xff) << 16) |
((unsafe.getByte(address + 2) & 0xff) << 8 ) |
((unsafe.getByte(address + 3) & 0xff) );
}
else
{
return ((unsafe.getByte(address + 3) ) << 24) |
((unsafe.getByte(address + 2) & 0xff) << 16) |
((unsafe.getByte(address + 1) & 0xff) << 8) |
((unsafe.getByte(address ) & 0xff) );
}
return LittleEndianMemoryUtil.getInt(peer + offset);
}
/**
@ -378,18 +256,18 @@ public class Memory implements AutoCloseable, ReadableMemory
int size = (int) (size() / result.length);
for (int i = 0 ; i < result.length - 1 ; i++)
{
result[i] = MemoryUtil.getByteBuffer(peer + offset, size);
result[i] = LittleEndianMemoryUtil.getByteBuffer(peer + offset, size);
offset += size;
length -= size;
}
result[result.length - 1] = MemoryUtil.getByteBuffer(peer + offset, (int) length);
result[result.length - 1] = LittleEndianMemoryUtil.getByteBuffer(peer + offset, (int) length);
return result;
}
public ByteBuffer asByteBuffer(long offset, int length)
{
checkBounds(offset, offset + length);
return MemoryUtil.getByteBuffer(peer + offset, length);
return LittleEndianMemoryUtil.getByteBuffer(peer + offset, length);
}
// MUST provide a buffer created via MemoryUtil.getHollowDirectByteBuffer()

146
src/java/org/apache/cassandra/utils/memory/LittleEndianMemoryUtil.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils.memory;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.utils.Architecture;
public class LittleEndianMemoryUtil extends MemoryUtil
{
public static int getUnsignedShort(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b1) == 0L)
return (Architecture.BIG_ENDIAN ? Short.reverseBytes(unsafe.getShort(address)) : unsafe.getShort(address)) & 0xffff;
else
return getShortByByte(address) & 0xffff;
}
public static int getInt(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b11) == 0L)
return Architecture.BIG_ENDIAN ? Integer.reverseBytes(unsafe.getInt(address)) : unsafe.getInt(address);
else
return getIntByByte(address);
}
public static long getLong(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b111) == 0L)
return Architecture.BIG_ENDIAN ? Long.reverseBytes(unsafe.getLong(address)) : unsafe.getLong(address);
else
return getLongByByte(address);
}
public static void setShort(long address, short s)
{
if (Architecture.IS_UNALIGNED || (address & 0b1) == 0L)
unsafe.putShort(address, Architecture.BIG_ENDIAN ? Short.reverseBytes(s) : s);
else
putShortByByte(address, s);
}
public static void setInt(long address, int l)
{
if (Architecture.IS_UNALIGNED || (address & 0b11) == 0L)
unsafe.putInt(address, Architecture.BIG_ENDIAN ? Integer.reverseBytes(l) : l);
else
putIntByByte(address, l);
}
public static void setLong(long address, long l)
{
if (Architecture.IS_UNALIGNED || (address & 0b111) == 0L)
unsafe.putLong(address, Architecture.BIG_ENDIAN ? Long.reverseBytes(l) : l);
else
putLongByByte(address, l);
}
@VisibleForTesting
static long getLongByByte(long address)
{
return (((long) unsafe.getByte(address + 7) ) << 56) |
(((long) unsafe.getByte(address + 6) & 0xff) << 48) |
(((long) unsafe.getByte(address + 5) & 0xff) << 40) |
(((long) unsafe.getByte(address + 4) & 0xff) << 32) |
(((long) unsafe.getByte(address + 3) & 0xff) << 24) |
(((long) unsafe.getByte(address + 2) & 0xff) << 16) |
(((long) unsafe.getByte(address + 1) & 0xff) << 8) |
(((long) unsafe.getByte(address ) & 0xff) );
}
@VisibleForTesting
static int getIntByByte(long address)
{
return (((int) unsafe.getByte(address + 3) ) << 24) |
(((int) unsafe.getByte(address + 2) & 0xff) << 16) |
(((int) unsafe.getByte(address + 1) & 0xff) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
@VisibleForTesting
static int getShortByByte(long address)
{
return (((int) unsafe.getByte(address + 1) ) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
@VisibleForTesting
static void putLongByByte(long address, long value)
{
unsafe.putByte(address + 7, (byte) (value >> 56));
unsafe.putByte(address + 6, (byte) (value >> 48));
unsafe.putByte(address + 5, (byte) (value >> 40));
unsafe.putByte(address + 4, (byte) (value >> 32));
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
@VisibleForTesting
static void putIntByByte(long address, int value)
{
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
@VisibleForTesting
static void putShortByByte(long address, short value)
{
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
public static ByteBuffer getByteBuffer(long address, int length)
{
return getByteBuffer(address, length, ByteOrder.LITTLE_ENDIAN);
}
public static ByteBuffer getHollowDirectByteBuffer()
{
return getHollowDirectByteBuffer(ByteOrder.LITTLE_ENDIAN);
}
}

173
src/java/org/apache/cassandra/utils/memory/MemoryUtil.java
View File

@ -24,24 +24,20 @@ import java.nio.ByteOrder;
import com.sun.jna.Native;
import org.apache.cassandra.utils.Architecture;
import sun.misc.Unsafe;
public abstract class MemoryUtil
{
private static final long UNSAFE_COPY_THRESHOLD = 1024 * 1024L; // copied from java.nio.Bits
private static final Unsafe unsafe;
protected static final Unsafe unsafe;
private static final Class<?> DIRECT_BYTE_BUFFER_CLASS, RO_DIRECT_BYTE_BUFFER_CLASS;
private static final long DIRECT_BYTE_BUFFER_ADDRESS_OFFSET;
private static final long DIRECT_BYTE_BUFFER_CAPACITY_OFFSET;
private static final long DIRECT_BYTE_BUFFER_LIMIT_OFFSET;
private static final long DIRECT_BYTE_BUFFER_POSITION_OFFSET;
private static final long DIRECT_BYTE_BUFFER_ATTACHMENT_OFFSET;
private static final Class<?> BYTE_BUFFER_CLASS;
private static final long BYTE_BUFFER_OFFSET_OFFSET;
private static final long BYTE_BUFFER_HB_OFFSET;
protected static final Class<?> BYTE_BUFFER_CLASS;
private static final long BYTE_ARRAY_BASE_OFFSET;
static
@ -61,8 +57,6 @@ public abstract class MemoryUtil
RO_DIRECT_BYTE_BUFFER_CLASS = ByteBuffer.allocateDirect(0).asReadOnlyBuffer().getClass();
clazz = ByteBuffer.allocate(0).getClass();
BYTE_BUFFER_OFFSET_OFFSET = unsafe.objectFieldOffset(ByteBuffer.class.getDeclaredField("offset"));
BYTE_BUFFER_HB_OFFSET = unsafe.objectFieldOffset(ByteBuffer.class.getDeclaredField("hb"));
BYTE_BUFFER_CLASS = clazz;
BYTE_ARRAY_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class);
@ -104,56 +98,11 @@ public abstract class MemoryUtil
unsafe.setMemory(address, count, b);
}
public static void setShort(long address, short s)
{
unsafe.putShort(address, Architecture.BIG_ENDIAN ? Short.reverseBytes(s) : s);
}
public static void setInt(long address, int l)
{
if (Architecture.IS_UNALIGNED)
unsafe.putInt(address, Architecture.BIG_ENDIAN ? Integer.reverseBytes(l) : l);
else
putIntByByte(address, l);
}
public static void setLong(long address, long l)
{
if (Architecture.IS_UNALIGNED)
unsafe.putLong(address, Architecture.BIG_ENDIAN ? Long.reverseBytes(l) : l);
else
putLongByByte(address, l);
}
public static byte getByte(long address)
{
return unsafe.getByte(address);
}
public static int getShort(long address)
{
if (Architecture.IS_UNALIGNED)
return (Architecture.BIG_ENDIAN ? Short.reverseBytes(unsafe.getShort(address)) : unsafe.getShort(address)) & 0xffff;
else
return getShortByByte(address) & 0xffff;
}
public static int getInt(long address)
{
if (Architecture.IS_UNALIGNED)
return Architecture.BIG_ENDIAN ? Integer.reverseBytes(unsafe.getInt(address)) : unsafe.getInt(address);
else
return getIntByByte(address);
}
public static long getLong(long address)
{
if (Architecture.IS_UNALIGNED)
return Architecture.BIG_ENDIAN ? Long.reverseBytes(unsafe.getLong(address)) : unsafe.getLong(address);
else
return getLongByByte(address);
}
public static ByteBuffer getByteBuffer(long address, int length)
{
return getByteBuffer(address, length, ByteOrder.nativeOrder());
@ -186,21 +135,6 @@ public abstract class MemoryUtil
return instance;
}
public static ByteBuffer getHollowByteBuffer()
{
ByteBuffer instance;
try
{
instance = (ByteBuffer) unsafe.allocateInstance(BYTE_BUFFER_CLASS);
}
catch (InstantiationException e)
{
throw new AssertionError(e);
}
instance.order(ByteOrder.nativeOrder());
return instance;
}
public static boolean isExactlyDirect(ByteBuffer buffer)
{
return buffer.getClass() == DIRECT_BYTE_BUFFER_CLASS;
@ -250,109 +184,6 @@ public abstract class MemoryUtil
unsafe.putInt(instance, DIRECT_BYTE_BUFFER_CAPACITY_OFFSET, capacity);
}
public static long getLongByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((long) unsafe.getByte(address ) ) << 56) |
(((long) unsafe.getByte(address + 1) & 0xff) << 48) |
(((long) unsafe.getByte(address + 2) & 0xff) << 40) |
(((long) unsafe.getByte(address + 3) & 0xff) << 32) |
(((long) unsafe.getByte(address + 4) & 0xff) << 24) |
(((long) unsafe.getByte(address + 5) & 0xff) << 16) |
(((long) unsafe.getByte(address + 6) & 0xff) << 8) |
(((long) unsafe.getByte(address + 7) & 0xff) );
}
else
{
return (((long) unsafe.getByte(address + 7) ) << 56) |
(((long) unsafe.getByte(address + 6) & 0xff) << 48) |
(((long) unsafe.getByte(address + 5) & 0xff) << 40) |
(((long) unsafe.getByte(address + 4) & 0xff) << 32) |
(((long) unsafe.getByte(address + 3) & 0xff) << 24) |
(((long) unsafe.getByte(address + 2) & 0xff) << 16) |
(((long) unsafe.getByte(address + 1) & 0xff) << 8) |
(((long) unsafe.getByte(address ) & 0xff) );
}
}
public static int getIntByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((int) unsafe.getByte(address ) ) << 24) |
(((int) unsafe.getByte(address + 1) & 0xff) << 16) |
(((int) unsafe.getByte(address + 2) & 0xff) << 8 ) |
(((int) unsafe.getByte(address + 3) & 0xff) );
}
else
{
return (((int) unsafe.getByte(address + 3) ) << 24) |
(((int) unsafe.getByte(address + 2) & 0xff) << 16) |
(((int) unsafe.getByte(address + 1) & 0xff) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
}
public static int getShortByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((int) unsafe.getByte(address ) ) << 8) |
(((int) unsafe.getByte(address + 1) & 0xff) );
}
else
{
return (((int) unsafe.getByte(address + 1) ) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
}
public static void putLongByByte(long address, long value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address, (byte) (value >> 56));
unsafe.putByte(address + 1, (byte) (value >> 48));
unsafe.putByte(address + 2, (byte) (value >> 40));
unsafe.putByte(address + 3, (byte) (value >> 32));
unsafe.putByte(address + 4, (byte) (value >> 24));
unsafe.putByte(address + 5, (byte) (value >> 16));
unsafe.putByte(address + 6, (byte) (value >> 8));
unsafe.putByte(address + 7, (byte) (value));
}
else
{
unsafe.putByte(address + 7, (byte) (value >> 56));
unsafe.putByte(address + 6, (byte) (value >> 48));
unsafe.putByte(address + 5, (byte) (value >> 40));
unsafe.putByte(address + 4, (byte) (value >> 32));
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address, (byte) (value));
}
}
public static void putIntByByte(long address, int value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address, (byte) (value >> 24));
unsafe.putByte(address + 1, (byte) (value >> 16));
unsafe.putByte(address + 2, (byte) (value >> 8));
unsafe.putByte(address + 3, (byte) (value));
}
else
{
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address, (byte) (value));
}
}
public static void setBytes(long address, ByteBuffer buffer)
{
int start = buffer.position();

214
src/java/org/apache/cassandra/utils/memory/NativeEndianMemoryUtil.java Normal file
View File

@ -0,0 +1,214 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils.memory;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.utils.Architecture;
/**
* Use this API only for data which are stored in-memory
* and not serialized directly (without converting to Java primitives) to disk and network
*/
public class NativeEndianMemoryUtil extends MemoryUtil
{
public static int getUnsignedShort(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b1) == 0L)
return unsafe.getShort(address) & 0xffff;
else
return getShortByByte(address) & 0xffff;
}
public static int getInt(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b11) == 0L)
return unsafe.getInt(address);
else
return getIntByByte(address);
}
public static long getLong(long address)
{
if (Architecture.IS_UNALIGNED || (address & 0b111) == 0L)
return unsafe.getLong(address);
else
return getLongByByte(address);
}
public static void setShort(long address, short s)
{
if (Architecture.IS_UNALIGNED || (address & 0b1) == 0L)
unsafe.putShort(address, s);
else
putShortByByte(address, s);
}
public static void setInt(long address, int l)
{
if (Architecture.IS_UNALIGNED || (address & 0b11) == 0L)
unsafe.putInt(address, l);
else
putIntByByte(address, l);
}
public static void setLong(long address, long l)
{
if (Architecture.IS_UNALIGNED || (address & 0b111) == 0L)
unsafe.putLong(address, l);
else
putLongByByte(address, l);
}
@VisibleForTesting
static long getLongByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((long) unsafe.getByte(address ) ) << 56) |
(((long) unsafe.getByte(address + 1) & 0xff) << 48) |
(((long) unsafe.getByte(address + 2) & 0xff) << 40) |
(((long) unsafe.getByte(address + 3) & 0xff) << 32) |
(((long) unsafe.getByte(address + 4) & 0xff) << 24) |
(((long) unsafe.getByte(address + 5) & 0xff) << 16) |
(((long) unsafe.getByte(address + 6) & 0xff) << 8) |
(((long) unsafe.getByte(address + 7) & 0xff) );
}
else
{
return (((long) unsafe.getByte(address + 7) ) << 56) |
(((long) unsafe.getByte(address + 6) & 0xff) << 48) |
(((long) unsafe.getByte(address + 5) & 0xff) << 40) |
(((long) unsafe.getByte(address + 4) & 0xff) << 32) |
(((long) unsafe.getByte(address + 3) & 0xff) << 24) |
(((long) unsafe.getByte(address + 2) & 0xff) << 16) |
(((long) unsafe.getByte(address + 1) & 0xff) << 8) |
(((long) unsafe.getByte(address ) & 0xff) );
}
}
@VisibleForTesting
static int getIntByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((int) unsafe.getByte(address ) ) << 24) |
(((int) unsafe.getByte(address + 1) & 0xff) << 16) |
(((int) unsafe.getByte(address + 2) & 0xff) << 8) |
(((int) unsafe.getByte(address + 3) & 0xff) );
}
else
{
return (((int) unsafe.getByte(address + 3) ) << 24) |
(((int) unsafe.getByte(address + 2) & 0xff) << 16) |
(((int) unsafe.getByte(address + 1) & 0xff) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
}
@VisibleForTesting
static int getShortByByte(long address)
{
if (Architecture.BIG_ENDIAN)
{
return (((int) unsafe.getByte(address ) ) << 8) |
(((int) unsafe.getByte(address + 1) & 0xff) );
}
else
{
return (((int) unsafe.getByte(address + 1) ) << 8) |
(((int) unsafe.getByte(address ) & 0xff) );
}
}
@VisibleForTesting
static void putLongByByte(long address, long value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address , (byte) (value >> 56));
unsafe.putByte(address + 1, (byte) (value >> 48));
unsafe.putByte(address + 2, (byte) (value >> 40));
unsafe.putByte(address + 3, (byte) (value >> 32));
unsafe.putByte(address + 4, (byte) (value >> 24));
unsafe.putByte(address + 5, (byte) (value >> 16));
unsafe.putByte(address + 6, (byte) (value >> 8));
unsafe.putByte(address + 7, (byte) (value ));
}
else
{
unsafe.putByte(address + 7, (byte) (value >> 56));
unsafe.putByte(address + 6, (byte) (value >> 48));
unsafe.putByte(address + 5, (byte) (value >> 40));
unsafe.putByte(address + 4, (byte) (value >> 32));
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
}
@VisibleForTesting
static void putIntByByte(long address, int value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address , (byte) (value >> 24));
unsafe.putByte(address + 1, (byte) (value >> 16));
unsafe.putByte(address + 2, (byte) (value >> 8));
unsafe.putByte(address + 3, (byte) (value ));
}
else
{
unsafe.putByte(address + 3, (byte) (value >> 24));
unsafe.putByte(address + 2, (byte) (value >> 16));
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
}
@VisibleForTesting
static void putShortByByte(long address, short value)
{
if (Architecture.BIG_ENDIAN)
{
unsafe.putByte(address , (byte) (value >> 8));
unsafe.putByte(address + 1, (byte) (value ));
}
else
{
unsafe.putByte(address + 1, (byte) (value >> 8));
unsafe.putByte(address , (byte) (value ));
}
}
public static ByteBuffer getByteBuffer(long address, int length)
{
return getByteBuffer(address, length, ByteOrder.nativeOrder());
}
public static ByteBuffer getHollowDirectByteBuffer()
{
return getHollowDirectByteBuffer(ByteOrder.nativeOrder());
}
}

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
1026070592

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Partitions.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Rows.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-Statistics.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_be_index_summary/da-500-bti-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
Digest.crc32
TOC.txt
CompressionInfo.db
Filter.db
Partitions.db
Rows.db

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Data.db Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

1
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
1371588035

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/ma/legacy_tables/legacy_ma_clust_be_index_summary/ma-306-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
CompressionInfo.db
Data.db
Digest.crc32
Summary.db
Index.db
Statistics.db
TOC.txt
Filter.db

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Data.db Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

1
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
849406636

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/mb/legacy_tables/legacy_mb_clust_be_index_summary/mb-307-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Index.db
Filter.db
TOC.txt
Digest.crc32
Summary.db
Data.db
Statistics.db
CompressionInfo.db

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Data.db Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

1
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
1373250029

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_be_index_summary/mc-3113-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Summary.db
CompressionInfo.db
Digest.crc32
Statistics.db
Index.db
Data.db
Filter.db
TOC.txt

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Data.db Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

1
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
2461228597

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/md/legacy_tables/legacy_md_clust_be_index_summary/md-31110-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
Filter.db
Summary.db
CompressionInfo.db
Index.db
TOC.txt
Digest.crc32

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Data.db Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

1
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
3525076442

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/me/legacy_tables/legacy_me_clust_be_index_summary/me-31111-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Index.db
CompressionInfo.db
Statistics.db
TOC.txt
Summary.db
Digest.crc32
Data.db
Filter.db

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
309317098

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_be_index_summary/na-40-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
TOC.txt
Data.db
Index.db
Statistics.db
Digest.crc32
CompressionInfo.db
Filter.db
Summary.db

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
3874015080

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/nb/legacy_tables/legacy_nb_clust_be_index_summary/nb-400-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Data.db
TOC.txt
Digest.crc32
Summary.db
Index.db
Filter.db
CompressionInfo.db
Statistics.db

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-CompressionInfo.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Data.db Normal file
View File

Binary file not shown.

1
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Digest.crc32 Normal file
View File

@ -0,0 +1 @@
1158768921

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Filter.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Index.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Statistics.db Normal file
View File

Binary file not shown.

BIN
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-Summary.db Normal file
View File

Binary file not shown.

8
test/data/legacy-sstables/oa/legacy_tables/legacy_oa_clust_be_index_summary/oa-500-big-TOC.txt Normal file
View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
Digest.crc32
TOC.txt
CompressionInfo.db
Filter.db
Index.db
Summary.db

2
test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
View File

@ -934,7 +934,7 @@ public class CompactionsCQLTest extends CQLTester
File tableDir = new File(ksDir, cfs.name);
Assert.assertTrue("The table directory " + tableDir + " was not found", tableDir.isDirectory());
for (File file : tableDir.tryList())
LegacySSTableTest.copyFile(cfDir, file);
LegacySSTableTest.copyFileToDir(file, cfDir);
}
cfs.loadNewSSTables();
}

79
test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
View File

@ -32,8 +32,10 @@ import com.google.common.collect.Lists;
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -89,6 +91,9 @@ public class LegacySSTableTest
{
private static final Logger logger = LoggerFactory.getLogger(LegacySSTableTest.class);
@ClassRule
public static TemporaryFolder tempFolder = new TemporaryFolder();
public static File LEGACY_SSTABLE_ROOT;
private static final String LEGACY_TABLES_KEYSPACE = "legacy_tables";
@ -163,11 +168,11 @@ public class LegacySSTableTest
/**
* Get a descriptor for the legacy sstable at the given version.
*/
protected Descriptor getDescriptor(String legacyVersion, String table) throws IOException
protected Descriptor getDescriptor(File dir) throws IOException
{
Path file = Files.list(getTableDir(legacyVersion, table).toPath())
Path file = Files.list(dir.toPath())
.findFirst()
.orElseThrow(() -> new RuntimeException(String.format("No files for verion=%s and table=%s", legacyVersion, table)));
.orElseThrow(() -> new RuntimeException(String.format("No files for path=%s", dir.absolutePath())));
return Descriptor.fromFile(new File(file));
}
@ -493,15 +498,19 @@ public class LegacySSTableTest
streamLegacyTable("legacy_%s_clust", legacyVersion);
streamLegacyTable("legacy_%s_clust_counter", legacyVersion);
streamLegacyTable("legacy_%s_tuple", legacyVersion);
streamLegacyTable("legacy_%s_clust_be_index_summary", legacyVersion);
}
private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception
{
String table = String.format(tablePattern, legacyVersion);
Descriptor descriptor = getDescriptor(legacyVersion, table);
// streaming can mutate test data (rewrite IndexSummary, so we have to copy them)
File testDataDir = new File(tempFolder.newFolder(LEGACY_TABLES_KEYSPACE, table));
copySstablesToTestData(legacyVersion, table, testDataDir);
Descriptor descriptor = getDescriptor(testDataDir);
if (null != descriptor)
{
SSTableReader sstable = SSTableReader.open(null, getDescriptor(legacyVersion, table));
SSTableReader sstable = SSTableReader.open(null, descriptor);
IPartitioner p = sstable.getPartitioner();
List<Range<Token>> ranges = new ArrayList<>();
ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100"))));
@ -525,6 +534,7 @@ public class LegacySSTableTest
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).truncateBlocking();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).truncateBlocking();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_tuple", legacyVersion)).truncateBlocking();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_be_index_summary", legacyVersion)).truncateBlocking();
CacheService.instance.invalidateCounterCache();
CacheService.instance.invalidateKeyCache();
}
@ -537,6 +547,7 @@ public class LegacySSTableTest
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust", legacyVersion)).forceMajorCompaction();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_counter", legacyVersion)).forceMajorCompaction();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_tuple", legacyVersion)).forceMajorCompaction();
Keyspace.open(LEGACY_TABLES_KEYSPACE).getColumnFamilyStore(String.format("legacy_%s_clust_be_index_summary", legacyVersion)).forceMajorCompaction();
}
public static void loadLegacyTables(String legacyVersion) throws Exception
@ -547,6 +558,7 @@ public class LegacySSTableTest
loadLegacyTable(legacyVersion, "clust");
loadLegacyTable(legacyVersion, "clust_counter");
loadLegacyTable(legacyVersion, "tuple");
loadLegacyTable(legacyVersion, "clust_be_index_summary");
}
private static void verifyCache(String legacyVersion, long startCount) throws InterruptedException, java.util.concurrent.ExecutionException
@ -584,7 +596,8 @@ public class LegacySSTableTest
readSimpleCounterTable(legacyVersion, pkValue);
}
readClusteringTable(legacyVersion, ck, ckValue, pkValue);
readClusteringTable("legacy_%s_clust", legacyVersion, ck, ckValue, pkValue);
readClusteringTable("legacy_%s_clust_be_index_summary", legacyVersion, ck, ckValue, pkValue);
readClusteringCounterTable(legacyVersion, ckValue, pkValue);
}
}
@ -600,16 +613,16 @@ public class LegacySSTableTest
Assert.assertEquals(1L, rs.one().getLong("val"));
}
private static void readClusteringTable(String legacyVersion, int ck, String ckValue, String pkValue)
private static void readClusteringTable(String tableName, String legacyVersion, int ck, String ckValue, String pkValue)
{
logger.debug("Read legacy_{}_clust", legacyVersion);
UntypedResultSet rs;
rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck=?", legacyVersion), pkValue, ckValue);
rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables." + tableName + " WHERE pk=? AND ck=?", legacyVersion), pkValue, ckValue);
assertLegacyClustRows(1, rs);
String ckValue2 = Integer.toString(ck < 10 ? 40 : ck - 1) + longString;
String ckValue3 = Integer.toString(ck > 39 ? 10 : ck + 1) + longString;
rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables.legacy_%s_clust WHERE pk=? AND ck IN (?, ?, ?)", legacyVersion), pkValue, ckValue, ckValue2, ckValue3);
rs = QueryProcessor.executeInternal(String.format("SELECT val FROM legacy_tables." + tableName + " WHERE pk=? AND ck IN (?, ?, ?)", legacyVersion), pkValue, ckValue, ckValue2, ckValue3);
assertLegacyClustRows(3, rs);
}
@ -644,7 +657,7 @@ public class LegacySSTableTest
QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_simple_counter (pk text PRIMARY KEY, val counter)", legacyVersion));
QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust (pk text, ck text, val text, PRIMARY KEY (pk, ck))", legacyVersion));
QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_counter (pk text, ck text, val counter, PRIMARY KEY (pk, ck))", legacyVersion));
QueryProcessor.executeInternal(String.format("CREATE TABLE legacy_tables.legacy_%s_clust_be_index_summary (pk text, ck text, val text, PRIMARY KEY (pk, ck))", legacyVersion));
QueryProcessor.executeInternal(String.format("CREATE TYPE legacy_tables.legacy_%s_tuple_udt (name tuple<text,text>)", legacyVersion));
@ -667,6 +680,7 @@ public class LegacySSTableTest
QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_simple_counter", legacyVersion));
QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust", legacyVersion));
QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust_counter", legacyVersion));
QueryProcessor.executeInternal(String.format("TRUNCATE legacy_tables.legacy_%s_clust_be_index_summary", legacyVersion));
CacheService.instance.invalidateCounterCache();
CacheService.instance.invalidateKeyCache();
}
@ -746,6 +760,13 @@ public class LegacySSTableTest
QueryProcessor.executeInternal(String.format("UPDATE legacy_tables.legacy_%s_clust_counter SET val = val + 1 WHERE pk = '%s' AND ck='%s'",
format.getLatestVersion(), valPk, valCk + longString));
// note: to emulate BE for offsets in Summary you can comment temporary the following line:
// offset = Integer.reverseBytes(offset);
// in org.apache.cassandra.io.sstable.indexsummary.IndexSummary.IndexSummarySerializer.serialize
QueryProcessor.executeInternal(String.format("INSERT INTO legacy_tables.legacy_%s_clust_be_index_summary (pk, ck, val) VALUES ('%s', '%s', '%s')",
format.getLatestVersion(), valPk, valCk + longString, randomString));
}
}
@ -758,6 +779,7 @@ public class LegacySSTableTest
copySstablesFromTestData(format.getLatestVersion(), "legacy_%s_clust", ksDir);
copySstablesFromTestData(format.getLatestVersion(), "legacy_%s_clust_counter", ksDir);
copySstablesFromTestData(format.getLatestVersion(), "legacy_%s_tuple", ksDir);
copySstablesFromTestData(format.getLatestVersion(), "legacy_%s_clust_be_index_summary", ksDir);
}
public static void copySstablesFromTestData(Version legacyVersion, String tablePattern, File ksDir) throws IOException
@ -773,42 +795,47 @@ public class LegacySSTableTest
for (File srcDir : Keyspace.open(ks).getColumnFamilyStore(table).getDirectories().getCFDirectories())
{
for (File file : srcDir.tryList())
for (File sourceFile : srcDir.tryList())
{
// Sequence IDs represent the C* version used when creating the SSTable, i.e. with #testGenerateSstables() (if not uuid based)
String newSeqId = FBUtilities.getReleaseVersionString().split("-")[0].replaceAll("[^0-9]", "");
File target = new File(cfDir, file.name().replace(legacyVersion + "-1-", legacyVersion + "-" + newSeqId + "-"));
copyFile(cfDir, file, target);
File target = new File(cfDir, sourceFile.name().replace(legacyVersion + "-1-", legacyVersion + "-" + newSeqId + "-"));
copyFile(sourceFile, target);
}
}
}
private static void copySstablesToTestData(String legacyVersion, String table, File cfDir) throws IOException
private static void copySstablesToTestData(String legacyVersion, String table, File targetDir) throws IOException
{
File tableDir = getTableDir(legacyVersion, table);
Assert.assertTrue("The table directory " + tableDir + " was not found", tableDir.isDirectory());
for (File file : tableDir.tryList())
copyFile(cfDir, file);
File testDataTableDir = getTestDataTableDir(legacyVersion, table);
Assert.assertTrue("The table directory " + testDataTableDir + " was not found", testDataTableDir.isDirectory());
for (File sourceTestFile : testDataTableDir.tryList())
copyFileToDir(sourceTestFile, targetDir);
}
private static File getTableDir(String legacyVersion, String table)
private static File getTestDataTableDir(File parentDir, String legacyVersion, String table)
{
return new File(LEGACY_SSTABLE_ROOT, String.format("%s/legacy_tables/%s", legacyVersion, table));
return new File(parentDir, String.format("%s/legacy_tables/%s", legacyVersion, table));
}
public static void copyFile(File cfDir, File file) throws IOException
private static File getTestDataTableDir(String legacyVersion, String table)
{
copyFile(cfDir, file, new File(cfDir, file.name()));
return getTestDataTableDir(LEGACY_SSTABLE_ROOT, legacyVersion, table);
}
public static void copyFile(File cfDir, File file, File target) throws IOException
public static void copyFileToDir(File sourceFile, File targetDir) throws IOException
{
copyFile(sourceFile, new File(targetDir, sourceFile.name()));
}
public static void copyFile(File sourceFile, File targetFile) throws IOException
{
byte[] buf = new byte[65536];
if (file.isFile())
if (sourceFile.isFile())
{
int rd;
try (FileInputStreamPlus is = new FileInputStreamPlus(file);
FileOutputStreamPlus os = new FileOutputStreamPlus(target);)
try (FileInputStreamPlus is = new FileInputStreamPlus(sourceFile);
FileOutputStreamPlus os = new FileOutputStreamPlus(targetFile);)
{
while ((rd = is.read(buf)) >= 0)
os.write(buf, 0, rd);

148
test/unit/org/apache/cassandra/utils/memory/LittleEndianMemoryUtilTest.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils.memory;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.junit.Assert;
import org.junit.Test;
public class LittleEndianMemoryUtilTest
{
private static final int TEST_BUFFER_LENGTH = 8;
private final ByteBuffer directBuffer = ByteBuffer.allocateDirect(TEST_BUFFER_LENGTH);
{
directBuffer.order(ByteOrder.LITTLE_ENDIAN);
}
private final long address = LittleEndianMemoryUtil.getAddress(directBuffer);
@Test
public void testGetSetLong()
{
long originalValue = 0xAB_CD_EF_12_34_56_78_90L;
directBuffer.putLong(originalValue);
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getLong(address));
directBuffer.rewind();
directBuffer.putLong(0);
LittleEndianMemoryUtil.setLong(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getLong(0));
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getLong(address));
}
@Test
public void testGetSetInt()
{
int originalValue = 0xAB_CD_EF_12;
directBuffer.putInt(originalValue);
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getInt(address));
directBuffer.rewind();
directBuffer.putInt(0);
LittleEndianMemoryUtil.setInt(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getInt(0));
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getInt(address));
}
@Test
public void testGetSetUnsighedShort()
{
short originalValue = (short) 0xAB_CD;
directBuffer.putShort(originalValue);
Assert.assertEquals(originalValue & 0xffff, LittleEndianMemoryUtil.getUnsignedShort(address));
directBuffer.rewind();
directBuffer.putShort((short) 0);
LittleEndianMemoryUtil.setShort(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getShort(0));
Assert.assertEquals(originalValue & 0xffff, LittleEndianMemoryUtil.getUnsignedShort(address));
}
@Test
public void testGetSetLongByBytes()
{
long originalValue = 0xAB_CD_EF_12_34_56_78_90L;
directBuffer.putLong(originalValue);
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getLongByByte(address));
directBuffer.rewind();
directBuffer.putLong(0);
LittleEndianMemoryUtil.putLongByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getLong(0));
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getLongByByte(address));
}
@Test
public void testGetSetIntByBytes()
{
int originalValue = 0xAB_CD_EF_12;
directBuffer.putInt(originalValue);
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getIntByByte(address));
directBuffer.rewind();
directBuffer.putInt(0);
LittleEndianMemoryUtil.putIntByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getInt(0));
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getIntByByte(address));
}
@Test
public void testGetSetShortByBytes()
{
short originalValue = (short) 0xAB_CD;
directBuffer.putShort(originalValue);
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getShortByByte(address));
directBuffer.rewind();
directBuffer.putShort((short) 0);
LittleEndianMemoryUtil.putShortByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getShort(0));
Assert.assertEquals(originalValue, LittleEndianMemoryUtil.getShortByByte(address));
}
@Test
public void testGetHollowDirectByteBuffer()
{
ByteBuffer byteBuffer = LittleEndianMemoryUtil.getHollowDirectByteBuffer();
Assert.assertEquals(directBuffer.getClass(), byteBuffer.getClass());
Assert.assertEquals(ByteOrder.LITTLE_ENDIAN, byteBuffer.order());
}
@Test
public void testGetByteBuffer()
{
ByteBuffer byteBuffer = LittleEndianMemoryUtil.getByteBuffer(address, TEST_BUFFER_LENGTH);
Assert.assertEquals(directBuffer.getClass(), byteBuffer.getClass());
Assert.assertEquals(ByteOrder.LITTLE_ENDIAN, byteBuffer.order());
Assert.assertEquals(TEST_BUFFER_LENGTH, byteBuffer.capacity());
Assert.assertEquals(0, byteBuffer.position());
}
}

148
test/unit/org/apache/cassandra/utils/memory/NativeEndianMemoryUtilTest.java Normal file
View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils.memory;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.junit.Assert;
import org.junit.Test;
public class NativeEndianMemoryUtilTest
{
private static final int TEST_BUFFER_LENGTH = 8;
private final ByteBuffer directBuffer = ByteBuffer.allocateDirect(TEST_BUFFER_LENGTH);
{
directBuffer.order(ByteOrder.nativeOrder());
}
private final long address = NativeEndianMemoryUtil.getAddress(directBuffer);
@Test
public void testGetSetLong()
{
long originalValue = 0xAB_CD_EF_12_34_56_78_90L;
directBuffer.putLong(originalValue);
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getLong(address));
directBuffer.rewind();
directBuffer.putLong(0);
NativeEndianMemoryUtil.setLong(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getLong(0));
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getLong(address));
}
@Test
public void testGetSetInt()
{
int originalValue = 0xAB_CD_EF_12;
directBuffer.putInt(originalValue);
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getInt(address));
directBuffer.rewind();
directBuffer.putInt(0);
NativeEndianMemoryUtil.setInt(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getInt(0));
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getInt(address));
}
@Test
public void testGetSetUnsighedShort()
{
short originalValue = (short) 0xAB_CD;
directBuffer.putShort(originalValue);
Assert.assertEquals(originalValue & 0xffff, NativeEndianMemoryUtil.getUnsignedShort(address));
directBuffer.rewind();
directBuffer.putShort((short) 0);
NativeEndianMemoryUtil.setShort(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getShort(0));
Assert.assertEquals(originalValue & 0xffff, NativeEndianMemoryUtil.getUnsignedShort(address));
}
@Test
public void testGetSetLongByBytes()
{
long originalValue = 0xAB_CD_EF_12_34_56_78_90L;
directBuffer.putLong(originalValue);
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getLongByByte(address));
directBuffer.rewind();
directBuffer.putLong(0);
NativeEndianMemoryUtil.putLongByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getLong(0));
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getLongByByte(address));
}
@Test
public void testGetSetIntByBytes()
{
int originalValue = 0xAB_CD_EF_12;
directBuffer.putInt(originalValue);
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getIntByByte(address));
directBuffer.rewind();
directBuffer.putInt(0);
NativeEndianMemoryUtil.putIntByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getInt(0));
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getIntByByte(address));
}
@Test
public void testGetSetShortByBytes()
{
short originalValue = (short) 0xAB_CD;
directBuffer.putShort(originalValue);
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getShortByByte(address));
directBuffer.rewind();
directBuffer.putShort((short) 0);
NativeEndianMemoryUtil.putShortByByte(address, originalValue);
Assert.assertEquals(originalValue, directBuffer.getShort(0));
Assert.assertEquals(originalValue, NativeEndianMemoryUtil.getShortByByte(address));
}
@Test
public void testGetHollowDirectByteBuffer()
{
ByteBuffer byteBuffer = NativeEndianMemoryUtil.getHollowDirectByteBuffer();
Assert.assertEquals(directBuffer.getClass(), byteBuffer.getClass());
Assert.assertEquals(ByteOrder.nativeOrder(), byteBuffer.order());
}
@Test
public void testGetByteBuffer()
{
ByteBuffer byteBuffer = NativeEndianMemoryUtil.getByteBuffer(address, TEST_BUFFER_LENGTH);
Assert.assertEquals(directBuffer.getClass(), byteBuffer.getClass());
Assert.assertEquals(ByteOrder.nativeOrder(), byteBuffer.order());
Assert.assertEquals(TEST_BUFFER_LENGTH, byteBuffer.capacity());
Assert.assertEquals(0, byteBuffer.position());
}
}