natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Accord: Serialization Improvements 

Improve:
 - Introduce pre/accept fast execution flags
 - Introduce searchable Deps serialization
 - Flatten AccordRoutingKey(s) into single type, using sentinel bits
 - Introduce new fast byte-comparable serialization methods for Token and TableId to support above
Fix:
 - Fix journal re-serialization logic
 - Enable RandomPartitioner for Accord by supporting fixed-width serialization for RouteIndex

patch by Benedict; reviewed by Alex Petrov for CASSANDRA-20349
This commit is contained in:
Benedict Elliott Smith 2025-02-15 22:45:19 +00:00 committed by David Capwell
parent 1d5146a22e
commit e9a0d8ef8e
98 changed files with 2947 additions and 2536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit a4a8255bebb6804b8f55289f2fbc846a5253e66c
Subproject commit 58f107625a183d77b154221efd2f6f0623214027

5
src/java/org/apache/cassandra/db/DecoratedKey.java
View File

@ -90,6 +90,11 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey
return cmp == 0 ? ByteBufferUtil.compareUnsigned(getKey(), otherKey.getKey()) : cmp;
}
public int compareBytesOnly(DecoratedKey that)
{
return ByteBufferUtil.compareUnsigned(getKey(), that.getKey());
}
public static int compareTo(IPartitioner partitioner, ByteBuffer key, PartitionPosition position)
{
// delegate to Token.KeyBound if needed

59
src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
View File

@ -36,12 +36,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.local.Cleanup;
import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.RedundantBefore;
import accord.utils.Invariants;
import accord.utils.UnhandledEnum;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.AbstractCompactionController;
@ -91,22 +89,22 @@ import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo;
import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfos;
import org.apache.cassandra.service.accord.JournalKey;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.service.paxos.uncommitted.PaxosRows;
import org.apache.cassandra.utils.TimeUUID;
import static accord.local.Cleanup.ERASE;
import static accord.local.Cleanup.EXPUNGE;
import static accord.local.Cleanup.Input.PARTIAL;
import static accord.local.Cleanup.NO;
import static com.google.common.base.Preconditions.checkState;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeysAccessor;
import static org.apache.cassandra.service.accord.AccordKeyspace.CFKAccessor;
/**
* Merge multiple iterators over the content of sstable into a "compacted" iterator.
@ -220,7 +218,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
if (isAccordJournal(cfs))
return new AccordJournalPurger(accordService, cfs);
if (isAccordCommandsForKey(cfs))
return new AccordCommandsForKeyPurger(AccordKeyspace.CommandsForKeysAccessor, accordService);
return new AccordCommandsForKeyPurger(AccordKeyspace.CFKAccessor, accordService);
throw new IllegalArgumentException("Unhandled accord table: " + cfs.keyspace.getName() + '.' + cfs.name);
}
@ -782,21 +780,24 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
class AccordCommandsForKeyPurger extends AbstractPurger
{
final CommandsForKeyAccessor accessor;
final Int2ObjectHashMap<RedundantBefore> redundantBefores;
final AccordCompactionInfos compactionInfos;
AccordCompactionInfo info;
int storeId;
TokenKey partitionKey;
TokenKey tokenKey;
AccordCommandsForKeyPurger(CommandsForKeyAccessor accessor, Supplier<IAccordService> accordService)
{
this.accessor = accessor;
this.redundantBefores = accordService.get().getCompactionInfo().redundantBefores;
this.compactionInfos = new AccordCompactionInfos(null, accordService.get().getCompactionInfo());
}
protected void beginPartition(UnfilteredRowIterator partition)
{
ByteBuffer[] partitionKeyComponents = accessor.splitPartitionKey(partition.partitionKey());
storeId = accessor.getStoreId(partitionKeyComponents);
partitionKey = accessor.getKey(partitionKeyComponents);
ByteBuffer key = partition.partitionKey().getKey();
storeId = CommandsForKeyAccessor.getCommandStoreId(key);
info = compactionInfos.get(storeId);
tokenKey = info == null ? null : CommandsForKeyAccessor.getUserTableKey(info.tableId, key);
}
@Override
@ -804,16 +805,16 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
{
updateProgress();
RedundantBefore redundantBefore = redundantBefores.get(storeId);
// TODO (expected): if the store has been retired, this should return null
if (redundantBefore == null)
// TODO (required): if the store has been retired, this should return null
if (info == null)
return row;
RedundantBefore.Entry redundantBeforeEntry = redundantBefore.get(partitionKey.toUnseekable());
RedundantBefore redundantBefore = info.redundantBefore;
RedundantBefore.Entry redundantBeforeEntry = redundantBefore.get(tokenKey.toUnseekable());
if (redundantBeforeEntry == null)
return row;
return CommandsForKeysAccessor.withoutRedundantCommands(partitionKey, row, redundantBeforeEntry);
return CFKAccessor.withoutRedundantCommands(tokenKey, row, redundantBeforeEntry);
}
@Override
@ -826,14 +827,13 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
class AccordJournalPurger extends AbstractPurger
{
final Int2ObjectHashMap<RedundantBefore> redundantBefores;
final Int2ObjectHashMap<DurableBefore> durableBefores;
final Int2ObjectHashMap<CommandStores.RangesForEpoch> ranges;
final AccordCompactionInfos infos;
final ColumnMetadata recordColumn;
final ColumnMetadata versionColumn;
final AccordService service;
final AccordAgent agent;
AccordCompactionInfo info;
JournalKey key = null;
Object builder = null;
FlyweightSerializer<Object, Object> serializer = null;
@ -847,12 +847,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
service = (AccordService) serviceSupplier.get();
// TODO: test serialization version logic
userVersion = service.journalConfiguration().userVersion();
IAccordService.CompactionInfo compactionInfo = service.getCompactionInfo();
this.agent = service.agent();
this.redundantBefores = compactionInfo.redundantBefores;
this.ranges = compactionInfo.ranges;
this.durableBefores = compactionInfo.durableBefores;
this.infos = service.getCompactionInfo();
this.recordColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("record", false));
this.versionColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("user_version", false));
}
@ -912,9 +909,13 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
return partition;
}
RedundantBefore redundantBefore = redundantBefores.get(key.commandStoreId);
DurableBefore durableBefore = durableBefores.get(key.commandStoreId);
Cleanup cleanup = commandBuilder.shouldCleanup(PARTIAL, agent, redundantBefore, durableBefore);
if (info != null && info.commandStoreId != key.commandStoreId) info = null;
if (info == null) info = infos.get(key.commandStoreId);
// TODO (required): should return null if commandStore has been removed
if (info == null)
return partition;
DurableBefore durableBefore = infos.durableBefore;
Cleanup cleanup = commandBuilder.shouldCleanup(PARTIAL, agent, info.redundantBefore, durableBefore);
if (cleanup != NO)
{
switch (cleanup)
@ -933,7 +934,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
PartitionUpdate.SimpleBuilder newVersion = PartitionUpdate.simpleBuilder(AccordKeyspace.Journal, partition.partitionKey());
Row.SimpleBuilder rowBuilder = newVersion.row(firstClustering);
rowBuilder.add("record", commandBuilder.asByteBuffer(redundantBefore, userVersion))
rowBuilder.add("record", commandBuilder.asByteBuffer(userVersion))
.add("user_version", userVersion);
return newVersion.build().unfilteredIterator();

29
src/java/org/apache/cassandra/db/marshal/ByteBufferAccessor.java
View File

@ -315,6 +315,35 @@ public class ByteBufferAccessor implements ValueAccessor<ByteBuffer>
return TypeSizes.FLOAT_SIZE;
}
@Override
public int putLeastSignificantBytes(ByteBuffer dst, int offset, long register, int bytes)
{
if (dst.remaining() < Long.BYTES)
{
return ValueAccessor.putLeastSignificantBytes(this, dst, offset, register, bytes);
}
else
{
int pos = dst.position() + offset;
dst.putLong(pos, register << (64 - (bytes * 8)));
}
return bytes;
}
@Override
public long getLeastSignificantBytes(ByteBuffer dst, int offset, int bytes)
{
if (dst.remaining() < Long.BYTES)
{
return ValueAccessor.getLeastSignificantBytes(this, dst, offset, bytes);
}
else
{
int pos = dst.position() + offset;
return dst.getLong(pos) >>> (64 - (bytes * 8));
}
}
@Override
public ByteBuffer empty()
{

85
src/java/org/apache/cassandra/db/marshal/ValueAccessor.java
View File

@ -340,6 +340,11 @@ public interface ValueAccessor<V>
return VIntCoding.getVInt32(value, this, offset);
}
default long getLeastSignificantBytes(V value, int offset, int bytes)
{
return getLeastSignificantBytes(this, value, offset, bytes);
}
float getFloat(V value, int offset);
double getDouble(V value, int offset);
/** returns a long from offset 0 */
@ -399,6 +404,18 @@ public interface ValueAccessor<V>
return ByteArrayAccessor.instance.copyTo(src, srcOffset, dst, this, offset, length);
}
/**
* An efficient way to write the type {@code bytes} of a long
*
* @param register - the long value to be written
* @param bytes - the number of bytes the register occupies. Valid values are between 1 and 8 inclusive.
* @throws IOException
*/
default int putLeastSignificantBytes(V dst, int offset, long register, int bytes)
{
return putLeastSignificantBytes(this, dst, offset, register, bytes);
}
default int putBytes(V dst, int offset, byte[] src)
{
return putBytes(dst, offset, src, 0, src.length);
@ -493,4 +510,72 @@ public interface ValueAccessor<V>
{
return compare(left, leftAccessor, right, rightAccessor) == 0;
}
public static <V> int putLeastSignificantBytes(ValueAccessor<V> accessor, V dst, int offset, long register, int bytes)
{
switch (bytes)
{
case 0:
break;
case 1:
accessor.putByte(dst, offset, (byte)register);
break;
case 2:
accessor.putShort(dst, offset, (short)register);
break;
case 3:
accessor.putShort(dst, offset, (short)(register >>> 8));
accessor.putByte(dst, offset, (byte)register);
break;
case 4:
accessor.putInt(dst, offset, (int)register);
break;
case 5:
accessor.putInt(dst, offset, (int)(register >>> 8));
accessor.putByte(dst, offset, (byte)register);
break;
case 6:
accessor.putInt(dst, offset, (int)(register >>> 16));
accessor.putShort(dst, offset, (short)register);
break;
case 7:
accessor.putInt(dst, offset, (int)(register >>> 24));
accessor.putShort(dst, offset, (short)(register >> 8));
accessor.putByte(dst, offset, (byte)register);
break;
case 8:
accessor.putLong(dst, offset, register);
break;
default:
throw new IllegalArgumentException();
}
return bytes;
}
public static <V> long getLeastSignificantBytes(ValueAccessor<V> accessor, V dst, int offset, int bytes)
{
switch (bytes)
{
case 0: return 0;
case 1: return accessor.getByte(dst, offset);
case 2: return accessor.getShort(dst, offset);
case 3:
return ((long)accessor.getShort(dst, offset) << 8)
| (long)accessor.getByte(dst, offset + 2);
case 4:
return accessor.getInt(dst, offset);
case 5:
return ((long)accessor.getInt(dst, offset) << 8)
| (long)accessor.getByte(dst, offset + 4);
case 6:
return ((long)accessor.getInt(dst, offset) << 16)
| (long)accessor.getShort(dst, offset + 4);
case 7:
return ((long)accessor.getInt(dst, offset) << 24)
| ((long)accessor.getShort(dst, offset + 4) << 8)
| (long)accessor.getByte(dst, offset + 6);
case 8: return accessor.getLong(dst, offset);
default: throw new IllegalArgumentException();
}
}
}

14
src/java/org/apache/cassandra/db/virtual/AccordDebugKeyspace.java
View File

@ -67,8 +67,7 @@ import org.apache.cassandra.service.accord.AccordExecutor;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.CommandStoreTxnBlockedGraph;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.service.consensus.migration.TableMigrationState;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -616,15 +615,8 @@ public class AccordDebugKeyspace extends VirtualKeyspace
private static ByteBuffer decompose(RoutingKey routingKey)
{
AccordRoutingKey key = (AccordRoutingKey) routingKey;
switch (key.kindOfRoutingKey())
{
case SENTINEL:
case TOKEN:
return ROUTING_KEY_TYPE.pack(UUIDType.instance.decompose(key.table().asUUID()), bytes(key.suffix()));
default:
throw new IllegalStateException("Unhandled key Kind " + key.kindOfRoutingKey());
}
TokenKey key = (TokenKey) routingKey;
return ROUTING_KEY_TYPE.pack(UUIDType.instance.decompose(key.table().asUUID()), bytes(key.suffix().toString()));
}
private static TableMetadata parse(String keyspace, String table, String comment, String schema, AbstractType<?> partitionKeyType)

2
src/java/org/apache/cassandra/db/virtual/AccordVirtualTables.java
View File

@ -187,7 +187,7 @@ public class AccordVirtualTables
private static String toStringNoTable(TokenRange tr)
{
// TokenRange extends Range.EndInclusive
return "(" + tr.start().suffix() + ", " + tr.end().suffix() + "]";
return "(" + tr.start().printableSuffix() + ", " + tr.end().printableSuffix() + "]";
}
private static Map<TableId, List<TokenRange>> groupByTable(Ranges ranges)

11
src/java/org/apache/cassandra/dht/AccordBytesSplitter.java
View File

@ -23,8 +23,7 @@ import java.math.BigInteger;
import accord.api.RoutingKey;
import accord.primitives.Ranges;
import accord.utils.Invariants;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import static accord.utils.Invariants.requireArgument;
import static java.math.BigInteger.ONE;
@ -51,8 +50,8 @@ public class AccordBytesSplitter extends AccordSplitter
if (bytesLength == 0)
{
requireArgument(ranges.size() <= 1);
requireArgument(ranges.isEmpty() || ranges.get(0).start().getClass() == SentinelKey.class);
requireArgument(ranges.isEmpty() || ranges.get(0).end().getClass() == SentinelKey.class);
requireArgument(ranges.isEmpty() || ((TokenKey)ranges.get(0).start()).isMin());
requireArgument(ranges.isEmpty() || ((TokenKey)ranges.get(0).end()).isMax());
// Intentionally does not match 16 that is used by ServerTestUtils.getRandomToken to elicit breakage
bytesLength = 8;
}
@ -94,9 +93,7 @@ public class AccordBytesSplitter extends AccordSplitter
private static int byteLength(RoutingKey routingKey)
{
AccordRoutingKey accordKey = (AccordRoutingKey) routingKey;
if (accordKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
return 0;
TokenKey accordKey = (TokenKey) routingKey;
return byteLength(accordKey.token());
}

18
src/java/org/apache/cassandra/dht/AccordSplitter.java
View File

@ -24,9 +24,7 @@ import accord.local.ShardDistributor;
import accord.primitives.Range;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import static java.math.BigInteger.ZERO;
@ -41,19 +39,21 @@ public abstract class AccordSplitter implements ShardDistributor.EvenSplit.Split
public BigInteger sizeOf(accord.primitives.Range range)
{
// note: minimum value
BigInteger start = range.start() instanceof SentinelKey ? minimumValue() : valueForToken(((AccordRoutingKey)range.start()).token());
BigInteger end = range.end() instanceof SentinelKey ? maximumValue() : valueForToken(((AccordRoutingKey)range.end()).token());
TokenKey startBound = (TokenKey)range.start();
TokenKey endBound = (TokenKey)range.end();
BigInteger start = startBound.isMin() ? minimumValue() : valueForToken(((TokenKey)range.start()).token());
BigInteger end = endBound.isMax() ? maximumValue() : valueForToken(((TokenKey)range.end()).token());
return end.subtract(start);
}
@Override
public TokenRange subRange(accord.primitives.Range range, BigInteger startOffset, BigInteger endOffset)
{
AccordRoutingKey startBound = (AccordRoutingKey)range.start();
AccordRoutingKey endBound = (AccordRoutingKey)range.end();
TokenKey startBound = (TokenKey)range.start();
TokenKey endBound = (TokenKey)range.end();
BigInteger start = startBound instanceof SentinelKey ? minimumValue() : valueForToken(startBound.token());
BigInteger end = endBound instanceof SentinelKey ? maximumValue() : valueForToken(endBound.token());
BigInteger start = startBound.isMin() ? minimumValue() : valueForToken(startBound.token());
BigInteger end = endBound.isMax() ? maximumValue() : valueForToken(endBound.token());
BigInteger sizeOfRange = end.subtract(start);
TableId tableId = startBound.table();

65
src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.cassandra.dht;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@ -35,10 +36,14 @@ import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.TokenKey.Serializer;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Hex;
@ -135,6 +140,12 @@ public class ByteOrderedPartitioner implements IPartitioner
return hashCode();
}
@Override
public TokenFactory tokenFactory()
{
return tokenFactory;
}
@Override
public double size(Token next)
{
@ -285,7 +296,7 @@ public class ByteOrderedPartitioner implements IPartitioner
return new BytesToken(buffer);
}
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
private static final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
{
@ -342,6 +353,58 @@ public class ByteOrderedPartitioner implements IPartitioner
return tokenFactory;
}
@Override
public boolean accordSupported()
{
return true;
}
@Override
public final void accordSerialize(Token token, DataOutputPlus out) throws IOException
{
Serializer.serializeWithEscapes(((BytesToken)token).token, out);
}
@Override
public final void accordSerialize(Token token, ByteBuffer out)
{
Serializer.serializeWithEscapes(((BytesToken)token).token, out);
}
@Override
public final Token accordDeserialize(DataInputPlus in, int length) throws IOException
{
byte[] bytes = Serializer.deserializeWithEscapes(in, length);
return new BytesToken(bytes);
}
@Override
public final Token accordDeserialize(ByteBuffer in, int length)
{
byte[] bytes = Serializer.deserializeWithEscapes(in, length);
return new BytesToken(bytes);
}
@Override
public final <V> Token accordDeserialize(V src, ValueAccessor<V> accessor, int offset, int length)
{
byte[] bytes = Serializer.deserializeWithEscapes(src, accessor, offset, length);
return new BytesToken(bytes);
}
@Override
public final int accordSerializedSize(Token token)
{
byte[] bytes = ((BytesToken)token).token;
return Serializer.serializedSize(bytes);
}
@Override
public final int accordFixedLength()
{
return -1;
}
public boolean preservesOrder()
{
return true;

13
src/java/org/apache/cassandra/dht/IPartitioner.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.cassandra.dht;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
@ -27,6 +28,9 @@ import java.util.function.Function;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import javax.annotation.Nullable;
@ -166,4 +170,13 @@ public interface IPartitioner
{
return Integer.MIN_VALUE;
}
default boolean accordSupported() { return false; }
default void accordSerialize(Token token, DataOutputPlus out) throws IOException { throw new UnsupportedOperationException(); }
default void accordSerialize(Token token, ByteBuffer out) { throw new UnsupportedOperationException(); }
default Token accordDeserialize(DataInputPlus in, int length) throws IOException { throw new UnsupportedOperationException(); }
default Token accordDeserialize(ByteBuffer in, int length) { throw new UnsupportedOperationException(); }
default <V> Token accordDeserialize(V src, ValueAccessor<V> accessor, int offset, int length) { throw new UnsupportedOperationException(); }
default int accordSerializedSize(Token token) { throw new UnsupportedOperationException(); }
default int accordFixedLength() { throw new UnsupportedOperationException(); }
}

341
src/java/org/apache/cassandra/dht/LocalCompositePrefixPartitioner.java
View File

@ -1,341 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.dht;
import accord.utils.Invariants;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.MergeIterator;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
* Local partitioner that supports doing range scans of composite primary keys using composite prefixes using the iterator
* methods it provides. This is neccessary for correctly handling exclusive start and inclusive end prefixes, since
* these won't work as intended given normal byte/component comparisons
*/
public class LocalCompositePrefixPartitioner extends LocalPartitioner
{
/**
* Composite type that only compares
*/
private static class PrefixCompositeType extends CompositeType
{
public PrefixCompositeType(List<AbstractType<?>> types)
{
super(types);
}
@Override
protected <VL, VR> int compareCustomRemainder(VL left, ValueAccessor<VL> accessorL, int offsetL, VR right, ValueAccessor<VR> accessorR, int offsetR)
{
return 0;
}
}
public abstract class AbstractCompositePrefixToken extends LocalToken
{
public AbstractCompositePrefixToken(ByteBuffer token)
{
super(token);
}
@Override
public int compareTo(Token o)
{
Invariants.requireArgument(o instanceof AbstractCompositePrefixToken);
AbstractCompositePrefixToken that = (AbstractCompositePrefixToken) o;
CompositeType comparator = comparatorForPrefixLength(Math.min(this.prefixSize(), that.prefixSize()));
return comparator.compare(this.token, that.token);
}
@Override
public int hashCode()
{
throw new UnsupportedOperationException();
}
@Override
public boolean equals(Object obj)
{
if (!(obj instanceof AbstractCompositePrefixToken))
return false;
return compareTo((AbstractCompositePrefixToken) obj) == 0;
}
@Override
public ByteSource asComparableBytes(ByteComparable.Version version)
{
return comparatorForPrefixLength(prefixSize()).asComparableBytes(ByteBufferAccessor.instance, token, version);
}
ByteBuffer token()
{
return token;
}
abstract int prefixSize();
}
public class FullToken extends AbstractCompositePrefixToken
{
public FullToken(ByteBuffer token)
{
super(token);
}
@Override
int prefixSize()
{
return prefixComparators.size();
}
}
public class PrefixToken extends AbstractCompositePrefixToken
{
final int prefixSize;
public PrefixToken(ByteBuffer token, int prefixSize)
{
super(token);
Invariants.requireArgument(prefixSize > 0);
this.prefixSize = prefixSize;
}
@Override
int prefixSize()
{
return prefixSize;
}
}
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
@Override
public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
{
ByteBuffer tokenData = comparator.fromComparableBytes(ByteBufferAccessor.instance, comparableBytes, version);
return new FullToken(tokenData);
}
@Override
public ByteBuffer toByteArray(Token token)
{
return ((FullToken)token).token();
}
@Override
public Token fromByteArray(ByteBuffer bytes)
{
return new FullToken(bytes);
}
@Override
public String toString(Token token)
{
return comparator.getString(((FullToken)token).token());
}
@Override
public void validate(String token)
{
comparator.validate(comparator.fromString(token));
}
@Override
public Token fromString(String string)
{
return new FullToken(comparator.fromString(string));
}
};
private final List<CompositeType> prefixComparators;
public LocalCompositePrefixPartitioner(CompositeType comparator)
{
super(comparator);
ArrayList<CompositeType> comparators = new ArrayList<>(comparator.subTypes().size());
comparators.add(comparator);
List<AbstractType<?>> subtypes = comparator.subTypes();
subtypes = subtypes.subList(0, subtypes.size() - 1);
while (!subtypes.isEmpty())
{
comparators.add(new PrefixCompositeType(subtypes));
subtypes = subtypes.subList(0, subtypes.size() - 1);
}
prefixComparators = ImmutableList.copyOf(Lists.reverse(comparators));
}
@SuppressWarnings("rawtypes")
public LocalCompositePrefixPartitioner(AbstractType... types)
{
this(CompositeType.getInstance(types));
}
private CompositeType comparatorForPrefixLength(int size)
{
return prefixComparators.get(size - 1);
}
public ByteBuffer createPrefixKey(Object... values)
{
return comparatorForPrefixLength(values.length).decompose(values);
}
public AbstractCompositePrefixToken createPrefixToken(Object... values)
{
ByteBuffer key = createPrefixKey(values);
return values.length == prefixComparators.size() ? new FullToken(key) : new PrefixToken(key, values.length);
}
public DecoratedKey decoratedKey(Object... values)
{
Invariants.requireArgument(values.length == prefixComparators.size());
ByteBuffer key = createPrefixKey(values);
return decorateKey(key);
}
@Override
public LocalToken getToken(ByteBuffer key)
{
return new FullToken(key);
}
@Override
public LocalToken getMinimumToken()
{
return new FullToken(ByteBufferUtil.EMPTY_BYTE_BUFFER);
}
@Override
public Token.TokenFactory getTokenFactory()
{
return tokenFactory;
}
/**
* Returns a DecoratedKey iterator for the given range. Skips reading data files for sstable formats with a partition index file
*/
private static CloseableIterator<DecoratedKey> keyIterator(Memtable memtable, AbstractBounds<PartitionPosition> range)
{
AbstractBounds<PartitionPosition> memtableRange = range.withNewRight(memtable.metadata().partitioner.getMinimumToken().maxKeyBound());
DataRange dataRange = new DataRange(memtableRange, new ClusteringIndexSliceFilter(Slices.ALL, false));
UnfilteredPartitionIterator iter = memtable.partitionIterator(ColumnFilter.NONE, dataRange, SSTableReadsListener.NOOP_LISTENER);
int rangeStartCmpMin = range.isStartInclusive() ? 0 : 1;
int rangeEndCmpMax = range.isEndInclusive() ? 0 : -1;
return new AbstractIterator<>()
{
@Override
protected DecoratedKey computeNext()
{
while (iter.hasNext())
{
DecoratedKey key = iter.next().partitionKey();
if (key.compareTo(range.left) < rangeStartCmpMin)
continue;
if (key.compareTo(range.right) > rangeEndCmpMax)
break;
return key;
}
return endOfData();
}
@Override
public void close()
{
iter.close();
}
};
}
public static CloseableIterator<DecoratedKey> keyIterator(TableMetadata metadata, AbstractBounds<PartitionPosition> range) throws IOException
{
ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata);
ColumnFamilyStore.ViewFragment view = cfs.select(View.selectLive(range));
List<CloseableIterator<?>> closeableIterators = new ArrayList<>();
List<Iterator<DecoratedKey>> iterators = new ArrayList<>();
try
{
for (Memtable memtable : view.memtables)
{
CloseableIterator<DecoratedKey> iter = keyIterator(memtable, range);
iterators.add(iter);
closeableIterators.add(iter);
}
for (SSTableReader sstable : view.sstables)
{
CloseableIterator<DecoratedKey> iter = sstable.keyIterator(range);
iterators.add(iter);
closeableIterators.add(iter);
}
}
catch (Throwable e)
{
for (CloseableIterator<?> iter: closeableIterators)
{
try
{
iter.close();
}
catch (Throwable e2)
{
e.addSuppressed(e2);
}
}
throw e;
}
return MergeIterator.get(iterators, DecoratedKey::compareTo, new MergeIterator.Reducer.Trivial<>());
}
}

70
src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
View File

@ -40,7 +40,9 @@ import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.MurmurHash;
@ -243,6 +245,12 @@ public class Murmur3Partitioner implements IPartitioner
return Long.hashCode(token);
}
@Override
public TokenFactory tokenFactory()
{
return tokenFactory;
}
@Override
public double size(Token next)
{
@ -258,7 +266,7 @@ public class Murmur3Partitioner implements IPartitioner
// CASSANDRA-17109 Added the below checks, but paxos tests were not updated, rather than fix
// the paxos tests, disabling the checks for now. The current paxos tests bias twards MIN but
// not for MAX, which makes the test very flaky as when MAX is generated the test fails...
// TODO (mustfix): This was done as part of CEP-15 and needs to be added back
// TODO (required): this check breaks a bunch of tests, but should be re-enabled
// if (token == MAXIMUM)
// throw new IllegalArgumentException("Cannot increase above MAXIMUM");
@ -323,6 +331,58 @@ public class Murmur3Partitioner implements IPartitioner
return MAXIMUM_TOKEN_SIZE;
}
public final boolean accordSupported()
{
return true;
}
@Override
public final void accordSerialize(Token token, DataOutputPlus out) throws IOException
{
out.writeLong(flip(((LongToken)token).token));
}
@Override
public final void accordSerialize(Token token, ByteBuffer out)
{
out.putLong(flip(((LongToken)token).token));
}
@Override
public final Token accordDeserialize(DataInputPlus in, int length) throws IOException
{
return new LongToken(flip(in.readLong()));
}
@Override
public final Token accordDeserialize(ByteBuffer in, int length)
{
return new LongToken(flip(in.getLong()));
}
@Override
public final <V> Token accordDeserialize(V src, ValueAccessor<V> accessor, int offset, int length)
{
return new LongToken(flip(accessor.getLong(src, offset)));
}
@Override
public final int accordSerializedSize(Token token)
{
return 8;
}
@Override
public final int accordFixedLength()
{
return 8;
}
private static long flip(long value)
{
return value ^ 0x8000000000000000L;
}
private long[] getHash(ByteBuffer key)
{
long[] hash = new long[2];
@ -391,7 +451,7 @@ public class Murmur3Partitioner implements IPartitioner
return tokenFactory;
}
private final Token.TokenFactory tokenFactory = new Token.TokenFactory()
private static final Token.TokenFactory tokenFactory = new Token.TokenFactory()
{
public Token fromComparableBytes(ByteSource.Peekable comparableBytes, ByteComparable.Version version)
{
@ -411,6 +471,12 @@ public class Murmur3Partitioner implements IPartitioner
out.writeLong(((LongToken) token).token);
}
@Override
public Token deserialize(DataInputPlus in, IPartitioner p) throws IOException
{
return new LongToken(in.readLong());
}
@Override
public void serialize(Token token, ByteBuffer out)
{

8
src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java
View File

@ -39,7 +39,7 @@ import org.apache.cassandra.gms.VersionedValue;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
@ -273,7 +273,7 @@ public class OrderPreservingPartitioner implements IPartitioner
{
// allTokens will contain the count and be returned, sorted_ranges is shorthand for token<->token math.
Map<Token, Float> allTokens = new HashMap<Token, Float>();
List<Range<Token>> sortedRanges = new ArrayList<Range<Token>>(sortedTokens.size());
List<Range<Token>> sortedRanges = new ArrayList<>(sortedTokens.size());
// this initializes the counts to 0 and calcs the ranges in order.
Token lastToken = sortedTokens.get(sortedTokens.size() - 1);
@ -364,9 +364,7 @@ public class OrderPreservingPartitioner implements IPartitioner
private static int charLength(RoutingKey routingKey)
{
AccordRoutingKey accordKey = (AccordRoutingKey) routingKey;
if (accordKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
return 0;
TokenKey accordKey = (TokenKey) routingKey;
return charLength(accordKey.token());
}

82
src/java/org/apache/cassandra/dht/RandomPartitioner.java
View File

@ -40,7 +40,9 @@ import org.apache.cassandra.db.marshal.ByteArrayAccessor;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.IntegerType;
import org.apache.cassandra.db.marshal.PartitionerDefinedOrder;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@ -405,6 +407,86 @@ public class RandomPartitioner implements IPartitioner
return ignore -> splitter;
}
public final boolean accordSupported()
{
return true;
}
private static final byte[] EMPTY_BYTES = new byte[16];
@Override
public final void accordSerialize(Token token, DataOutputPlus out) throws IOException
{
byte[] bytes = increment(((BigIntegerToken)token).token.toByteArray());
if (bytes.length < 16)
out.write(EMPTY_BYTES, 0, 16 - bytes.length);
out.write(bytes);
}
@Override
public final void accordSerialize(Token token, ByteBuffer out)
{
byte[] bytes = increment(((BigIntegerToken)token).token.toByteArray());
if (bytes.length < 16)
out.put(EMPTY_BYTES, 0, 16 - bytes.length);
out.put(bytes);
}
@Override
public final Token accordDeserialize(DataInputPlus in, int length) throws IOException
{
byte[] bytes = new byte[16];
in.readFully(bytes);
decrement(bytes);
return new BigIntegerToken(new BigInteger(bytes));
}
@Override
public final Token accordDeserialize(ByteBuffer in, int length)
{
byte[] bytes = new byte[16];
in.get(bytes);
decrement(bytes);
return new BigIntegerToken(new BigInteger(bytes));
}
@Override
public final <V> Token accordDeserialize(V src, ValueAccessor<V> accessor, int offset, int length)
{
byte[] bytes = accessor.toArray(src, offset, 16);
decrement(bytes);
return new BigIntegerToken(new BigInteger(bytes));
}
public static byte[] increment(byte[] bytes)
{
int i = bytes.length;
while (--i >= 0 && ++bytes[i] == 0);
if (i == 0)
{
bytes = new byte[bytes.length + 1];
bytes[0] = (byte)1;
}
return bytes;
}
public static void decrement(byte[] bytes)
{
for (int i = bytes.length - 1 ; i >= 0 && bytes[i]-- == 0 ; --i);
}
@Override
public final int accordSerializedSize(Token token)
{
return 16;
}
@Override
public final int accordFixedLength()
{
return 16;
}
private static BigInteger hashToBigInteger(ByteBuffer data)
{
MessageDigest messageDigest = localMD5Digest.get();

18
src/java/org/apache/cassandra/dht/Token.java
View File

@ -37,7 +37,6 @@ import org.apache.cassandra.tcm.serialization.PartitionerAwareMetadataSerializer
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.apache.cassandra.utils.vint.VIntCoding;
public abstract class Token implements RingPosition<Token>, Serializable
@ -55,16 +54,6 @@ public abstract class Token implements RingPosition<Token>, Serializable
public abstract ByteBuffer toByteArray(Token token);
public abstract Token fromByteArray(ByteBuffer bytes);
public byte[] toOrderedByteArray(Token token, ByteComparable.Version version)
{
return ByteSourceInverse.readBytes(asComparableBytes(token, version));
}
public Token fromOrderedByteArray(byte[] bytes, ByteComparable.Version version)
{
return fromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(bytes)), version);
}
/**
* Produce a byte-comparable representation of the token.
* See {@link Token#asComparableBytes}
@ -109,6 +98,12 @@ public abstract class Token implements RingPosition<Token>, Serializable
return p.getTokenFactory().fromByteArray(ByteBuffer.wrap(bytes));
}
public void skip(DataInputPlus in, IPartitioner p) throws IOException
{
int size = p.isFixedLength() ? p.getMaxTokenSize() : in.readUnsignedVInt32();
in.skipBytesFully(size);
}
public Token fromByteBuffer(ByteBuffer bytes, int position, int length)
{
bytes = bytes.duplicate();
@ -260,6 +255,7 @@ public abstract class Token implements RingPosition<Token>, Serializable
abstract public long getHeapSize();
abstract public Object getTokenValue();
abstract public int tokenHash();
public TokenFactory tokenFactory() { return getPartitioner().getTokenFactory(); }
/**
* This method exists so that callers can access the primitive {@code long} value for this {@link Token}, if

33
src/java/org/apache/cassandra/index/accord/OrderedRouteSerializer.java
View File

@ -18,42 +18,25 @@
package org.apache.cassandra.index.accord;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.ByteBufferUtil;
public class OrderedRouteSerializer
{
private static final AccordRoutingKeyByteSource.FixedLength SERIALIZER = AccordRoutingKeyByteSource.fixedLength(DatabaseDescriptor.getPartitioner());
public static ByteBuffer serializeRoutingKey(AccordRoutingKey key)
public static ByteBuffer serialize(TokenKey key)
{
return ByteBuffer.wrap(SERIALIZER.serialize(key));
return TokenKey.serializer.serialize(key);
}
public static byte[] serializeRoutingKeyNoTable(AccordRoutingKey key)
public static byte[] serializeTokenOnly(TokenKey key)
{
return SERIALIZER.serializeNoTable(key);
return ByteBufferUtil.getArrayUnsafe(TokenKey.serializer.serializeWithoutPrefixOrLength(key));
}
public static byte[] unwrap(AccordRoutingKey key)
public static TokenKey deserialize(ByteBuffer bb)
{
return SERIALIZER.serialize(key);
}
public static AccordRoutingKey deserializeRoutingKey(ByteBuffer bb)
{
try
{
return SERIALIZER.fromComparableBytes(ByteBufferAccessor.instance, bb);
}
catch (IOException e)
{
throw new UnsupportedOperationException(e);
}
return TokenKey.serializer.deserialize(bb);
}
}

10
src/java/org/apache/cassandra/index/accord/RangeMemoryIndex.java
View File

@ -52,8 +52,11 @@ import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.RTree;
import org.apache.cassandra.utils.RangeTree;
import static org.apache.cassandra.index.accord.RouteIndexFormat.deserializeRoute;
public class RangeMemoryIndex
{
@GuardedBy("this")
private final Map<Group, RangeTree<byte[], Range, DecoratedKey>> map = new HashMap<>();
@GuardedBy("this")
@ -107,7 +110,7 @@ public class RangeMemoryIndex
Route<?> route;
try
{
route = AccordKeyspace.deserializeParticipantsRouteOnlyOrNull(value);
route = deserializeRoute(value);
}
catch (IOException e)
{
@ -117,7 +120,6 @@ public class RangeMemoryIndex
return add(key, route);
}
public synchronized long add(DecoratedKey key, Route<?> route)
{
if (route == null || route.domain() != Routable.Domain.Range)
@ -137,8 +139,8 @@ public class RangeMemoryIndex
int storeId = AccordKeyspace.JournalColumns.getStoreId(key);
TableId tableId = ts.table();
Group group = new Group(storeId, tableId);
byte[] start = OrderedRouteSerializer.serializeRoutingKeyNoTable(ts.start());
byte[] end = OrderedRouteSerializer.serializeRoutingKeyNoTable(ts.end());
byte[] start = OrderedRouteSerializer.serializeTokenOnly(ts.start());
byte[] end = OrderedRouteSerializer.serializeTokenOnly(ts.end());
Range range = new Range(start, end);
map.computeIfAbsent(group, ignore -> createRangeTree()).add(range, key);
Metadata metadata = groupMetadata.computeIfAbsent(group, ignore -> new Metadata());

43
src/java/org/apache/cassandra/index/accord/RouteIndexFormat.java
View File

@ -34,23 +34,32 @@ import java.util.zip.Checksum;
import com.google.common.collect.Maps;
import accord.local.StoreParticipants;
import accord.primitives.Route;
import accord.primitives.TxnId;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.index.accord.IndexDescriptor.IndexComponent;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.LocalVersionedSerializer;
import org.apache.cassandra.io.MessageVersionProvider;
import org.apache.cassandra.io.sstable.SSTableFlushObserver;
import org.apache.cassandra.io.util.ChecksumedRandomAccessReader;
import org.apache.cassandra.io.util.ChecksumedSequentialWriter;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.FileHandle;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.serializers.UUIDSerializer;
import org.apache.cassandra.service.accord.AccordJournal;
import org.apache.cassandra.service.accord.AccordJournalTable;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordSerializerVersion;
import org.apache.cassandra.service.accord.JournalKey;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.cassandra.utils.Throwables;
@ -62,6 +71,34 @@ public class RouteIndexFormat
{
public static final Supplier<Checksum> CHECKSUM_SUPPLIER = CRC32C::new;
static final LocalVersionedSerializer<Route<?>> route = localSerializer(KeySerializers.route);
private static <T> LocalVersionedSerializer<T> localSerializer(IVersionedSerializer<T> serializer)
{
return new LocalVersionedSerializer<>(AccordSerializerVersion.CURRENT, AccordSerializerVersion.serializer, serializer);
}
public static ByteBuffer serialize(Route<?> value) throws IOException
{
int size = Math.toIntExact(route.serializedSize(value));
try (DataOutputBuffer buffer = new DataOutputBuffer(size))
{
route.serialize(value, buffer);
return buffer.buffer(true);
}
}
static Route<?> deserializeRoute(ByteBuffer bytes) throws IOException
{
if (bytes == null || ByteBufferAccessor.instance.isEmpty(bytes))
return null;
try (DataInputBuffer in = new DataInputBuffer(bytes, true))
{
MessageVersionProvider versionProvider = route.deserializeVersion(in);
return KeySerializers.route.deserialize(in, versionProvider.messageVersion());
}
}
public interface Writer extends SSTableFlushObserver
{
@ -126,9 +163,13 @@ public class RouteIndexFormat
StoreParticipants participants = builder.participants();
if (participants == null)
return null;
Route<?> route = participants.route();
if (route == null)
return null;
try
{
return AccordKeyspace.LocalVersionedSerializers.serialize(participants);
return serialize(participants.route());
}
catch (IOException e)
{

13
src/java/org/apache/cassandra/index/accord/RouteJournalIndex.java
View File

@ -85,7 +85,7 @@ import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.accord.AccordJournalTable;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.JournalKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@ -451,10 +451,9 @@ public class RouteJournalIndex implements Index, INotificationConsumer
TableId tableId;
byte[] start;
{
AccordRoutingKey route = OrderedRouteSerializer.deserializeRoutingKey(key);
TokenKey route = OrderedRouteSerializer.deserialize(key);
tableId = route.table();
start = OrderedRouteSerializer.serializeRoutingKeyNoTable(route);
start = OrderedRouteSerializer.serializeTokenOnly(route);
}
NavigableSet<ByteBuffer> matches = sstableManager.search(storeId, tableId, start);
matches.addAll(memtableIndexManager.search(storeId, tableId, start));
@ -490,11 +489,11 @@ public class RouteJournalIndex implements Index, INotificationConsumer
byte[] start;
{
AccordRoutingKey route = OrderedRouteSerializer.deserializeRoutingKey(startTableWithToken);
TokenKey route = OrderedRouteSerializer.deserialize(startTableWithToken);
tableId = route.table();
start = OrderedRouteSerializer.serializeRoutingKeyNoTable(route);
start = OrderedRouteSerializer.serializeTokenOnly(route);
}
byte[] end = OrderedRouteSerializer.serializeRoutingKeyNoTable(OrderedRouteSerializer.deserializeRoutingKey(endTableWithToken));
byte[] end = OrderedRouteSerializer.serializeTokenOnly(OrderedRouteSerializer.deserialize(endTableWithToken));
NavigableSet<ByteBuffer> matches = sstableManager.search(storeId, tableId, start, startInclusive, end, endInclusive);
matches.addAll(memtableIndexManager.search(storeId, tableId, start, startInclusive, end, endInclusive));
return matches;

16
src/java/org/apache/cassandra/io/util/BufferedDataOutputStreamPlus.java
View File

@ -178,6 +178,22 @@ public class BufferedDataOutputStreamPlus extends DataOutputStreamPlus
}
}
@Override
public void writeLeastSignificantBytes(long register, int bytes) throws IOException
{
assert buffer != null : "Attempt to use a closed data output";
if (buffer.remaining() < Long.BYTES)
{
super.writeLeastSignificantBytes(register, bytes);
}
else
{
int pos = buffer.position();
buffer.putLong(pos, register << (64 - (bytes * 8)));
buffer.position(pos + bytes);
}
}
@Override
public void writeShort(int v) throws IOException
{

27
src/java/org/apache/cassandra/io/util/DataInputPlus.java
View File

@ -84,6 +84,33 @@ public interface DataInputPlus extends DataInput
return VIntCoding.readUnsignedVInt32(this);
}
default long readLeastSignificantBytes(int bytes) throws IOException
{
switch (bytes)
{
case 0: return 0;
case 1: return readByte();
case 2: return readShort();
case 3:
return ((long)readShort() << 8)
| (long)readByte();
case 4:
return readInt();
case 5:
return ((long)readInt() << 8)
| (long)readByte();
case 6:
return ((long)readInt() << 16)
| (long)readShort();
case 7:
return ((long)readInt() << 24)
| ((long)readShort() << 8)
| (long)readByte();
case 8: return readLong();
default: throw new IllegalArgumentException();
}
}
/**
* Always skips the requested number of bytes, unless EOF is reached
*

47
src/java/org/apache/cassandra/io/util/DataOutputPlus.java
View File

@ -132,6 +132,53 @@ public interface DataOutputPlus extends DataOutput
}
}
/**
* An efficient way to write the type {@code bytes} of a long
*
* @param register - the long value to be written
* @param bytes - the number of bytes the register occupies. Valid values are between 1 and 8 inclusive.
* @throws IOException
*/
default void writeLeastSignificantBytes(long register, int bytes) throws IOException
{
switch (bytes)
{
case 0:
break;
case 1:
writeByte((int)register);
break;
case 2:
writeShort((int)register);
break;
case 3:
writeShort((int)(register >> 8));
writeByte((int)register);
break;
case 4:
writeInt((int)register);
break;
case 5:
writeInt((int)(register >> 8));
writeByte((int)register);
break;
case 6:
writeInt((int)(register >> 16));
writeShort((int)register);
break;
case 7:
writeInt((int)(register >> 24));
writeShort((int)(register >> 8));
writeByte((int)register);
break;
case 8:
writeLong(register);
break;
default:
throw new IllegalArgumentException();
}
}
/**
* Returns the current position of the underlying target like a file-pointer
* or the position withing a buffer. Not every implementation may support this

12
src/java/org/apache/cassandra/io/util/RebufferingInputStream.java
View File

@ -275,6 +275,18 @@ public abstract class RebufferingInputStream extends DataInputStreamPlus impleme
return retval;
}
@Override
public long readLeastSignificantBytes(int bytes) throws IOException
{
if (buffer.remaining() < 8)
return super.readLeastSignificantBytes(bytes);
long retval = buffer.getLong(buffer.position());
retval >>>= 64 - (bytes * 8);
buffer.position(buffer.position() + bytes);
return retval;
}
@Override
public int readUnsignedVInt32() throws IOException
{

7
src/java/org/apache/cassandra/io/util/TrackedDataOutputPlus.java
View File

@ -167,6 +167,13 @@ public class TrackedDataOutputPlus implements DataOutputPlus
position += bytes;
}
@Override
public void writeLeastSignificantBytes(long register, int bytes) throws IOException
{
out.writeLeastSignificantBytes(register, bytes);
position += bytes;
}
@Override
public long position()
{

144
src/java/org/apache/cassandra/schema/TableId.java
View File

@ -24,10 +24,10 @@ import java.nio.ByteBuffer;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.function.LongUnaryOperator;
import javax.annotation.Nullable;
import accord.utils.Invariants;
import org.agrona.collections.Hashing;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -44,7 +44,6 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.UUIDGen;
import org.apache.cassandra.utils.vint.VIntCoding;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
@ -203,20 +202,46 @@ public final class TableId implements Comparable<TableId>
return position - offset;
}
public final long msb()
{
return msb;
}
public final long lsb()
{
return lsb;
}
public final int serializedSize()
{
return 16;
}
private static final int MAGIC_BYTE = (int) ((flipSign(MAGIC) >>> 56) & 0xf0);
public void serializeCompact(DataOutputPlus out) throws IOException
{
if (msb == MAGIC && lsb < Long.MAX_VALUE - 1)
serializeCompact(out, Long.compare(msb, MAGIC), msb, lsb);
}
public void serializeCompactComparable(DataOutputPlus out) throws IOException
{
serializeCompact(out, Long.compare(msb, MAGIC), flipSign(msb), flipSign(lsb));
}
private static void serializeCompact(DataOutputPlus out, int compareMagic, long msb, long lsb) throws IOException
{
// make this an ordered compact serialization at the cost of one byte
// TODO (desired): we could use 6 bits of the byte for encoding the vint header and avoid any extra bytes in most cases
if (compareMagic == 0)
{
out.writeUnsignedVInt(1 + lsb);
int bytes = numberOfBytes(lsb);
out.writeByte(MAGIC_BYTE | bytes);
out.writeLeastSignificantBytes(lsb, bytes);
}
else
{
out.writeByte(0);
out.writeByte(MAGIC_BYTE + (compareMagic > 0 ? 0x10 : -0x10));
out.writeLong(msb);
out.writeLong(lsb);
}
@ -224,25 +249,48 @@ public final class TableId implements Comparable<TableId>
public <V> int serializeCompact(V dst, ValueAccessor<V> accessor, int offset)
{
if (msb == MAGIC && lsb < Long.MAX_VALUE - 1)
return serializeCompact(dst, accessor, offset, Long.compare(msb, MAGIC), msb, lsb);
}
public <V> int serializeCompactComparable(V dst, ValueAccessor<V> accessor, int offset)
{
return serializeCompact(dst, accessor, offset, Long.compare(msb, MAGIC), flipSign(msb), flipSign(lsb));
}
private static <V> int serializeCompact(V dst, ValueAccessor<V> accessor, int offset, int compareMagic, long msb, long lsb)
{
if (compareMagic == 0)
{
return accessor.putUnsignedVInt(dst, offset, 1 + lsb);
int bytes = numberOfBytes(lsb);
accessor.putByte(dst, offset, (byte) (MAGIC_BYTE | bytes));
accessor.putLeastSignificantBytes(dst, offset + 1, lsb, bytes);
return 1 + bytes;
}
else
{
int position = offset;
position += accessor.putByte(dst, position, (byte)0);
position += accessor.putByte(dst, position, (byte) (MAGIC_BYTE + (compareMagic > 0 ? 0x10 : -0x10)));
position += accessor.putLong(dst, position, msb);
position += accessor.putLong(dst, position, lsb);
return position - offset;
}
}
private static int numberOfBytes(long lsb)
{
return (64 + 7 - Long.numberOfLeadingZeros(lsb)) / 8;
}
public final int serializedCompactSize()
{
if (msb == MAGIC && lsb < Long.MAX_VALUE - 1)
return VIntCoding.computeUnsignedVIntSize(1 + lsb);
return 17;
// make this an ordered compact serialization at the cost of one byte
return msb == MAGIC ? 1 + numberOfBytes(lsb) : 17;
}
public final int serializedCompactComparableSize()
{
// make this an ordered compact serialization at the cost of one byte
return msb == MAGIC ? 1 + numberOfBytes(flipSign(lsb)) : 17;
}
public static int staticSerializedSize()
@ -250,11 +298,18 @@ public final class TableId implements Comparable<TableId>
return 16;
}
public static void skip(DataInputPlus in) throws IOException
{
in.skipBytesFully(16);
}
public static void skipCompact(DataInputPlus in) throws IOException
{
long compact = in.readUnsignedVInt();
if (compact == 0)
int b = in.readByte();
if ((b & 0xf0) != MAGIC_BYTE)
in.skipBytesFully(16);
else
in.skipBytesFully(b & 0xf);
}
public static TableId deserialize(DataInput in) throws IOException
@ -262,27 +317,68 @@ public final class TableId implements Comparable<TableId>
return new TableId(in.readLong(), in.readLong());
}
private static TableId deserialize(DataInput in, LongUnaryOperator transform) throws IOException
{
return new TableId(transform.applyAsLong(in.readLong()), transform.applyAsLong(in.readLong()));
}
private static long flipSign(long bits)
{
return bits ^ Long.MIN_VALUE;
}
private static long keepSign(long bits)
{
return bits;
}
public static <V> TableId deserialize(V src, ValueAccessor<V> accessor, int offset)
{
return new TableId(accessor.getLong(src, offset), accessor.getLong(src, offset + TypeSizes.LONG_SIZE));
return deserialize(src, accessor, offset, TableId::keepSign);
}
public static <V> TableId deserializeComparable(V src, ValueAccessor<V> accessor, int offset)
{
return deserialize(src, accessor, offset, TableId::flipSign);
}
private static <V> TableId deserialize(V src, ValueAccessor<V> accessor, int offset, LongUnaryOperator transform)
{
return new TableId(transform.applyAsLong(accessor.getLong(src, offset)), transform.applyAsLong(accessor.getLong(src, offset + TypeSizes.LONG_SIZE)));
}
public static TableId deserializeCompact(DataInputPlus in) throws IOException
{
long compact = in.readUnsignedVInt();
if (compact > 0)
return fromLong(compact - 1);
Invariants.require(compact == 0);
return deserialize(in);
return deserializeCompact(in, TableId::keepSign);
}
public static TableId deserializeCompactComparable(DataInputPlus in) throws IOException
{
return deserializeCompact(in, TableId::flipSign);
}
private static TableId deserializeCompact(DataInputPlus in, LongUnaryOperator transform) throws IOException
{
int b = in.readByte();
if ((b & 0xf0) != MAGIC_BYTE) return deserialize(in, transform);
else return new TableId(MAGIC, transform.applyAsLong(in.readLeastSignificantBytes(b & 0xf)));
}
public static <V> TableId deserializeCompact(V src, ValueAccessor<V> accessor, int offset)
{
long compact = accessor.getUnsignedVInt(src, offset);
if (compact > 0)
return fromLong(compact - 1);
Invariants.require(compact == 0);
return deserialize(src, accessor, offset + 1);
return deserializeCompact(src, accessor, offset, TableId::keepSign);
}
public static <V> TableId deserializeCompactComparable(V src, ValueAccessor<V> accessor, int offset)
{
return deserializeCompact(src, accessor, offset, TableId::flipSign);
}
private static <V> TableId deserializeCompact(V src, ValueAccessor<V> accessor, int offset, LongUnaryOperator transform)
{
int b = accessor.getByte(src, offset++);
if ((b & 0xf0) != MAGIC_BYTE) return deserialize(src, accessor, offset, transform);
else return new TableId(MAGIC, transform.applyAsLong(accessor.getLeastSignificantBytes(src, offset, b & 0x0f)));
}
public TableId intern()

31
src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
View File

@ -59,7 +59,9 @@ import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
@ -140,6 +142,7 @@ public class AccordCommandStore extends CommandStore
private final ExclusiveCaches caches;
private long lastSystemTimestampMicros = Long.MIN_VALUE;
private final CommandsForRanges.Manager commandsForRanges;
private final TableId tableId;
private AccordSafeCommandStore current;
private Thread currentThread;
@ -157,7 +160,7 @@ public class AccordCommandStore extends CommandStore
AccordExecutor executor)
{
super(id, node, agent, dataStore, progressLogFactory, listenerFactory, epochUpdateHolder);
loggingId = String.format("[%s]", id);
this.loggingId = String.format("[%s]", id);
this.journal = journal;
this.rangeSearcher = RangeSearcher.extractRangeSearcher(journal);
this.executor = executor;
@ -179,6 +182,19 @@ public class AccordCommandStore extends CommandStore
maybeLoadBootstrapBeganAt(journal.loadBootstrapBeganAt(id()));
maybeLoadSafeToRead(journal.loadSafeToRead(id()));
maybeLoadRangesForEpoch(journal.loadRangesForEpoch(id()));
CommandStores.RangesForEpoch ranges = this.rangesForEpoch;
if (ranges == null || ranges.all().isEmpty())
{
EpochUpdate update = epochUpdateHolder.get();
if (update != null)
ranges = update.newRangesForEpoch;
Invariants.require(ranges != null, "CommandStore %d created with no ranges", id);
}
tableId = (TableId)ranges.all().stream().map(r -> r.start().prefix()).reduce((a, b) -> {
Invariants.require(a.equals(b), "CommandStore created with multiple distinct TableId (%s and %s)", a, b);
return a;
}).orElseThrow(() -> Invariants.illegalState("CommandStore %d created with no ranges", id));
}
static Factory factory(IntFunction<AccordExecutor> executorFactory)
@ -211,6 +227,11 @@ public class AccordCommandStore extends CommandStore
task.presetup(current.task);
}
public final TableId tableId()
{
return tableId;
}
public AccordExecutor executor()
{
return executor;
@ -278,7 +299,7 @@ public class AccordCommandStore extends CommandStore
CommandsForKey loadCommandsForKey(RoutableKey key)
{
return AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
return CommandsForKeyAccessor.load(id, (TokenKey) key);
}
boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting)
@ -286,14 +307,14 @@ public class AccordCommandStore extends CommandStore
if (!Invariants.isParanoid())
return true;
CommandsForKey reloaded = AccordKeyspace.loadCommandsForKey(id, (TokenKey) key);
CommandsForKey reloaded = CommandsForKeyAccessor.load(id, (TokenKey) key);
return Objects.equals(evicting, reloaded);
}
@Nullable
Runnable saveCommandsForKey(RoutingKey key, CommandsForKey after, Object serialized)
{
return AccordKeyspace.getCommandsForKeyUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros());
return CommandsForKeyAccessor.systemTableUpdater(id, (TokenKey) key, after, serialized, nextSystemTimestampMicros());
}
public long nextSystemTimestampMicros()

6
src/java/org/apache/cassandra/service/accord/AccordCommandStores.java
View File

@ -44,7 +44,7 @@ import org.apache.cassandra.metrics.AccordCacheMetrics;
import org.apache.cassandra.metrics.CacheSizeMetrics;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordExecutor.AccordExecutorFactory;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static org.apache.cassandra.config.AccordSpec.QueueShardModel.THREAD_PER_SHARD;
@ -131,14 +131,14 @@ public class AccordCommandStores extends CommandStores implements CacheSize
if (!super.shouldBootstrap(node, previous, updated, range))
return false;
// we see new ranges when a new keyspace is added, so avoid bootstrap in these cases
return contains(previous, ((AccordRoutingKey) range.start()).table());
return contains(previous, ((TokenKey) range.start()).table());
}
private static boolean contains(Topology previous, TableId searchTable)
{
for (Range range : previous.ranges())
{
TableId table = ((AccordRoutingKey) range.start()).table();
TableId table = ((TokenKey) range.start()).table();
if (table.equals(searchTable))
return true;
}

121
src/java/org/apache/cassandra/service/accord/AccordJournal.java
View File

@ -47,6 +47,7 @@ import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.PersistentField;
import accord.utils.UnhandledEnum;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
@ -81,8 +82,7 @@ import static accord.impl.CommandChange.getFlags;
import static accord.impl.CommandChange.isChanged;
import static accord.impl.CommandChange.isNull;
import static accord.impl.CommandChange.nextSetField;
import static accord.impl.CommandChange.setChanged;
import static accord.impl.CommandChange.setFieldIsNullAndChanged;
import static accord.impl.CommandChange.toIterableNonNullFields;
import static accord.impl.CommandChange.toIterableSetFields;
import static accord.impl.CommandChange.unsetIterable;
import static accord.impl.CommandChange.validateFlags;
@ -620,22 +620,91 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
{
super(txnId, load);
}
public ByteBuffer asByteBuffer(RedundantBefore redundantBefore, int userVersion) throws IOException
public ByteBuffer asByteBuffer(int userVersion) throws IOException
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
serialize(out, redundantBefore, userVersion);
serialize(out, userVersion);
return out.asNewBuffer();
}
}
public void serialize(DataOutputPlus out, RedundantBefore redundantBefore, int userVersion) throws IOException
public void serialize(DataOutputPlus out, int userVersion) throws IOException
{
Invariants.require(mask == 0);
Invariants.require(flags != 0);
int flags = validateFlags(this.flags);
Writer.serialize(construct(redundantBefore), flags, out, userVersion);
serialize(flags, out, userVersion);
}
private void serialize(int flags, DataOutputPlus out, int userVersion) throws IOException
{
Invariants.require(flags != 0);
out.writeInt(flags);
int iterable = toIterableNonNullFields(flags);
for (Field field = nextSetField(iterable) ; field != null; iterable = unsetIterable(field, iterable), field = nextSetField(iterable))
{
switch (field)
{
default: throw new UnhandledEnum(field);
case CLEANUP: throw UnhandledEnum.invalid(field);
case EXECUTE_AT:
Invariants.require(txnId != null);
Invariants.require(executeAt != null);
ExecuteAtSerializer.serialize(txnId, executeAt, out);
break;
case EXECUTES_AT_LEAST:
Invariants.require(executesAtLeast != null);
ExecuteAtSerializer.serialize(executesAtLeast, out);
break;
case MIN_UNIQUE_HLC:
Invariants.require(minUniqueHlc != 0);
out.writeUnsignedVInt(minUniqueHlc);
break;
case SAVE_STATUS:
Invariants.require(saveStatus != null);
out.writeByte(saveStatus.ordinal());
break;
case DURABILITY:
Invariants.require(durability != null);
out.writeByte(durability.ordinal());
break;
case ACCEPTED:
Invariants.require(acceptedOrCommitted != null);
CommandSerializers.ballot.serialize(acceptedOrCommitted, out, userVersion);
break;
case PROMISED:
Invariants.require(promised != null);
CommandSerializers.ballot.serialize(promised, out, userVersion);
break;
case PARTICIPANTS:
Invariants.require(participants != null);
CommandSerializers.participants.serialize(participants, out, userVersion);
break;
case PARTIAL_TXN:
Invariants.require(partialTxn != null);
CommandSerializers.partialTxn.serialize(partialTxn, out, userVersion);
break;
case PARTIAL_DEPS:
Invariants.require(partialDeps != null);
DepsSerializers.partialDeps.serialize(partialDeps, out, userVersion);
break;
case WAITING_ON:
Invariants.require(waitingOn != null);
((WaitingOnSerializer.Provider)waitingOn).reserialize(out, userVersion);
break;
case WRITES:
Invariants.require(writes != null);
CommandSerializers.writes.serialize(writes, out, userVersion);
break;
case RESULT:
Invariants.require(result != null);
ResultSerializers.result.serialize(result, out, userVersion);
break;
}
}
}
public void deserializeNext(DataInputPlus in, int userVersion) throws IOException
@ -646,33 +715,22 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
nextCalled = true;
count++;
int iterable = toIterableSetFields(readFlags);
while (iterable != 0)
// batch-apply any new nulls
setNulls(readFlags);
// iterator sets low 16 bits; low readFlag bits are nulls, so masking with ~readFlags restricts to non-null changed fields
int iterable = toIterableSetFields(readFlags) & ~readFlags;
for (Field field = nextSetField(iterable) ; field != null; field = nextSetField(iterable = unsetIterable(field, iterable)))
{
Field field = nextSetField(iterable);
// Since we are iterating in reverse order, we skip the fields that were
// set by entries writter later (i.e. already read ones).
if (isChanged(field, this.flags) || isNull(field, mask))
{
if (!isNull(field, readFlags))
skip(txnId, field, in, userVersion);
iterable = unsetIterable(field, iterable);
continue;
}
if (isNull(field, readFlags))
{
this.flags = setFieldIsNullAndChanged(field, this.flags);
}
if (isChanged(field, flags))
skip(txnId, field, in, userVersion);
else
{
this.flags = setChanged(field, this.flags);
deserialize(field, in, userVersion);
}
iterable = unsetIterable(field, iterable);
}
// upper 16 bits are changed flags, lower are nulls; by masking upper by ~lower we restrict to only non-null changed fields
this.flags |= readFlags & (~readFlags << 16);
}
private void deserialize(Field field, DataInputPlus in, int userVersion) throws IOException
@ -683,7 +741,7 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
executeAt = ExecuteAtSerializer.deserialize(txnId, in);
break;
case EXECUTES_AT_LEAST:
executeAtLeast = ExecuteAtSerializer.deserialize(in);
executesAtLeast = ExecuteAtSerializer.deserialize(in);
break;
case MIN_UNIQUE_HLC:
minUniqueHlc = in.readUnsignedVInt();
@ -740,9 +798,8 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
in.readUnsignedVInt();
break;
case SAVE_STATUS:
in.readByte();
break;
case DURABILITY:
case CLEANUP:
in.readByte();
break;
case ACCEPTED:
@ -750,7 +807,6 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
CommandSerializers.ballot.skip(in);
break;
case PARTICIPANTS:
// TODO (expected): skip
CommandSerializers.participants.deserialize(in, userVersion);
break;
case PARTIAL_TXN:
@ -767,9 +823,6 @@ public class AccordJournal implements accord.api.Journal, RangeSearcher.Supplier
// TODO (expected): skip
CommandSerializers.writes.deserialize(in, userVersion);
break;
case CLEANUP:
in.readByte();
break;
case RESULT:
// TODO (expected): skip
ResultSerializers.result.deserialize(in, userVersion);

16
src/java/org/apache/cassandra/service/accord/AccordJournalTable.java
View File

@ -72,7 +72,7 @@ import org.apache.cassandra.journal.Journal;
import org.apache.cassandra.journal.KeySupport;
import org.apache.cassandra.journal.RecordConsumer;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -276,28 +276,28 @@ public class AccordJournalTable<K extends JournalKey, V> implements RangeSearche
}
@Override
public Result search(int commandStoreId, AccordRoutingKey key, TxnId minTxnId, Timestamp maxTxnId)
public Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId)
{
CloseableIterator<TxnId> inMemory = index.search(commandStoreId, key, minTxnId, maxTxnId).results();
CloseableIterator<TxnId> table = tableSearch(commandStoreId, key);
return new DefaultResult(minTxnId, maxTxnId, MergeIterator.get(Arrays.asList(inMemory, table)));
}
private CloseableIterator<TxnId> tableSearch(int store, AccordRoutingKey start, AccordRoutingKey end)
private CloseableIterator<TxnId> tableSearch(int store, TokenKey start, TokenKey end)
{
RowFilter rowFilter = RowFilter.create(false);
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GT, OrderedRouteSerializer.serializeRoutingKey(start));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serializeRoutingKey(end));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GT, OrderedRouteSerializer.serialize(start));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(end));
rowFilter.add(AccordJournalTable.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
return process(store, rowFilter);
}
private CloseableIterator<TxnId> tableSearch(int store, AccordRoutingKey key)
private CloseableIterator<TxnId> tableSearch(int store, TokenKey key)
{
RowFilter rowFilter = RowFilter.create(false);
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GTE, OrderedRouteSerializer.serializeRoutingKey(key));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serializeRoutingKey(key));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.GTE, OrderedRouteSerializer.serialize(key));
rowFilter.add(AccordJournalTable.SyntheticColumn.participants.metadata, Operator.LTE, OrderedRouteSerializer.serialize(key));
rowFilter.add(AccordJournalTable.SyntheticColumn.store_id.metadata, Operator.EQ, Int32Type.instance.decompose(store));
return process(store, rowFilter);

1
src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java
View File

@ -90,7 +90,6 @@ public class AccordJournalValueSerializers
from.serialize(out,
// In CompactionIterator, we are dealing with relatively recent records, so we do not pass redundant before here.
// However, we do on load and during Journal SSTable compaction.
RedundantBefore.EMPTY,
userVersion);
}

639
src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
View File

@ -21,12 +21,11 @@ package org.apache.cassandra.service.accord;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -41,15 +40,9 @@ import org.slf4j.LoggerFactory;
import accord.local.Node;
import accord.local.RedundantBefore;
import accord.local.StoreParticipants;
import accord.local.cfk.CommandsForKey;
import accord.local.cfk.Serialize;
import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
import accord.primitives.Status.Durability;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.topology.Topology;
import accord.utils.Invariants;
@ -60,18 +53,22 @@ import org.apache.cassandra.db.Clustering;
import org.apache.cassandra.db.ClusteringComparator;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Columns;
import org.apache.cassandra.db.DataRange;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.ReadExecutionController;
import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.WriteContext;
import org.apache.cassandra.db.filter.ClusteringIndexFilter;
import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.ByteType;
import org.apache.cassandra.db.marshal.BytesType;
@ -79,9 +76,9 @@ import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
@ -93,17 +90,14 @@ import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.ExcludingBounds;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.IncludingExcludingBounds;
import org.apache.cassandra.dht.LocalCompositePrefixPartitioner;
import org.apache.cassandra.dht.LocalPartitioner;
import org.apache.cassandra.dht.LocalPartitioner.LocalToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.index.accord.RouteJournalIndex;
import org.apache.cassandra.index.transactions.UpdateTransaction;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.LocalVersionedSerializer;
import org.apache.cassandra.io.MessageVersionProvider;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.sstable.SSTableReadsListener;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.CompactionParams;
import org.apache.cassandra.schema.IndexMetadata;
@ -112,22 +106,20 @@ import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.schema.Views;
import org.apache.cassandra.serializers.UUIDSerializer;
import org.apache.cassandra.service.accord.AccordConfigurationService.SyncStatus;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.Clock.Global;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.MergeIterator;
import org.apache.cassandra.utils.btree.BTreeSet;
import org.apache.cassandra.utils.concurrent.OpOrder;
@ -138,9 +130,7 @@ import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
import static org.apache.cassandra.db.partitions.PartitionUpdate.singleRowUpdate;
import static org.apache.cassandra.db.rows.BTreeRow.singleCellRow;
import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
import static org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource.currentVersion;
import static org.apache.cassandra.service.accord.serializers.KeySerializers.blobMapToRanges;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
public class AccordKeyspace
{
@ -159,26 +149,6 @@ public class AccordKeyspace
private static final ClusteringIndexFilter FULL_PARTITION = new ClusteringIndexNamesFilter(BTreeSet.of(new ClusteringComparator(), Clustering.EMPTY), false);
private static final ConcurrentMap<TableId, AccordRoutingKeyByteSource.Serializer> TABLE_SERIALIZERS = new ConcurrentHashMap<>();
private static AccordRoutingKeyByteSource.Serializer getRoutingKeySerializer(AccordRoutingKey key)
{
return TABLE_SERIALIZERS.computeIfAbsent(key.table(), ignore -> {
IPartitioner partitioner;
if (key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.TOKEN)
partitioner = key.asTokenKey().token().getPartitioner();
else
partitioner = SchemaHolder.schema.getTablePartitioner(key.table());
return AccordRoutingKeyByteSource.variableLength(partitioner);
});
}
// Schema needs all system keyspace, and this is a system keyspace! So can not touch schema in init
private static class SchemaHolder
{
private static SchemaProvider schema = Objects.requireNonNull(Schema.instance);
}
public static TableMetadata journalMetadata(String tableName, boolean index)
{
TableMetadata.Builder builder = parse(tableName,
@ -206,27 +176,6 @@ public class AccordKeyspace
public static final TableMetadata Journal = journalMetadata(JOURNAL, true);
// TODO: naming is not very clearly distinct from the base serializers
public static class LocalVersionedSerializers
{
static final LocalVersionedSerializer<StoreParticipants> participants = localSerializer(CommandSerializers.participants);
private static <T> LocalVersionedSerializer<T> localSerializer(IVersionedSerializer<T> serializer)
{
return new LocalVersionedSerializer<>(AccordSerializerVersion.CURRENT, AccordSerializerVersion.serializer, serializer);
}
public static ByteBuffer serialize(StoreParticipants value) throws IOException
{
int size = Math.toIntExact(participants.serializedSize(value));
try (DataOutputBuffer buffer = new DataOutputBuffer(size))
{
participants.serialize(value, buffer);
return buffer.buffer(true);
}
}
}
private static ColumnMetadata getColumn(TableMetadata metadata, String name)
{
ColumnMetadata column = metadata.getColumn(new ColumnIdentifier(name, true));
@ -235,19 +184,17 @@ public class AccordKeyspace
return column;
}
private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance);
private static final LocalPartitioner CFKPartitioner = new LocalPartitioner(BytesType.instance);
public static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
public static final CommandsForKeyAccessor CFKAccessor = new CommandsForKeyAccessor(CommandsForKeys);
private static TableMetadata commandsForKeysTable(String tableName)
{
return parse(tableName,
"accord commands per key",
"CREATE TABLE %s ("
+ "store_id int, "
+ "table_id uuid, "
+ "key_token blob, " // can't use "token" as this is restricted word in CQL
+ "key blob, "
+ "data blob, "
+ "PRIMARY KEY((store_id, table_id, key_token))"
+ "PRIMARY KEY(key)"
+ ')'
+ " WITH compression = {'class':'NoopCompressor'};")
.partitioner(CFKPartitioner)
@ -260,11 +207,7 @@ public class AccordKeyspace
{
final TableMetadata table;
final ClusteringComparator keyComparator;
final CompositeType partitionKeyType;
final ColumnFilter allColumns;
final ColumnMetadata store_id;
final ColumnMetadata table_id;
final ColumnMetadata key_token;
final ColumnMetadata data;
final RegularAndStaticColumns columns;
@ -273,42 +216,61 @@ public class AccordKeyspace
{
this.table = table;
this.keyComparator = table.partitionKeyAsClusteringComparator();
this.partitionKeyType = (CompositeType) table.partitionKeyType;
this.allColumns = ColumnFilter.all(table);
this.store_id = getColumn(table, "store_id");
this.table_id = getColumn(table, "table_id");
this.key_token = getColumn(table, "key_token");
this.data = getColumn(table, "data");
this.columns = new RegularAndStaticColumns(Columns.NONE, Columns.from(Lists.newArrayList(data)));
}
public ByteBuffer[] splitPartitionKey(DecoratedKey key)
public static int getCommandStoreId(ByteBuffer partitionKey)
{
return partitionKeyType.split(key.getKey());
return partitionKey.getInt(partitionKey.position());
}
public int getStoreId(ByteBuffer[] partitionKeyComponents)
public static TokenKey getUserTableKey(TableId tableId, DecoratedKey key)
{
return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
return getUserTableKey(tableId, key.getKey());
}
public TableId getTableId(ByteBuffer[] partitionKeyComponents)
public static TokenKey getUserTableKey(TableId tableId, ByteBuffer partitionKey)
{
return TableId.fromUUID(UUIDType.instance.compose(partitionKeyComponents[table_id.position()]));
return TokenKey.serializer.deserializeWithPrefixAndImpliedLength(tableId, partitionKey, ByteBufferAccessor.instance, 4);
}
public TokenKey getKey(DecoratedKey key)
public static TokenKey getUserTableKey(TableId tableId, DecoratedKey key, IPartitioner partitioner)
{
return getKey(splitPartitionKey(key));
return getUserTableKey(tableId, key.getKey(), partitioner);
}
public TokenKey getKey(ByteBuffer[] partitionKeyComponents)
public static TokenKey getUserTableKey(TableId tableId, ByteBuffer partitionKey, IPartitioner partitioner)
{
TableId tableId = TableId.fromUUID(UUIDSerializer.instance.deserialize(partitionKeyComponents[table_id.position()]));
return deserializeTokenKeySeparateTable(tableId, partitionKeyComponents[key_token.position()]);
return TokenKey.serializer.deserializeWithPrefixAndImpliedLength(tableId, partitionKey, ByteBufferAccessor.instance, 4, partitioner);
}
public CommandsForKey getCommandsForKey(TokenKey key, Row row)
public static DecoratedKey makeSystemTableKey(int commandStoreId, TokenKey key)
{
return CFKPartitioner.decorateKey(makeSystemTableKeyBytes(commandStoreId, key));
}
public static LocalToken makeSystemTableToken(int commandStore, TokenKey key)
{
return CFKPartitioner.getToken(makeSystemTableKeyBytes(commandStore, key));
}
public static ByteBuffer makeSystemTableKeyBytes(int commandStore, TokenKey key)
{
ByteBuffer result = ByteBuffer.allocate(4 + TokenKey.serializer.serializedSizeWithoutPrefix(key));
result.putInt(commandStore);
TokenKey.serializer.serializeWithoutPrefixOrLength(key, result);
result.flip();
return result;
}
public static ByteBuffer serializeUserTableKey(TokenKey key)
{
return TokenKey.serializer.serializeWithoutPrefixOrLength(key);
}
public CommandsForKey fromRow(TokenKey key, Row row)
{
Cell<?> cell = row.getCell(data);
if (cell == null)
@ -317,11 +279,37 @@ public class AccordKeyspace
return Serialize.fromBytes(key, cell.buffer());
}
@VisibleForTesting
public ByteBuffer serializeKeyNoTable(AccordRoutingKey key)
public static CommandsForKey load(int commandStoreId, TokenKey key)
{
byte[] bytes = getRoutingKeySerializer(key).serializeNoTable(key);
return ByteBuffer.wrap(bytes);
return unsafeLoad(CFKAccessor, commandStoreId, key);
}
static CommandsForKey unsafeLoad(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
{
long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
SinglePartitionReadCommand command = makeRead(accessor, commandStoreId, key, nowInSeconds);
try (ReadExecutionController controller = command.executionController();
FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
{
if (!partitions.hasNext())
return null;
try (RowIterator partition = partitions.next())
{
Invariants.require(partition.hasNext());
Row row = partition.next();
ByteBuffer data = cellValue(row, accessor.data);
return Serialize.fromBytes(key, data);
}
}
catch (Throwable t)
{
logger.error("Exception loading AccordCommandsForKey " + key, t);
throw t;
}
}
// TODO (expected): garbage-free filtering, reusing encoding
@ -347,21 +335,180 @@ public class AccordKeyspace
return BTreeRow.singleCellRow(Clustering.EMPTY, BufferCell.live(data, cell.timestamp(), buffer));
}
public LocalCompositePrefixPartitioner.AbstractCompositePrefixToken getPrefixToken(int commandStore, AccordRoutingKey key)
public static SinglePartitionReadCommand makeRead(int storeId, TokenKey key, int nowInSeconds)
{
if (key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.TOKEN)
{
ByteBuffer tokenBytes = ByteBuffer.wrap(getRoutingKeySerializer(key).serializeNoTable(key));
return CFKPartitioner.createPrefixToken(commandStore, key.table().asUUID(), tokenBytes);
}
return makeRead(CFKAccessor, storeId, key, nowInSeconds);
}
private static SinglePartitionReadCommand makeRead(CommandsForKeyAccessor accessor, int storeId, TokenKey key, long nowInSeconds)
{
return SinglePartitionReadCommand.create(accessor.table, nowInSeconds,
accessor.allColumns,
RowFilter.none(),
DataLimits.NONE,
makeSystemTableKey(storeId, key),
FULL_PARTITION);
}
private static PartitionUpdate makeUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, Object serialized, long timestampMicros)
{
ByteBuffer bytes;
if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized;
else bytes = Serialize.toBytesWithoutKey(commandsForKey);
return makeUpdate(storeId, key, timestampMicros, bytes);
}
@VisibleForTesting
public static PartitionUpdate makeUpdate(int storeId, TokenKey key, long timestampMicros, ByteBuffer bytes)
{
return singleRowUpdate(CFKAccessor.table,
CommandsForKeyAccessor.makeSystemTableKey(storeId, key),
singleCellRow(Clustering.EMPTY, BufferCell.live(CFKAccessor.data, timestampMicros, bytes)));
}
public static Runnable systemTableUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros)
{
PartitionUpdate upd = makeUpdate(storeId, key, update, serialized, timestampMicros);
return () -> {
ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey;
try (OpOrder.Group group = Keyspace.writeOrder.start())
{
cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
}
};
}
/**
* Calculates token bounds based on key prefixes.
*/
public static void findAllKeysBetween(int commandStore, TableId tableId, IPartitioner partitioner,
TokenKey start, boolean startInclusive,
TokenKey end, boolean endInclusive,
Consumer<TokenKey> consumer)
{
Token startToken = CommandsForKeyAccessor.makeSystemTableToken(commandStore, start);
Token endToken = CommandsForKeyAccessor.makeSystemTableToken(commandStore, end);
if (start.isTableSentinel())
startInclusive = true;
if (end.isTableSentinel())
endInclusive = true;
PartitionPosition startPosition = startInclusive ? startToken.minKeyBound() : startToken.maxKeyBound();
PartitionPosition endPosition = endInclusive ? endToken.maxKeyBound() : endToken.minKeyBound();
AbstractBounds<PartitionPosition> bounds;
if (startInclusive && endInclusive)
bounds = new Bounds<>(startPosition, endPosition);
else if (endInclusive)
bounds = new Range<>(startPosition, endPosition);
else if (startInclusive)
bounds = new IncludingExcludingBounds<>(startPosition, endPosition);
else
bounds = new ExcludingBounds<>(startPosition, endPosition);
ColumnFamilyStore baseCfs = AccordColumnFamilyStores.commandsForKey;
try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
CloseableIterator<DecoratedKey> iter = keyIterator(CommandsForKeys, bounds))
{
return CFKPartitioner.createPrefixToken(commandStore, key.table().asUUID());
// Need the second try to handle callback errors vs read errors.
// Callback will see the read errors, but if the callback fails the outer try will see those errors
while (iter.hasNext())
{
TokenKey pk = CommandsForKeyAccessor.getUserTableKey(tableId, iter.next(), partitioner);
consumer.accept(pk);
}
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
public static final CommandsForKeyAccessor CommandsForKeysAccessor = new CommandsForKeyAccessor(CommandsForKeys);
/**
* Returns a DecoratedKey iterator for the given range. Skips reading data files for sstable formats with a partition index file
*/
private static CloseableIterator<DecoratedKey> keyIterator(Memtable memtable, AbstractBounds<PartitionPosition> range)
{
// TODO (required): why are we replacing the right bound with max bound?
AbstractBounds<PartitionPosition> memtableRange = range.withNewRight(memtable.metadata().partitioner.getMinimumToken().maxKeyBound());
DataRange dataRange = new DataRange(memtableRange, new ClusteringIndexSliceFilter(Slices.ALL, false));
UnfilteredPartitionIterator iter = memtable.partitionIterator(ColumnFilter.NONE, dataRange, SSTableReadsListener.NOOP_LISTENER);
int rangeStartCmpMin = range.isStartInclusive() ? 0 : 1;
int rangeEndCmpMax = range.isEndInclusive() ? 0 : -1;
return new AbstractIterator<>()
{
@Override
protected DecoratedKey computeNext()
{
while (iter.hasNext())
{
DecoratedKey key = iter.next().partitionKey();
if (key.compareTo(range.left) < rangeStartCmpMin)
continue;
if (key.compareTo(range.right) > rangeEndCmpMax)
break;
return key;
}
return endOfData();
}
@Override
public void close()
{
iter.close();
}
};
}
private static CloseableIterator<DecoratedKey> keyIterator(TableMetadata metadata, AbstractBounds<PartitionPosition> range) throws IOException
{
ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata);
ColumnFamilyStore.ViewFragment view = cfs.select(View.selectLive(range));
List<CloseableIterator<?>> closeableIterators = new ArrayList<>();
List<Iterator<DecoratedKey>> iterators = new ArrayList<>();
try
{
for (Memtable memtable : view.memtables)
{
CloseableIterator<DecoratedKey> iter = keyIterator(memtable, range);
iterators.add(iter);
closeableIterators.add(iter);
}
for (SSTableReader sstable : view.sstables)
{
CloseableIterator<DecoratedKey> iter = sstable.keyIterator(range);
iterators.add(iter);
closeableIterators.add(iter);
}
}
catch (Throwable e)
{
for (CloseableIterator<?> iter: closeableIterators)
{
try
{
iter.close();
}
catch (Throwable e2)
{
e.addSuppressed(e2);
}
}
throw e;
}
return MergeIterator.get(iterators, DecoratedKey::compareTo, new MergeIterator.Reducer.Trivial<>());
}
}
public static final TableMetadata Topologies =
parse(TOPOLOGIES,
@ -418,234 +565,6 @@ public class AccordKeyspace
}
}
private static <T> ByteBuffer serialize(T obj, LocalVersionedSerializer<T> serializer) throws IOException
{
int size = (int) serializer.serializedSize(obj);
try (DataOutputBuffer out = new DataOutputBuffer(size))
{
serializer.serialize(obj, out);
ByteBuffer bb = out.buffer();
assert size == bb.limit() : format("Expected to write %d but wrote %d", size, bb.limit());
return bb;
}
}
private static <T> T deserialize(ByteBuffer bytes, LocalVersionedSerializer<T> serializer) throws IOException
{
try (DataInputBuffer in = new DataInputBuffer(bytes, true))
{
return serializer.deserialize(in);
}
}
public static ByteBuffer serializeTimestamp(Timestamp timestamp)
{
return TIMESTAMP_TYPE.pack(bytes(timestamp.msb), bytes(timestamp.lsb), bytes(timestamp.node.id));
}
public interface TimestampFactory<T extends Timestamp>
{
T create(long msb, long lsb, Node.Id node);
}
@Nullable
public static <T extends Timestamp> T deserializeTimestampOrNull(ByteBuffer bytes, TimestampFactory<T> factory)
{
if (bytes == null || ByteBufferAccessor.instance.isEmpty(bytes))
return null;
List<ByteBuffer> split = TIMESTAMP_TYPE.unpack(bytes, ByteBufferAccessor.instance);
return factory.create(split.get(0).getLong(), split.get(1).getLong(), new Node.Id(split.get(2).getInt()));
}
public static <V> Timestamp deserializeTimestampOrNull(Cell<V> cell)
{
if (cell == null)
return null;
ValueAccessor<V> accessor = cell.accessor();
V value = cell.value();
if (accessor.isEmpty(value))
return null;
List<V> split = TIMESTAMP_TYPE.unpack(value, accessor);
return Timestamp.fromBits(accessor.getLong(split.get(0), 0), accessor.getLong(split.get(1), 0), new Node.Id(accessor.getInt(split.get(2), 0)));
}
public static <V, T extends Timestamp> T deserializeTimestampOrNull(V value, ValueAccessor<V> accessor, TimestampFactory<T> factory)
{
if (value == null || accessor.isEmpty(value))
return null;
List<V> split = TIMESTAMP_TYPE.unpack(value, accessor);
return factory.create(accessor.getLong(split.get(0), 0), accessor.getLong(split.get(1), 0), new Node.Id(accessor.getInt(split.get(2), 0)));
}
private static <T extends Timestamp> T deserializeTimestampOrNull(UntypedResultSet.Row row, String name, TimestampFactory<T> factory)
{
return deserializeTimestampOrNull(row.getBlob(name), factory);
}
public static Durability deserializeDurabilityOrNull(Cell cell)
{
return cell == null ? null : CommandSerializers.durability.forOrdinal(cell.accessor().getInt(cell.value(), 0));
}
public static SaveStatus deserializeSaveStatusOrNull(Cell cell)
{
return cell == null ? null : CommandSerializers.saveStatus.forOrdinal(cell.accessor().getInt(cell.value(), 0));
}
/**
* Calculates token bounds based on key prefixes.
*/
public static void findAllKeysBetween(int commandStore,
AccordRoutingKey start, boolean startInclusive,
AccordRoutingKey end, boolean endInclusive,
Consumer<TokenKey> consumer)
{
Token startToken = CommandsForKeysAccessor.getPrefixToken(commandStore, start);
Token endToken = CommandsForKeysAccessor.getPrefixToken(commandStore, end);
if (start instanceof AccordRoutingKey.SentinelKey)
startInclusive = true;
if (end instanceof AccordRoutingKey.SentinelKey)
endInclusive = true;
PartitionPosition startPosition = startInclusive ? startToken.minKeyBound() : startToken.maxKeyBound();
PartitionPosition endPosition = endInclusive ? endToken.maxKeyBound() : endToken.minKeyBound();
AbstractBounds<PartitionPosition> bounds;
if (startInclusive && endInclusive)
bounds = new Bounds<>(startPosition, endPosition);
else if (endInclusive)
bounds = new Range<>(startPosition, endPosition);
else if (startInclusive)
bounds = new IncludingExcludingBounds<>(startPosition, endPosition);
else
bounds = new ExcludingBounds<>(startPosition, endPosition);
ColumnFamilyStore baseCfs = AccordColumnFamilyStores.commandsForKey;
try (OpOrder.Group baseOp = baseCfs.readOrdering.start();
WriteContext writeContext = baseCfs.keyspace.getWriteHandler().createContextForRead();
CloseableIterator<DecoratedKey> iter = LocalCompositePrefixPartitioner.keyIterator(CommandsForKeys, bounds))
{
// Need the second try to handle callback errors vs read errors.
// Callback will see the read errors, but if the callback fails the outer try will see those errors
while (iter.hasNext())
{
TokenKey pk = CommandsForKeysAccessor.getKey(iter.next());
consumer.accept(pk);
}
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
public static TxnId deserializeTxnId(UntypedResultSet.Row row)
{
return deserializeTimestampOrNull(row, "txn_id", TxnId::fromBits);
}
public static Status.Durability deserializeDurability(UntypedResultSet.Row row)
{
// TODO (performance, expected): something less brittle than ordinal, more efficient than values()
return Status.Durability.values()[row.getInt("durability", 0)];
}
public static StoreParticipants deserializeParticipantsOrNull(ByteBuffer bytes) throws IOException
{
return bytes != null && !ByteBufferAccessor.instance.isEmpty(bytes) ? deserialize(bytes, LocalVersionedSerializers.participants) : null;
}
public static Route<?> deserializeParticipantsRouteOnlyOrNull(ByteBuffer bytes) throws IOException
{
if (bytes == null ||ByteBufferAccessor.instance.isEmpty(bytes))
return null;
try (DataInputBuffer in = new DataInputBuffer(bytes, true))
{
MessageVersionProvider versionProvider = LocalVersionedSerializers.participants.deserializeVersion(in);
return CommandSerializers.participants.deserializeRouteOnly(in, versionProvider.messageVersion());
}
}
public static ByteBuffer serializeParticipants(StoreParticipants participants) throws IOException
{
return serialize(participants, LocalVersionedSerializers.participants);
}
public static StoreParticipants deserializeParticipantsOrNull(Cell<?> cell)
{
if (cell == null)
return null;
try
{
return deserializeParticipantsOrNull(cell.buffer());
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
public static TokenKey deserializeTokenKeySeparateTable(TableId tableId, ByteBuffer tokenBytes)
{
return (TokenKey) AccordRoutingKeyByteSource.Serializer.fromComparableBytes(ByteBufferAccessor.instance, tokenBytes, tableId, currentVersion, null);
}
private static DecoratedKey makeKeySeparateTable(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
{
ByteBuffer pk = accessor.keyComparator.make(commandStoreId,
UUIDSerializer.instance.serialize(key.table().asUUID()),
serializeRoutingKeyNoTable(key)).serializeAsPartitionKey();
return accessor.table.partitioner.decorateKey(pk);
}
public static ByteBuffer serializeRoutingKeyNoTable(AccordRoutingKey key)
{
return CommandsForKeysAccessor.serializeKeyNoTable(key);
}
public static AccordRoutingKeyByteSource.Serializer serializer(TableId tableId)
{
return TABLE_SERIALIZERS.computeIfAbsent(tableId, id -> AccordRoutingKeyByteSource.variableLength(partitioner(tableId)));
}
public static IPartitioner partitioner(TableId tableId)
{
return SchemaHolder.schema.getTablePartitioner(tableId);
}
private static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, CommandsForKey commandsForKey, Object serialized, long timestampMicros)
{
ByteBuffer bytes;
if (serialized instanceof ByteBuffer) bytes = (ByteBuffer) serialized;
else bytes = Serialize.toBytesWithoutKey(commandsForKey);
return getCommandsForKeyPartitionUpdate(storeId, key, timestampMicros, bytes);
}
@VisibleForTesting
public static PartitionUpdate getCommandsForKeyPartitionUpdate(int storeId, TokenKey key, long timestampMicros, ByteBuffer bytes)
{
return singleRowUpdate(CommandsForKeysAccessor.table,
makeKeySeparateTable(CommandsForKeysAccessor, storeId, key),
singleCellRow(Clustering.EMPTY, BufferCell.live(CommandsForKeysAccessor.data, timestampMicros, bytes)));
}
public static Runnable getCommandsForKeyUpdater(int storeId, TokenKey key, CommandsForKey update, Object serialized, long timestampMicros)
{
PartitionUpdate upd = getCommandsForKeyPartitionUpdate(storeId, key, update, serialized, timestampMicros);
return () -> {
ColumnFamilyStore cfs = AccordColumnFamilyStores.commandsForKey;
try (OpOrder.Group group = Keyspace.writeOrder.start())
{
cfs.getCurrentMemtable().put(upd, UpdateTransaction.NO_OP, group, true);
}
};
}
private static <T> ByteBuffer cellValue(Cell<T> cell)
{
return cell.accessor().toBuffer(cell.value());
@ -658,54 +577,6 @@ public class AccordKeyspace
return (cell != null && !cell.isTombstone()) ? cellValue(cell) : null;
}
private static SinglePartitionReadCommand getCommandsForKeyRead(CommandsForKeyAccessor accessor, int storeId, TokenKey key, long nowInSeconds)
{
return SinglePartitionReadCommand.create(accessor.table, nowInSeconds,
accessor.allColumns,
RowFilter.none(),
DataLimits.NONE,
makeKeySeparateTable(accessor, storeId, key),
FULL_PARTITION);
}
public static SinglePartitionReadCommand getCommandsForKeyRead(int storeId, TokenKey key, int nowInSeconds)
{
return getCommandsForKeyRead(CommandsForKeysAccessor, storeId, key, nowInSeconds);
}
static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, int commandStoreId, TokenKey key)
{
long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Global.currentTimeMillis());
int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStoreId, key, nowInSeconds);
try (ReadExecutionController controller = command.executionController();
FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
{
if (!partitions.hasNext())
return null;
try (RowIterator partition = partitions.next())
{
Invariants.require(partition.hasNext());
Row row = partition.next();
ByteBuffer data = cellValue(row, accessor.data);
return Serialize.fromBytes(key, data);
}
}
catch (Throwable t)
{
logger.error("Exception loading AccordCommandsForKey " + key, t);
throw t;
}
}
public static CommandsForKey loadCommandsForKey(int commandStoreId, TokenKey key)
{
return unsafeLoadCommandsForKey(CommandsForKeysAccessor, commandStoreId, key);
}
public static class EpochDiskState
{
public static final EpochDiskState EMPTY = new EpochDiskState(0, 0);
@ -1012,19 +883,11 @@ public class AccordKeyspace
}
}
@VisibleForTesting
public static void unsafeSetSchema(SchemaProvider provider)
{
SchemaHolder.schema = provider;
}
@VisibleForTesting
public static void unsafeClear()
{
for (ColumnFamilyStore store : Keyspace.open(SchemaConstants.ACCORD_KEYSPACE_NAME).getColumnFamilyStores())
store.truncateBlockingWithoutSnapshot();
TABLE_SERIALIZERS.clear();
SchemaHolder.schema = Schema.instance;
}
public static class AccordColumnFamilyStores

8
src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
View File

@ -60,9 +60,9 @@ import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.primitives.Writes;
import accord.utils.ImmutableBitSet;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.service.accord.txn.AccordUpdate;
@ -86,11 +86,11 @@ public class AccordObjectSizes
public static long key(RoutingKey key)
{
return ((AccordRoutingKey) key).estimatedSizeOnHeap();
return ((TokenKey) key).estimatedSizeOnHeap();
}
private static final TableId EMPTY_ID = TableId.fromUUID(new UUID(0, 0));
private static final long EMPTY_RANGE_SIZE = measure(TokenRange.fullRange(EMPTY_ID));
private static final long EMPTY_RANGE_SIZE = measure(TokenRange.fullRange(EMPTY_ID, Murmur3Partitioner.instance));
public static long range(Range range)
{
return EMPTY_RANGE_SIZE + key(range.start()) + key(range.end());

28
src/java/org/apache/cassandra/service/accord/AccordService.java
View File

@ -75,13 +75,10 @@ import accord.impl.progresslog.DefaultProgressLogs;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.DurableBefore;
import accord.local.KeyHistory;
import accord.local.Node;
import accord.local.Node.Id;
import accord.local.PreLoadContext;
import accord.local.RedundantBefore;
import accord.local.SafeCommand;
import accord.local.ShardDistributor.EvenSplit;
import accord.local.cfk.CommandsForKey;
@ -113,7 +110,6 @@ import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
@ -142,8 +138,8 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordSyncPropagator.Notification;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.KeyspaceSplitter;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey.KeyspaceSplitter;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.AccordScheduler;
import org.apache.cassandra.service.accord.api.AccordTimeService;
import org.apache.cassandra.service.accord.api.AccordTopologySorter;
@ -1329,23 +1325,25 @@ public class AccordService implements IAccordService, Shutdownable
}
@Override
public CompactionInfo getCompactionInfo()
public AccordCompactionInfos getCompactionInfo()
{
Int2ObjectHashMap<RedundantBefore> redundantBefores = new Int2ObjectHashMap<>();
Int2ObjectHashMap<DurableBefore> durableBefores = new Int2ObjectHashMap<>();
Int2ObjectHashMap<RangesForEpoch> ranges = new Int2ObjectHashMap<>();
AccordCompactionInfos compactionInfos = new AccordCompactionInfos(node.durableBefore());
if (node.commandStores().all().length > 0)
{
AsyncChains.getBlockingAndRethrow(node.commandStores().forEach(safeStore -> {
synchronized (redundantBefores)
synchronized (compactionInfos)
{
redundantBefores.put(safeStore.commandStore().id(), safeStore.redundantBefore());
ranges.put(safeStore.commandStore().id(), safeStore.ranges());
durableBefores.put(safeStore.commandStore().id(), safeStore.durableBefore());
int id = safeStore.commandStore().id();
compactionInfos.put(id, new AccordCompactionInfo(
id,
safeStore.redundantBefore(),
safeStore.ranges(),
((AccordCommandStore)safeStore.commandStore()).tableId()
));
}
}));
}
return new CompactionInfo(redundantBefores, ranges, durableBefores);
return compactionInfos;
}
@Override

16
src/java/org/apache/cassandra/service/accord/AccordTask.java
View File

@ -59,7 +59,8 @@ import org.apache.cassandra.service.accord.AccordCacheEntry.Status;
import org.apache.cassandra.service.accord.AccordCommandStore.Caches;
import org.apache.cassandra.service.accord.AccordExecutor.Task;
import org.apache.cassandra.service.accord.AccordExecutor.TaskQueue;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.concurrent.Condition;
@ -67,6 +68,7 @@ import static accord.primitives.Routable.Domain.Key;
import static accord.primitives.Txn.Kind.EphemeralRead;
import static accord.utils.Invariants.illegalState;
import static org.apache.cassandra.config.CassandraRelevantProperties.DTEST_ACCORD_JOURNAL_SANITY_CHECK_ENABLED;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
import static org.apache.cassandra.service.accord.AccordTask.State.CANCELLED;
import static org.apache.cassandra.service.accord.AccordTask.State.FAILED;
import static org.apache.cassandra.service.accord.AccordTask.State.FAILING;
@ -937,7 +939,7 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
}
}
final Set<AccordRoutingKey.TokenKey> intersectingKeys = new ObjectHashSet<>();
final Set<TokenKey> intersectingKeys = new ObjectHashSet<>();
final KeyWatcher keyWatcher = new KeyWatcher();
final Ranges ranges = ((AbstractRanges) preLoadContext.keys()).toRanges();
final AccordCache.Type<RoutingKey, CommandsForKey, AccordSafeCommandsForKey>.Instance commandsForKeyCache;
@ -953,10 +955,10 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
{
for (Range range : ranges)
{
AccordKeyspace.findAllKeysBetween(commandStore.id(),
(AccordRoutingKey) range.start(), range.startInclusive(),
(AccordRoutingKey) range.end(), range.endInclusive(),
intersectingKeys::add);
CommandsForKeyAccessor.findAllKeysBetween(commandStore.id(), commandStore.tableId(), getPartitioner(),
(TokenKey) range.start(), range.startInclusive(),
(TokenKey) range.end(), range.endInclusive(),
intersectingKeys::add);
}
super.runInternal();
}
@ -995,7 +997,7 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
for (RoutingKey key : caches.commandsForKeys().keySet())
{
if (ranges.contains(key))
intersectingKeys.add((AccordRoutingKey.TokenKey) key);
intersectingKeys.add((TokenKey) key);
}
caches.commandsForKeys().register(keyWatcher);
super.startInternal(caches);

16
src/java/org/apache/cassandra/service/accord/AccordTopology.java
View File

@ -43,6 +43,7 @@ import accord.utils.Invariants;
import accord.utils.SortedArrays.SortedArrayList;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
@ -53,8 +54,7 @@ import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.fastpath.FastPathStrategy;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
@ -221,24 +221,24 @@ public class AccordTopology
static TokenRange minRange(TableId table, Token token)
{
return TokenRange.create(SentinelKey.min(table), new TokenKey(table, token));
return TokenRange.create(TokenKey.min(table, token.getPartitioner()), new TokenKey(table, token));
}
static TokenRange maxRange(TableId table, Token token)
{
return TokenRange.create(new TokenKey(table, token), SentinelKey.max(table));
return TokenRange.create(new TokenKey(table, token), TokenKey.max(table, token.getPartitioner()));
}
static TokenRange fullRange(TableId table)
static TokenRange fullRange(TableId table, IPartitioner partitioner)
{
return TokenRange.create(SentinelKey.min(table), SentinelKey.max(table));
return TokenRange.create(TokenKey.min(table, partitioner), TokenKey.max(table, partitioner));
}
static TokenRange range(TableId table, Range<Token> range)
{
Token minToken = range.left.minValue();
return TokenRange.create(range.left.equals(minToken) ? SentinelKey.min(table) : new TokenKey(table, range.left),
range.right.equals(minToken) ? SentinelKey.max(table) : new TokenKey(table, range.right));
return TokenRange.create(range.left.equals(minToken) ? TokenKey.min(table, minToken.getPartitioner()) : new TokenKey(table, range.left),
range.right.equals(minToken) ? TokenKey.max(table, minToken.getPartitioner()) : new TokenKey(table, range.right));
}
public static accord.primitives.Ranges toAccordRanges(TableId tableId, Collection<Range<Token>> ranges)

2
src/java/org/apache/cassandra/service/accord/CommandStoreTxnBlockedGraph.java
View File

@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableSet;
import accord.primitives.SaveStatus;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
public class CommandStoreTxnBlockedGraph
{

4
src/java/org/apache/cassandra/service/accord/CommandsForRanges.java
View File

@ -40,7 +40,7 @@ import accord.primitives.Unseekable;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
import org.agrona.collections.ObjectHashSet;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import static accord.local.CommandSummaries.SummaryStatus.NOT_DIRECTLY_WITNESSED;
@ -141,7 +141,7 @@ public class CommandsForRanges extends TreeMap<Timestamp, Summary> implements Co
break;
case Key:
for (Unseekable key : searchKeysOrRanges)
manager.searcher.search(manager.commandStore.id(), (AccordRoutingKey) key, minTxnId, maxTxnId).consume(forEach);
manager.searcher.search(manager.commandStore.id(), (TokenKey) key, minTxnId, maxTxnId).consume(forEach);
}
if (!manager.transitive.isEmpty())

46
src/java/org/apache/cassandra/service/accord/IAccordService.java
View File

@ -23,7 +23,6 @@ import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@ -45,6 +44,7 @@ import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.topology.Topology;
import accord.topology.TopologyManager;
import accord.utils.Invariants;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
@ -152,26 +152,42 @@ public interface IAccordService
void receive(Message<List<AccordSyncPropagator.Notification>> message);
class CompactionInfo
class AccordCompactionInfo
{
static final Supplier<CompactionInfo> NO_OP = () -> new CompactionInfo(new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>());
public final int commandStoreId;
public final RedundantBefore redundantBefore;
public final RangesForEpoch ranges;
public final TableId tableId;
public final Int2ObjectHashMap<RedundantBefore> redundantBefores;
public final Int2ObjectHashMap<DurableBefore> durableBefores;
public final Int2ObjectHashMap<RangesForEpoch> ranges;
public CompactionInfo(Int2ObjectHashMap<RedundantBefore> redundantBefores, Int2ObjectHashMap<RangesForEpoch> ranges, Int2ObjectHashMap<DurableBefore> durableBefores)
public AccordCompactionInfo(int commandStoreId, RedundantBefore redundantBefore, RangesForEpoch ranges, TableId tableId)
{
this.redundantBefores = redundantBefores;
this.ranges = ranges;
this.durableBefores = durableBefores;
this.commandStoreId = commandStoreId;
this.redundantBefore = Invariants.nonNull(redundantBefore);
this.ranges = Invariants.nonNull(ranges);
this.tableId = Invariants.nonNull(tableId);
}
}
class AccordCompactionInfos extends Int2ObjectHashMap<AccordCompactionInfo>
{
public final DurableBefore durableBefore;
public AccordCompactionInfos(DurableBefore durableBefore)
{
this.durableBefore = durableBefore;
}
public AccordCompactionInfos(DurableBefore durableBefore, AccordCompactionInfos copy)
{
super(copy);
this.durableBefore = durableBefore;
}
}
/**
* Fetch the redundnant befores for every command store
*/
CompactionInfo getCompactionInfo();
AccordCompactionInfos getCompactionInfo();
Agent agent();
@ -300,9 +316,9 @@ public interface IAccordService
public void receive(Message<List<AccordSyncPropagator.Notification>> message) {}
@Override
public CompactionInfo getCompactionInfo()
public AccordCompactionInfos getCompactionInfo()
{
return new CompactionInfo(new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>(), new Int2ObjectHashMap<>());
return new AccordCompactionInfos(DurableBefore.EMPTY);
}
@Override
@ -487,7 +503,7 @@ public interface IAccordService
}
@Override
public CompactionInfo getCompactionInfo()
public AccordCompactionInfos getCompactionInfo()
{
return delegate.getCompactionInfo();
}

6
src/java/org/apache/cassandra/service/accord/RangeSearcher.java
View File

@ -22,13 +22,13 @@ import java.util.function.Consumer;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.CloseableIterator;
public interface RangeSearcher
{
Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId);
Result search(int commandStoreId, AccordRoutingKey key, TxnId minTxnId, Timestamp maxTxnId);
Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId);
static RangeSearcher extractRangeSearcher(Object o)
{
@ -120,7 +120,7 @@ public interface RangeSearcher
}
@Override
public Result search(int commandStoreId, AccordRoutingKey key, TxnId minTxnId, Timestamp maxTxnId)
public Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId)
{
return NoopResult.instance;
}

14
src/java/org/apache/cassandra/service/accord/RouteInMemoryIndex.java
View File

@ -37,7 +37,7 @@ import org.apache.cassandra.index.accord.OrderedRouteSerializer;
import org.apache.cassandra.index.accord.RouteJournalIndex;
import org.apache.cassandra.journal.StaticSegment;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.cassandra.utils.CloseableIterator;
import org.apache.cassandra.utils.FastByteOperations;
@ -70,8 +70,8 @@ public class RouteInMemoryIndex<V> implements RangeSearcher
public RangeSearcher.Result search(int commandStoreId, TokenRange range, TxnId minTxnId, Timestamp maxTxnId)
{
NavigableSet<TxnId> result = search(commandStoreId, range.table(),
OrderedRouteSerializer.serializeRoutingKeyNoTable(range.start()),
OrderedRouteSerializer.serializeRoutingKeyNoTable(range.end()));
OrderedRouteSerializer.serializeTokenOnly(range.start()),
OrderedRouteSerializer.serializeTokenOnly(range.end()));
return new DefaultResult(minTxnId, maxTxnId, CloseableIterator.wrap(result.iterator()));
}
@ -83,9 +83,9 @@ public class RouteInMemoryIndex<V> implements RangeSearcher
}
@Override
public RangeSearcher.Result search(int commandStoreId, AccordRoutingKey key, TxnId minTxnId, Timestamp maxTxnId)
public RangeSearcher.Result search(int commandStoreId, TokenKey key, TxnId minTxnId, Timestamp maxTxnId)
{
NavigableSet<TxnId> result = search(commandStoreId, key.table(), OrderedRouteSerializer.serializeRoutingKeyNoTable(key));
NavigableSet<TxnId> result = search(commandStoreId, key.table(), OrderedRouteSerializer.serializeTokenOnly(key));
return new DefaultResult(minTxnId, maxTxnId, CloseableIterator.wrap(result.iterator()));
}
@ -176,8 +176,8 @@ public class RouteInMemoryIndex<V> implements RangeSearcher
public void add(TxnId id, TokenRange ts)
{
byte[] start = OrderedRouteSerializer.serializeRoutingKeyNoTable(ts.start());
byte[] end = OrderedRouteSerializer.serializeRoutingKeyNoTable(ts.end());
byte[] start = OrderedRouteSerializer.serializeTokenOnly(ts.start());
byte[] end = OrderedRouteSerializer.serializeTokenOnly(ts.end());
IndexRange range = new IndexRange(start, end);
index.add(range, id);

52
src/java/org/apache/cassandra/service/accord/TokenRange.java
View File

@ -27,26 +27,26 @@ import accord.primitives.Range;
import accord.utils.Invariants;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.ObjectSizes;
public class TokenRange extends Range.EndInclusive
{
public static final long EMPTY_SIZE = ObjectSizes.measure(new TokenRange(SentinelKey.min(TableId.fromLong(0)), SentinelKey.max(TableId.fromLong(0))));
public static final long EMPTY_SIZE = ObjectSizes.measure(new TokenRange(TokenKey.min(TableId.fromLong(0), Murmur3Partitioner.instance), TokenKey.max(TableId.fromLong(0), Murmur3Partitioner.instance)));
// Don't make this public use create or createUnsafe
private TokenRange(AccordRoutingKey start, AccordRoutingKey end)
private TokenRange(TokenKey start, TokenKey end)
{
super(start, end);
}
public static TokenRange create(AccordRoutingKey start, AccordRoutingKey end)
public static TokenRange create(TokenKey start, TokenKey end)
{
Invariants.requireArgument(start.table().equals(end.table()),
"Token ranges cannot cover more than one keyspace start:%s, end:%s",
@ -54,7 +54,7 @@ public class TokenRange extends Range.EndInclusive
return new TokenRange(start, end);
}
public static TokenRange createUnsafe(AccordRoutingKey start, AccordRoutingKey end)
public static TokenRange createUnsafe(TokenKey start, TokenKey end)
{
return new TokenRange(start, end);
}
@ -65,15 +65,15 @@ public class TokenRange extends Range.EndInclusive
}
@Override
public AccordRoutingKey start()
public TokenKey start()
{
return (AccordRoutingKey) super.start();
return (TokenKey) super.start();
}
@Override
public AccordRoutingKey end()
public TokenKey end()
{
return (AccordRoutingKey) super.end();
return (TokenKey) super.end();
}
public long estimatedSizeOnHeap()
@ -83,7 +83,7 @@ public class TokenRange extends Range.EndInclusive
public boolean isFullRange()
{
return start().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL && end().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL;
return start().isMin() && end().isMax();
}
@VisibleForTesting
@ -92,15 +92,15 @@ public class TokenRange extends Range.EndInclusive
return new TokenRange(start().withTable(table), end().withTable(table));
}
public static TokenRange fullRange(TableId table)
public static TokenRange fullRange(TableId table, IPartitioner partitioner)
{
return new TokenRange(SentinelKey.min(table), SentinelKey.max(table));
return new TokenRange(TokenKey.min(table, partitioner), TokenKey.max(table, partitioner));
}
@Override
public TokenRange newRange(RoutingKey start, RoutingKey end)
{
return new TokenRange((AccordRoutingKey) start, (AccordRoutingKey) end);
return new TokenRange((TokenKey) start, (TokenKey) end);
}
/*
@ -111,10 +111,10 @@ public class TokenRange extends Range.EndInclusive
public org.apache.cassandra.dht.Range<Token> toKeyspaceRange()
{
IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
AccordRoutingKey start = start();
AccordRoutingKey end = end();
Token left = start instanceof SentinelKey ? partitioner.getMinimumToken() : start.token();
Token right = end instanceof SentinelKey ? partitioner.getMinimumToken() : end.token();
TokenKey start = start();
TokenKey end = end();
Token left = start.isMin() ? partitioner.getMinimumToken() : start.token();
Token right = end.isMax() ? partitioner.getMinimumToken() : end.token();
return new org.apache.cassandra.dht.Range<>(left, right);
}
@ -125,28 +125,28 @@ public class TokenRange extends Range.EndInclusive
@Override
public void serialize(TokenRange range, DataOutputPlus out, int version) throws IOException
{
AccordRoutingKey.serializer.serialize(range.start(), out, version);
AccordRoutingKey.serializer.serialize(range.end(), out, version);
TokenKey.serializer.serialize(range.start(), out, version);
TokenKey.serializer.serialize(range.end(), out, version);
}
public void skip(DataInputPlus in, int version) throws IOException
{
AccordRoutingKey.serializer.skip(in, version);
AccordRoutingKey.serializer.skip(in, version);
TokenKey.serializer.skip(in, version);
TokenKey.serializer.skip(in, version);
}
@Override
public TokenRange deserialize(DataInputPlus in, int version) throws IOException
{
return TokenRange.create(AccordRoutingKey.serializer.deserialize(in, version),
AccordRoutingKey.serializer.deserialize(in, version));
return TokenRange.create(TokenKey.serializer.deserialize(in, version),
TokenKey.serializer.deserialize(in, version));
}
@Override
public long serializedSize(TokenRange range, int version)
{
return AccordRoutingKey.serializer.serializedSize(range.start(), version)
+ AccordRoutingKey.serializer.serializedSize(range.end(), version);
return TokenKey.serializer.serializedSize(range.start(), version)
+ TokenKey.serializer.serializedSize(range.end(), version);
}
};
}

81
src/java/org/apache/cassandra/service/accord/api/AccordRoutableKey.java
View File

@ -20,22 +20,41 @@ package org.apache.cassandra.service.accord.api;
import java.io.IOException;
import javax.annotation.Nonnull;
import accord.primitives.RoutableKey;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
public abstract class AccordRoutableKey implements RoutableKey
{
public interface AccordKeySerializer<T> extends IVersionedSerializer<T>
public interface AccordKeySerializer<K> extends IVersionedSerializer<K>
{
void skip(DataInputPlus in, int version) throws IOException;
}
public interface AccordSearchableKeySerializer<K> extends AccordKeySerializer<K>
{
// -1 means dynamic
int fixedKeyLengthForPrefix(Object prefix);
int serializedSizeOfPrefix(Object prefix);
int serializedSizeWithoutPrefix(K key);
void serializePrefix(Object prefix, DataOutputPlus out, int version) throws IOException;
void serializeWithoutPrefixOrLength(K key, DataOutputPlus out, int version) throws IOException;
Object deserializePrefix(DataInputPlus in, int version) throws IOException;
K deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException;
}
static final byte MAX_TABLE_SENTINEL = 0x48;
static final byte NORMAL_SENTINEL = 0x28;
static final byte BEFORE_TOKEN_SENTINEL = 0x24;
static final byte MIN_TABLE_SENTINEL = 0x18;
static final int PREFIX_MASK = 0xF0;
static final int SUFFIX_MASK = 0x0F;
final TableId table; // TODO (desired): use an id (TrM)
protected AccordRoutableKey(TableId table)
@ -49,6 +68,7 @@ public abstract class AccordRoutableKey implements RoutableKey
}
public abstract Token token();
abstract byte sentinel();
@Override
public Object prefix()
@ -74,37 +94,40 @@ public abstract class AccordRoutableKey implements RoutableKey
return compareTo((AccordRoutableKey) that);
}
@Override
public int compareAsRoutingKey(@Nonnull RoutableKey that)
{
return compareAsRoutingKey((AccordRoutableKey) that);
}
public final int compareAsRoutingKey(@Nonnull AccordRoutableKey that)
{
int c = this.table.compareTo(that.table);
if (c != 0) return c;
int thisSentinel = this.sentinel(), thatSentinel = that.sentinel();
c = (thisSentinel & PREFIX_MASK) - (thatSentinel & PREFIX_MASK);
if (c == 0) c = this.token().compareTo(that.token());
if (c == 0) c = (thisSentinel & SUFFIX_MASK) - (thatSentinel & SUFFIX_MASK);
return c;
}
public final int compareTo(AccordRoutableKey that)
{
int cmp = this.table().compareTo(that.table());
if (cmp != 0)
return cmp;
int c = compareAsRoutingKey(that);
if (c != 0)
return c;
Class<?> thisClass = this.getClass();
Class<?> thatClass = that.getClass();
if (thisClass == SentinelKey.class || thatClass == SentinelKey.class)
boolean thisIsRoutingKey = this.getClass() == TokenKey.class;
boolean thatIsRoutingKey = that.getClass() == TokenKey.class;
if (thisIsRoutingKey | thatIsRoutingKey)
{
int leftInt = thisClass == SentinelKey.class ? ((SentinelKey) this).asInt() : 0;
int rightInt = thatClass == SentinelKey.class ? ((SentinelKey) that).asInt() : 0;
return Integer.compare(leftInt, rightInt);
if (thisIsRoutingKey & thatIsRoutingKey)
return 0;
return thisIsRoutingKey ? 1 : -1;
}
cmp = this.token().compareTo(that.token());
if (cmp != 0)
{
return cmp;
}
// MinTokenKey is < all TokenKey, PartitionKey with the same token
if (thisClass == MinTokenKey.class)
return thatClass == MinTokenKey.class ? 0 : -1;
if (thatClass == MinTokenKey.class)
return 1;
if (thisClass == TokenKey.class)
return thatClass == TokenKey.class ? 0 : 1;
return thatClass == TokenKey.class ? -1 : 0;
return ((PartitionKey)this).key.compareBytesOnly(((PartitionKey)that).key);
}
@Override

582
src/java/org/apache/cassandra/service/accord/api/AccordRoutingKey.java
View File

@ -1,582 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.api;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.BiFunction;
import accord.api.Key;
import accord.api.RoutingKey;
import accord.local.ShardDistributor;
import accord.primitives.Range;
import accord.primitives.RangeFactory;
import accord.primitives.Ranges;
import accord.utils.Invariants;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.utils.ObjectSizes;
import static com.google.common.base.Preconditions.checkState;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
public abstract class AccordRoutingKey extends AccordRoutableKey implements RoutingKey, RangeFactory
{
public enum RoutingKeyKind
{
TOKEN, SENTINEL, MIN_TOKEN
}
protected AccordRoutingKey(TableId table)
{
super(table);
}
public abstract RoutingKeyKind kindOfRoutingKey();
public abstract long estimatedSizeOnHeap();
public abstract AccordRoutingKey withTable(TableId table);
@Override
public RangeFactory rangeFactory()
{
return this;
}
@Override
public Range newRange(RoutingKey start, RoutingKey end)
{
return TokenRange.create((AccordRoutingKey) start, (AccordRoutingKey) end);
}
@Override
public Range newAntiRange(RoutingKey start, RoutingKey end)
{
return TokenRange.createUnsafe((AccordRoutingKey) start, (AccordRoutingKey) end);
}
public SentinelKey asSentinelKey()
{
return (SentinelKey) this;
}
public TokenKey asTokenKey()
{
return (TokenKey) this;
}
@Override
public RoutingKey toUnseekable()
{
return this;
}
@Override
public RoutingKey asRoutingKey()
{
return asTokenKey();
}
public static AccordRoutingKey of(Key key)
{
return (AccordRoutingKey) key;
}
// final in part because we refer to its class directly in AccordRoutableKey.compareTo
public static final class SentinelKey extends AccordRoutingKey
{
private static final long EMPTY_SIZE = ObjectSizes.measure(new SentinelKey(null, true, false));
// Is this a min sentinel or a max sentinel
public final boolean isMinSentinel;
// Is this a minumum of a max or min sentinel
// Allows conversion of a token key to a range
public final boolean isMinMinSentinel;
public SentinelKey(TableId table, boolean isMinSentinel, boolean isMinMinSentinel)
{
super(table);
this.isMinSentinel = isMinSentinel;
this.isMinMinSentinel = isMinMinSentinel;
}
@Override
public int hashCode()
{
int result = table.hashCode();
result = 31 * result + (isMinSentinel ? 1 : 0);
result = 31 * result + (isMinMinSentinel ? 1 : 0);
return result;
}
@Override
public RoutingKeyKind kindOfRoutingKey()
{
return RoutingKeyKind.SENTINEL;
}
@Override
public long estimatedSizeOnHeap()
{
return EMPTY_SIZE;
}
public AccordRoutingKey withTable(TableId table)
{
return new SentinelKey(table, isMinSentinel, isMinMinSentinel);
}
public static SentinelKey min(TableId table)
{
return new SentinelKey(table, true, false);
}
public static SentinelKey max(TableId table)
{
return new SentinelKey(table, false, false);
}
public TokenKey toTokenKeyBroken()
{
IPartitioner partitioner = getPartitioner();
return new TokenKey(table, isMinSentinel ?
partitioner.getMinimumToken().nextValidToken() :
partitioner.getMaximumTokenForSplitting());
}
@Override
public Token token()
{
throw new UnsupportedOperationException();
}
int asInt()
{
if (isMinSentinel)
{
if (isMinMinSentinel)
return -2;
else
return -1;
}
else
{
if (isMinMinSentinel)
return 1;
else
return 2;
}
}
@Override
public String suffix()
{
return isMinSentinel ? "-Inf" : "+Inf";
}
public static final AccordKeySerializer<SentinelKey> serializer = new AccordKeySerializer<SentinelKey>()
{
@Override
public void serialize(SentinelKey key, DataOutputPlus out, int version) throws IOException
{
key.table.serializeCompact(out);
out.writeBoolean(key.isMinSentinel);
out.writeBoolean(key.isMinMinSentinel);
}
@Override
public void skip(DataInputPlus in, int version) throws IOException
{
TableId.skipCompact(in);
in.skipBytesFully(2);
}
@Override
public SentinelKey deserialize(DataInputPlus in, int version) throws IOException
{
TableId table = TableId.deserializeCompact(in);
boolean isMin = in.readBoolean();
boolean isMinMin = in.readBoolean();
return new SentinelKey(table, isMin, isMinMin);
}
@Override
public long serializedSize(SentinelKey key, int version)
{
return key.table().serializedCompactSize() + TypeSizes.BOOL_SIZE + TypeSizes.BOOL_SIZE;
}
};
@Override
public Range asRange()
{
checkState(!isMinMinSentinel, "It might be possible to support converting a minmin sentinel to a range, but it needs to be evaluated in the context where it is failing");
return TokenRange.create(new SentinelKey(table, isMinSentinel, true), this);
}
}
private static class TokenKeySerializer<T extends TokenKey> implements AccordKeySerializer<T>
{
private final BiFunction<TableId, Token, T> factory;
private TokenKeySerializer(BiFunction<TableId, Token, T> factory)
{
this.factory = factory;
}
@Override
public void serialize(T key, DataOutputPlus out, int version) throws IOException
{
key.table.serializeCompact(out);
Invariants.require(key.token.getPartitioner() == getPartitioner());
Token.compactSerializer.serialize(key.token, out, version);
}
@Override
public void skip(DataInputPlus in, int version) throws IOException
{
TableId.skipCompact(in);
Token.compactSerializer.skip(in, getPartitioner(), version);
}
@Override
public T deserialize(DataInputPlus in, int version) throws IOException
{
TableId table = TableId.deserializeCompact(in).intern();
Token token = Token.compactSerializer.deserialize(in, getPartitioner(), version);
return factory.apply(table, token);
}
public T fromBytes(ByteBuffer bytes, IPartitioner partitioner)
{
TableId tableId = TableId.deserializeCompact(bytes, ByteBufferAccessor.instance, 0).intern();
bytes.position(tableId.serializedCompactSize());
Token token = Token.compactSerializer.deserialize(bytes, partitioner);
return factory.apply(tableId, token);
}
public ByteBuffer toBytes(T tokenKey)
{
int size = (int) (tokenKey.table.serializedCompactSize() + Token.compactSerializer.serializedSize(tokenKey.token));
ByteBuffer out = ByteBuffer.allocate(size);
int position = tokenKey.table.serializeCompact(out, ByteBufferAccessor.instance, 0);
out.position(position);
Token.compactSerializer.serialize(tokenKey.token, out);
out.flip();
return out;
}
@Override
public long serializedSize(TokenKey key, int version)
{
return key.table.serializedCompactSize() + Token.compactSerializer.serializedSize(key.token(), version);
}
}
// Should be final in part because we refer to its class directly in AccordRoutableKey.compareTo
// but it's helpful for MinTokenKey to be able to extend so `asTokenKey` also works with it
public static class TokenKey extends AccordRoutingKey
{
private static final long EMPTY_SIZE = ObjectSizes.measure(new TokenKey(null, null));
@Override
public Range asRange()
{
AccordRoutingKey before = token.isMinimum()
? new SentinelKey(table, true, false)
: new MinTokenKey(table, token);
return TokenRange.create(before, this);
}
final Token token;
public TokenKey(TableId tableId, Token token)
{
super(tableId);
this.token = token;
}
public TokenKey withToken(Token token)
{
return new TokenKey(table, token);
}
@Override
public Token token()
{
return token;
}
@Override
public RoutingKeyKind kindOfRoutingKey()
{
return RoutingKeyKind.TOKEN;
}
@Override
public String suffix()
{
return token.toString();
}
public long estimatedSizeOnHeap()
{
return EMPTY_SIZE + token().getHeapSize();
}
public AccordRoutingKey withTable(TableId table)
{
return new TokenKey(table, token);
}
public static final TokenKeySerializer<TokenKey> serializer = new TokenKeySerializer<>(TokenKey::new);
}
// Allows the creation of a Range that is begin inclusive or end exclusive for a given Token
public static final class MinTokenKey extends TokenKey
{
private static final long EMPTY_SIZE = ObjectSizes.measure(new MinTokenKey(null, null));
@Override
public Range asRange()
{
AccordRoutingKey before = token.isMinimum()
? new SentinelKey(table, true, false)
: new TokenKey(table, token.decreaseSlightly());
return TokenRange.create(before, this);
}
public MinTokenKey(TableId tableId, Token token)
{
super(tableId, token);
}
public MinTokenKey withToken(Token token)
{
return new MinTokenKey(table, token);
}
@Override
public Token token()
{
return token;
}
@Override
public RoutingKeyKind kindOfRoutingKey()
{
return RoutingKeyKind.MIN_TOKEN;
}
@Override
public String suffix()
{
return token.toString();
}
public long estimatedSizeOnHeap()
{
return EMPTY_SIZE + token().getHeapSize();
}
public AccordRoutingKey withTable(TableId table)
{
return new MinTokenKey(table, token);
}
public static final TokenKeySerializer<MinTokenKey> serializer = new TokenKeySerializer<>(MinTokenKey::new);
}
public static class Serializer implements AccordKeySerializer<AccordRoutingKey>
{
static final RoutingKeyKind[] kinds = RoutingKeyKind.values();
@Override
public void serialize(AccordRoutingKey key, DataOutputPlus out, int version) throws IOException
{
out.write(key.kindOfRoutingKey().ordinal());
switch (key.kindOfRoutingKey())
{
case TOKEN:
TokenKey.serializer.serialize((TokenKey) key, out, version);
break;
case SENTINEL:
SentinelKey.serializer.serialize((SentinelKey) key, out, version);
break;
case MIN_TOKEN:
MinTokenKey.serializer.serialize((MinTokenKey) key, out, version);
break;
default:
throw new IllegalArgumentException();
}
}
public ByteBuffer serialize(AccordRoutingKey key)
{
try (DataOutputBuffer buffer = new DataOutputBuffer((int)serializedSize(key, 0)))
{
try
{
serialize(key, buffer, 0);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
return buffer.asNewBuffer();
}
}
public AccordRoutingKey deserialize(ByteBuffer buffer)
{
try (DataInputBuffer in = new DataInputBuffer(buffer, true))
{
try
{
return deserialize(in, 0);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
@Override
public void skip(DataInputPlus in, int version) throws IOException
{
RoutingKeyKind kind = kinds[in.readByte()];
switch (kind)
{
case TOKEN:
TokenKey.serializer.skip(in, version);
break;
case SENTINEL:
SentinelKey.serializer.skip(in, version);
break;
case MIN_TOKEN:
MinTokenKey.serializer.skip(in, version);
break;
default:
throw new IllegalArgumentException();
}
}
@Override
public AccordRoutingKey deserialize(DataInputPlus in, int version) throws IOException
{
RoutingKeyKind kind = kinds[in.readByte()];
switch (kind)
{
case TOKEN:
return TokenKey.serializer.deserialize(in, version);
case SENTINEL:
return SentinelKey.serializer.deserialize(in, version);
case MIN_TOKEN:
return MinTokenKey.serializer.deserialize(in, version);
default:
throw new IllegalArgumentException();
}
}
@Override
public long serializedSize(AccordRoutingKey key, int version)
{
long size = TypeSizes.BYTE_SIZE; // kind ordinal
switch (key.kindOfRoutingKey())
{
case TOKEN:
size += TokenKey.serializer.serializedSize((TokenKey) key, version);
break;
case SENTINEL:
size += SentinelKey.serializer.serializedSize((SentinelKey) key, version);
break;
case MIN_TOKEN:
size += MinTokenKey.serializer.serializedSize((MinTokenKey)key, version);
break;
default:
throw new IllegalArgumentException();
}
return size;
}
}
public static final Serializer serializer = new Serializer();
public static class KeyspaceSplitter implements ShardDistributor
{
final EvenSplit<BigInteger> subSplitter;
public KeyspaceSplitter(EvenSplit<BigInteger> subSplitter)
{
this.subSplitter = subSplitter;
}
@Override
public List<Ranges> split(Ranges ranges)
{
Map<TableId, List<Range>> byTable = new TreeMap<>();
for (Range range : ranges)
{
byTable.computeIfAbsent(((AccordRoutableKey)range.start()).table, ignore -> new ArrayList<>())
.add(range);
}
List<Ranges> results = new ArrayList<>();
for (List<Range> keyspaceRanges : byTable.values())
{
List<Ranges> splits = subSplitter.split(Ranges.ofSortedAndDeoverlapped(keyspaceRanges.toArray(new Range[0])));
for (int i = 0; i < splits.size(); i++)
{
if (i == results.size()) results.add(Ranges.EMPTY);
results.set(i, results.get(i).with(splits.get(i)));
}
}
return results;
}
@Override
public Range splitRange(Range range, int from, int to, int numSplits)
{
return subSplitter.splitRange(range, from, to, numSplits);
}
@Override
public int numberOfSplitsPossible(Range range)
{
return subSplitter.numberOfSplitsPossible(range);
}
}
}

11
src/java/org/apache/cassandra/service/accord/api/PartitionKey.java
View File

@ -40,7 +40,6 @@ import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
@ -95,9 +94,15 @@ public final class PartitionKey extends AccordRoutableKey implements Key
}
@Override
public TokenKey toUnseekable()
public org.apache.cassandra.service.accord.api.TokenKey toUnseekable()
{
return new TokenKey(table, token());
return new org.apache.cassandra.service.accord.api.TokenKey(table, token());
}
@Override
byte sentinel()
{
return org.apache.cassandra.service.accord.api.TokenKey.NORMAL_SENTINEL;
}
public long estimatedSizeOnHeap()

599
src/java/org/apache/cassandra/service/accord/api/TokenKey.java Normal file
View File

@ -0,0 +1,599 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.api;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.google.common.annotations.VisibleForTesting;
import accord.api.RoutingKey;
import accord.local.ShardDistributor;
import accord.primitives.Range;
import accord.primitives.RangeFactory;
import accord.primitives.Ranges;
import accord.utils.Invariants;
import accord.utils.VIntCoding;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.utils.ObjectSizes;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
public final class TokenKey extends AccordRoutableKey implements RoutingKey, RangeFactory
{
public enum RoutingKeyKind
{
TOKEN, SENTINEL, MIN_TOKEN
}
private static final long EMPTY_SIZE = ObjectSizes.measure(new TokenKey(null, null));
@Override
public Range asRange()
{
return TokenRange.create(before(), this);
}
// we use the first 2 bits as a prefix, and the last 6 bits as a postfix comparison
final byte sentinel;
final Token token;
public TokenKey(TableId tableId, byte sentinel, Token token)
{
super(tableId);
this.sentinel = sentinel;
this.token = token;
}
public TokenKey(TableId tableId, Token token)
{
this(tableId, NORMAL_SENTINEL, token);
}
public TokenKey withToken(Token token)
{
return new TokenKey(table, sentinel, token);
}
@Override
public Token token()
{
return token;
}
@Override
public byte sentinel()
{
return sentinel;
}
public byte prefixSentinel()
{
return (byte) (sentinel & PREFIX_MASK);
}
public byte suffixSentinel()
{
return (byte) (sentinel & SUFFIX_MASK);
}
// this can be invoked to a depth of 3 from a real token
@VisibleForTesting
public TokenKey before()
{
int lowestBit = Integer.lowestOneBit(sentinel);
Invariants.require(lowestBit != 1);
byte newSentinel = (byte)((sentinel ^ lowestBit) | (lowestBit >>> 1));
return new TokenKey(table, newSentinel, token);
}
// this can be invoked to a depth of 2 from a real token
@VisibleForTesting
public TokenKey after()
{
int lowestBit = Integer.lowestOneBit(sentinel);
// we can't use 0xf as we would not be able to disambiguate with variable length byte encoding escape
Invariants.require((lowestBit != 1) && (sentinel & 0xf) != 0xe);
byte newSentinel = (byte)(sentinel | (lowestBit >>> 1));
return new TokenKey(table, newSentinel, token);
}
@Override
public Object suffix()
{
return token;
}
public Object printableSuffix()
{
Object suffix = suffix();
if (isSentinel())
{
if (isTableSentinel()) suffix = isMin() ? "-Inf" : "+Inf";
else suffix = suffix + "(-epsilon)";
}
return suffix;
}
@Override
public String toString()
{
return prefix() + ":" + printableSuffix();
}
public boolean isMin()
{
return sentinel == MIN_TABLE_SENTINEL;
}
public boolean isMax()
{
return sentinel == MAX_TABLE_SENTINEL;
}
public long estimatedSizeOnHeap()
{
return EMPTY_SIZE + token().getHeapSize();
}
public TokenKey withTable(TableId table)
{
return new TokenKey(table, sentinel, token);
}
@Override
public RangeFactory rangeFactory()
{
return this;
}
@Override
public Range newRange(RoutingKey start, RoutingKey end)
{
return TokenRange.create((TokenKey) start, (TokenKey) end);
}
@Override
public Range newAntiRange(RoutingKey start, RoutingKey end)
{
return TokenRange.createUnsafe((TokenKey) start, (TokenKey) end);
}
@Override
public RoutingKey toUnseekable()
{
return this;
}
public boolean isSentinel()
{
return sentinel != NORMAL_SENTINEL;
}
public boolean isTableSentinel()
{
return (sentinel & PREFIX_MASK) != (NORMAL_SENTINEL & PREFIX_MASK);
}
public boolean isTokenSentinel()
{
return (sentinel & SUFFIX_MASK) != (NORMAL_SENTINEL & SUFFIX_MASK);
}
public static TokenKey min(TableId table, IPartitioner partitioner)
{
return new TokenKey(table, MIN_TABLE_SENTINEL, partitioner.getMinimumToken());
}
public static TokenKey max(TableId table, IPartitioner partitioner)
{
return new TokenKey(table, MAX_TABLE_SENTINEL, partitioner.getMinimumToken());
}
public static TokenKey before(TableId table, Token token)
{
return new TokenKey(table, BEFORE_TOKEN_SENTINEL, token);
}
public static final class Serializer implements AccordSearchableKeySerializer<TokenKey>
{
private Serializer() {}
// stream serialization methods - including a dynamic length for variable size tokens
// types are byte comparable only after any length component
@Override
public long serializedSize(TokenKey key, int version)
{
IPartitioner partitioner = key.token.getPartitioner();
int size = 2 + key.table.serializedCompactComparableSize();
int tokenSize = partitioner.accordFixedLength();
if (tokenSize >= 0)
return size + tokenSize;
tokenSize = partitioner.accordSerializedSize(key.token);
return size + tokenSize + VIntCoding.sizeOfUnsignedVInt(tokenSize);
}
@Override
public void serialize(TokenKey key, DataOutputPlus out, int version) throws IOException
{
IPartitioner partitioner = key.token.getPartitioner();
int fixedLength = partitioner.accordFixedLength();
if (fixedLength < 0)
{
int len = partitioner.accordSerializedSize(key.token);
out.writeUnsignedVInt32(len);
}
key.table.serializeCompactComparable(out);
serializeWithoutPrefixOrLength(key, out, version);
}
@Override
public TokenKey deserialize(DataInputPlus in, int version) throws IOException
{
return deserialize(in, version, getPartitioner());
}
public TokenKey deserialize(DataInputPlus in, int version, IPartitioner partitioner) throws IOException
{
int len = partitioner.accordFixedLength();
if (len < 0) len = in.readUnsignedVInt32();
TableId tableId = deserializePrefix(in, version);
return deserializeWithPrefix(tableId, len + 2, in, version, partitioner);
}
@Override
public void skip(DataInputPlus in, int version) throws IOException
{
skip(in, version, getPartitioner());
}
public void skip(DataInputPlus in, int version, IPartitioner partitioner) throws IOException
{
int len = partitioner.accordFixedLength();
if (len < 0) len = in.readUnsignedVInt32();
TableId.skipCompact(in);
in.skipBytesFully(len + 2);
}
// methods for encoding/decoding a single ByteBuffer value
public ByteBuffer serialize(TokenKey key)
{
int size = key.table.serializedCompactComparableSize() + serializedSizeWithoutPrefix(key);
ByteBuffer result = ByteBuffer.allocate(size);
result.position(key.table.serializeCompactComparable(result, ByteBufferAccessor.instance, 0));
serializeWithoutPrefixOrLength(key, result);
result.flip();
return result;
}
public TokenKey deserialize(ByteBuffer buffer)
{
return deserialize(buffer, getPartitioner());
}
public TokenKey deserialize(ByteBuffer buffer, IPartitioner partitioner)
{
int offset = buffer.position();
TableId tableId = TableId.deserializeCompactComparable(buffer, ByteBufferAccessor.instance, offset);
buffer.position(offset + tableId.serializedCompactComparableSize());
return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner);
}
// methods for encoding searchable tokens separately from tableIds
@Override
public int fixedKeyLengthForPrefix(Object prefix)
{
return getPartitioner().accordFixedLength();
}
@Override
public int serializedSizeWithoutPrefix(TokenKey key)
{
return 2 + key.token.getPartitioner().accordSerializedSize(key.token);
}
@Override
public int serializedSizeOfPrefix(Object prefix)
{
return ((TableId) prefix).serializedCompactComparableSize();
}
@Override
public void serializePrefix(Object prefix, DataOutputPlus out, int version) throws IOException
{
((TableId)prefix).serializeCompactComparable(out);
}
@Override
public void serializeWithoutPrefixOrLength(TokenKey key, DataOutputPlus out, int version) throws IOException
{
out.write(key.prefixSentinel());
key.token.getPartitioner().accordSerialize(key.token, out);
out.write(key.suffixSentinel());
}
public ByteBuffer serializeWithoutPrefixOrLength(TokenKey key)
{
IPartitioner partitioner = key.token.getPartitioner();
ByteBuffer result = ByteBuffer.allocate(serializedSizeWithoutPrefix(key));
serializeWithoutPrefixOrLength(key, result, partitioner);
result.flip();
return result;
}
public void serializeWithoutPrefixOrLength(TokenKey key, ByteBuffer out)
{
serializeWithoutPrefixOrLength(key, out, key.token.getPartitioner());
}
private static void serializeWithoutPrefixOrLength(TokenKey key, ByteBuffer out, IPartitioner partitioner)
{
out.put(key.prefixSentinel());
partitioner.accordSerialize(key.token, out);
out.put(key.suffixSentinel());
}
@Override
public TableId deserializePrefix(DataInputPlus in, int version) throws IOException
{
return TableId.deserializeCompactComparable(in);
}
@Override
public TokenKey deserializeWithPrefix(Object tableId, int length, DataInputPlus in, int version) throws IOException
{
return deserializeWithPrefix(tableId, length, in, version, getPartitioner());
}
public TokenKey deserializeWithPrefix(Object tableId, int length, DataInputPlus in, int version, IPartitioner partitioner) throws IOException
{
byte sentinel = in.readByte();
Token token = partitioner.accordDeserialize(in, length - 2);
sentinel |= in.readByte();
return new TokenKey((TableId) tableId, sentinel, token);
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, V src, ValueAccessor<V> accessor, int offset)
{
return deserializeWithPrefixAndImpliedLength(tableId, src, accessor, offset, getPartitioner());
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, V src, ValueAccessor<V> accessor, int offset, IPartitioner partitioner)
{
return deserializeWithPrefix(tableId, accessor.remaining(src, offset), src, accessor, offset, partitioner);
}
public <V> TokenKey deserializeWithPrefix(Object tableId, int length, V src, ValueAccessor<V> accessor, int offset, IPartitioner partitioner)
{
byte sentinel = accessor.getByte(src, offset++);
Token token = partitioner.accordDeserialize(src, accessor, offset, length - 2);
offset += partitioner.accordSerializedSize(token);
sentinel |= accessor.getByte(src, offset);
return new TokenKey((TableId) tableId, sentinel, token);
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer)
{
return deserializeWithPrefixAndImpliedLength(tableId, buffer, getPartitioner());
}
public <V> TokenKey deserializeWithPrefixAndImpliedLength(Object tableId, ByteBuffer buffer, IPartitioner partitioner)
{
return deserializeWithPrefix(tableId, buffer.remaining(), buffer, partitioner);
}
public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer)
{
return deserializeWithPrefix(tableId, length, buffer, getPartitioner());
}
public TokenKey deserializeWithPrefix(Object tableId, int length, ByteBuffer buffer, IPartitioner partitioner)
{
byte sentinel = buffer.get();
Token token = partitioner.accordDeserialize(buffer, length - 2);
sentinel |= buffer.get();
return new TokenKey((TableId) tableId, sentinel, token);
}
public static final byte ESCAPE_BYTE = 0x0f;
private static final byte[] ESCAPE_BYTES = new byte[] { ESCAPE_BYTE };
private static final int UNESCAPE = ESCAPE_BYTE;
private static final int UNESCAPE_MASK = 0xffff;
public static int countEscapes(byte[] bytes)
{
int escapeLimit = escapeLimit(bytes);
int i = 0;
int count = 0;
while ((i = nextEscape(bytes, i, escapeLimit)) >= 0)
{
++count;
++i;
}
return count;
}
public static int serializedSize(byte[] bytes)
{
return 1 + bytes.length + countEscapes(bytes);
}
private static int escapeLimit(byte[] bytes)
{
return bytes.length - 1;
}
private static int nextEscape(byte[] bytes, int index, int escapeLimit)
{
while (index <= escapeLimit)
{
if (bytes[index] == 0 && (index == escapeLimit || bytes[index + 1] <= ESCAPE_BYTE))
return index;
++index;
}
return -1;
}
public static void serializeWithEscapes(byte[] bytes, ByteBuffer out)
{
serializeWithEscapesInternal(bytes, out, ByteBuffer::put);
out.put(trailingByte(bytes));
}
public static void serializeWithEscapes(byte[] bytes, DataOutputPlus out) throws IOException
{
serializeWithEscapesInternal(bytes, out, DataOutputPlus::write);
out.writeByte(trailingByte(bytes));
}
interface WriteBytes<V, T extends Throwable>
{
void write(V out, byte[] bytes, int offset, int length) throws T;
}
private static byte trailingByte(byte[] bytes)
{
return 0;
}
private static <V, T extends Throwable> void serializeWithEscapesInternal(byte[] bytes, V out, WriteBytes<V, T> write) throws T
{
int i = 0, escapeLimit = escapeLimit(bytes);
while (true)
{
int nexti = nextEscape(bytes, i, escapeLimit);
if (nexti < 0)
break;
write.write(out, bytes, i, 1 + nexti - i);
write.write(out, ESCAPE_BYTES, 0, 1);
i = nexti + 1;
}
write.write(out, bytes, i, bytes.length - i);
}
public static byte[] deserializeWithEscapes(ByteBuffer in, int escapedLength)
{
Invariants.require(escapedLength >= 1);
--escapedLength;
byte[] bytes = new byte[escapedLength];
in.get(bytes, 0, Math.min(escapedLength, in.remaining()));
byte[] result = removeEscapes(bytes);
byte trailingEscape = in.get();
Invariants.require(trailingEscape == trailingByte(result));
return result;
}
public static <V> byte[] deserializeWithEscapes(V src, ValueAccessor<V> accessor, int offset, int escapedLength)
{
Invariants.require(--escapedLength >= 0);
byte[] result = removeEscapes(accessor.toArray(src, offset, Math.min(escapedLength, accessor.remaining(src, offset))));
Invariants.require(trailingByte(result) == accessor.getByte(src, offset + escapedLength));
return result;
}
public static byte[] deserializeWithEscapes(DataInputPlus in, int escapedLength) throws IOException
{
byte[] result = new byte[escapedLength - 1];
in.readFully(result);
result = removeEscapes(result);
byte trailingEscape = in.readByte();
Invariants.require(trailingEscape == trailingByte(result));
return result;
}
private static byte[] removeEscapes(byte[] bytes)
{
if (bytes.length == 0)
return bytes;
int count = 1;
int escapeMatcher = bytes[0];
for (int i = 1; i < bytes.length ; ++i)
{
byte next = bytes[i];
escapeMatcher = (escapeMatcher << 8) | next;
if ((escapeMatcher & UNESCAPE_MASK) != UNESCAPE)
{
if (count != i)
bytes[count] = next;
count++;
}
}
if (bytes.length != count)
bytes = Arrays.copyOf(bytes, count);
return bytes;
}
}
public static final Serializer serializer = new Serializer();
public static class KeyspaceSplitter implements ShardDistributor
{
final EvenSplit<BigInteger> subSplitter;
public KeyspaceSplitter(EvenSplit<BigInteger> subSplitter)
{
this.subSplitter = subSplitter;
}
@Override
public List<Ranges> split(Ranges ranges)
{
Map<TableId, List<Range>> byTable = new TreeMap<>();
for (Range range : ranges)
{
byTable.computeIfAbsent(((AccordRoutableKey)range.start()).table, ignore -> new ArrayList<>())
.add(range);
}
List<Ranges> results = new ArrayList<>();
for (List<Range> keyspaceRanges : byTable.values())
results.addAll(subSplitter.split(Ranges.ofSortedAndDeoverlapped(keyspaceRanges.toArray(new Range[0]))));
return results;
}
@Override
public Range splitRange(Range range, int from, int to, int numSplits)
{
return subSplitter.splitRange(range, from, to, numSplits);
}
@Override
public int numberOfSplitsPossible(Range range)
{
return subSplitter.numberOfSplitsPossible(range);
}
}
}

5
src/java/org/apache/cassandra/service/accord/interop/AccordInteropAdapter.java
View File

@ -27,6 +27,7 @@ import accord.api.Result;
import accord.api.Update;
import accord.coordinate.CoordinationAdapter;
import accord.coordinate.CoordinationAdapter.Adapters.TxnAdapter;
import accord.coordinate.ExecuteFlag.ExecuteFlags;
import accord.coordinate.ExecutePath;
import accord.local.Node;
import accord.messages.Apply;
@ -83,10 +84,10 @@ public class AccordInteropAdapter extends TxnAdapter
}
@Override
public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer<? super Result, Throwable> callback)
public void execute(Node node, Topologies any, FullRoute<?> route, ExecutePath path, ExecuteFlags executeFlags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer<? super Result, Throwable> callback)
{
if (!doInteropExecute(node, route, txnId, txn, executeAt, stableDeps, callback))
super.execute(node, any, route, path, txnId, txn, executeAt, stableDeps, sendDeps, callback);
super.execute(node, any, route, path, executeFlags, txnId, txn, executeAt, stableDeps, sendDeps, callback);
}
@Override

5
src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java
View File

@ -78,8 +78,7 @@ import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.accord.AccordEndpointMapper;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.interop.AccordInteropReadCallback.MaximalCommitSender;
import org.apache.cassandra.service.accord.txn.AccordUpdate;
@ -211,7 +210,7 @@ public class AccordInteropExecution implements ReadCoordinator, MaximalCommitSen
@Override
public EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata doNotUse, KeyspaceMetadata keyspace, TableId tableId, Token token)
{
AccordRoutingKey.TokenKey key = new AccordRoutingKey.TokenKey(tableId, token);
TokenKey key = new TokenKey(tableId, token);
Shard shard = executeTopology.forKey(key);
Range<Token> range = ((TokenRange) shard.range).toKeyspaceRange();

17
src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java
View File

@ -59,8 +59,7 @@ import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.RequestCallback;
import org.apache.cassandra.service.accord.AccordMessageSink.AccordMessageType;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.serializers.ReadDataSerializers;
@ -112,7 +111,7 @@ public class AccordInteropRead extends ReadData
private static class LocalReadData implements Data
{
private static final Comparator<Pair<AccordRoutingKey, ReadResponse>> RESPONSE_COMPARATOR = Comparator.comparing(Pair::left);
private static final Comparator<Pair<TokenKey, ReadResponse>> RESPONSE_COMPARATOR = Comparator.comparing(Pair::left);
static final IVersionedSerializer<LocalReadData> serializer = new IVersionedSerializer<>()
{
@ -138,13 +137,13 @@ public class AccordInteropRead extends ReadData
};
// Will be null at coordinator
List<Pair<AccordRoutingKey, ReadResponse>> localResponses;
List<Pair<TokenKey, ReadResponse>> localResponses;
// Will be null at coordinator
final ReadCommand readCommand;
// Will be not null at coordinator, but null at the node creating the response until it serialized
ReadResponse remoteResponse;
public LocalReadData(@Nullable AccordRoutingKey start, @Nonnull ReadResponse response, @Nonnull ReadCommand readCommand)
public LocalReadData(@Nullable TokenKey start, @Nonnull ReadResponse response, @Nonnull ReadCommand readCommand)
{
checkNotNull(response, "response is null");
checkNotNull(readCommand, "readCommand is null");
@ -178,7 +177,7 @@ public class AccordInteropRead extends ReadData
checkState(readCommand == other.readCommand, "Should share the same ReadCommand");
if (localResponses.size() == 1)
{
List<Pair<AccordRoutingKey, ReadResponse>> merged = new ArrayList<>();
List<Pair<TokenKey, ReadResponse>> merged = new ArrayList<>();
merged.add(localResponses.get(0));
localResponses = merged;
}
@ -191,7 +190,7 @@ public class AccordInteropRead extends ReadData
if (remoteResponse != null)
return;
// Range reads will be spread across command stores and need to be merged in token order
List<Pair<AccordRoutingKey, ReadResponse>> responses = localResponses;
List<Pair<TokenKey, ReadResponse>> responses = localResponses;
if (responses.size() == 1)
{
remoteResponse = responses.get(0).right;
@ -251,7 +250,7 @@ public class AccordInteropRead extends ReadData
for (Range r : ranges)
{
ReadCommand readCommand = this.command;
AccordRoutingKey routingKey = null;
TokenKey routingKey = null;
final ReadCommand readCommandFinal;
if (readCommand.isRangeRequest())
{
@ -272,7 +271,7 @@ public class AccordInteropRead extends ReadData
continue;
readCommandFinal = ((SinglePartitionReadCommand)readCommand).withTransactionalSettings(TxnNamedRead.readsWithoutReconciliation(txnRead.cassandraConsistencyLevel()), nowInSeconds);
}
AccordRoutingKey routingKeyFinal = routingKey;
TokenKey routingKeyFinal = routingKey;
chains.add(AsyncChains.ofCallable(Stage.READ.executor(), () -> new LocalReadData(routingKeyFinal, ReadCommandVerbHandler.instance.doRead(readCommandFinal, false), readCommand)));
}

3
src/java/org/apache/cassandra/service/accord/repair/RepairSyncPointAdapter.java
View File

@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
import accord.api.Result;
import accord.coordinate.CoordinationAdapter;
import accord.coordinate.ExecuteFlag;
import accord.coordinate.ExecutePath;
import accord.coordinate.ExecuteSyncPoint;
import accord.local.Node;
@ -58,7 +59,7 @@ public class RepairSyncPointAdapter<U extends Unseekable> extends CoordinationAd
}
@Override
public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
public void execute(Node node, Topologies all, FullRoute<?> route, ExecutePath path, ExecuteFlag.ExecuteFlags executeFlags, TxnId txnId, Txn txn, Timestamp executeAt, Deps stableDeps, Deps sendDeps, BiConsumer<? super SyncPoint<U>, Throwable> callback)
{
RequiredResponseTracker tracker = new RequiredResponseTracker(requiredResponses, all);
ExecuteSyncPoint.ExecuteInclusive<U> execute = new ExecuteSyncPoint.ExecuteInclusive<>(node, new SyncPoint<>(txnId, executeAt, stableDeps, (FullRoute<U>) route), tracker, executeAt);

13
src/java/org/apache/cassandra/service/accord/serializers/AcceptSerializers.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import accord.coordinate.ExecuteFlag.ExecuteFlags;
import accord.local.Commands.AcceptOutcome;
import accord.messages.Accept;
import accord.messages.Accept.AcceptReply;
@ -112,10 +113,12 @@ public class AcceptSerializers
public static final IVersionedSerializer<AcceptReply> reply = new ReplySerializer();
public static class ReplySerializer implements IVersionedSerializer<AcceptReply>
{
// we have one spare bit at 0x04 for either another flag or more AcceptOutcome variants
private static final int SUPERSEDED_BY = 0x08;
private static final int COMMITTED_EXECUTE_AT = 0x10;
private static final int SUCCESSFUL = 0x20;
private static final int DEPS = 0x40;
private static final int FLAGS = 0x80;
@Override
public void serialize(AcceptReply reply, DataOutputPlus out, int version) throws IOException
{
@ -123,7 +126,8 @@ public class AcceptSerializers
| (reply.supersededBy != null ? SUPERSEDED_BY : 0)
| (reply.committedExecuteAt != null ? COMMITTED_EXECUTE_AT : 0)
| (reply.successful != null ? SUCCESSFUL : 0)
| (reply.deps != null ? DEPS : 0);
| (reply.deps != null ? DEPS : 0)
| (!reply.flags.isEmpty() ? FLAGS : 0);
out.writeByte(flags);
if (reply.supersededBy != null)
@ -134,6 +138,8 @@ public class AcceptSerializers
KeySerializers.participants.serialize(reply.successful, out, version);
if (reply.deps != null)
DepsSerializers.deps.serialize(reply.deps, out, version);
if (!reply.flags.isEmpty())
out.writeUnsignedVInt32(reply.flags.bits());
}
private final AcceptOutcome[] outcomes = AcceptOutcome.values();
@ -146,7 +152,8 @@ public class AcceptSerializers
Timestamp committedExecuteAt = (flags & COMMITTED_EXECUTE_AT) == 0 ? null : ExecuteAtSerializer.deserialize(in);
Participants<?> successful = (flags & SUCCESSFUL) == 0 ? null : KeySerializers.participants.deserialize(in, version);
Deps deps = (flags & DEPS) == 0 ? null : DepsSerializers.deps.deserialize(in, version);
return new AcceptReply(outcome, supersededBy, successful, deps, committedExecuteAt);
ExecuteFlags executeFlags = (flags & FLAGS) == 0 ? ExecuteFlags.none() : ExecuteFlags.get(in.readUnsignedVInt32());
return new AcceptReply(outcome, supersededBy, successful, deps, committedExecuteAt, executeFlags);
}
@Override
@ -161,6 +168,8 @@ public class AcceptSerializers
size += KeySerializers.participants.serializedSize(reply.successful, version);
if (reply.deps != null)
size += DepsSerializers.deps.serializedSize(reply.deps, version);
if (!reply.flags.isEmpty())
size += TypeSizes.sizeofUnsignedVInt(reply.flags.bits());
return size;
}
};

368
src/java/org/apache/cassandra/service/accord/serializers/AccordRoutingKeyByteSource.java
View File

@ -1,368 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import java.util.UUID;
import java.util.function.BiFunction;
import javax.annotation.Nullable;
import org.apache.cassandra.db.marshal.ByteArrayAccessor;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.ValueAccessor;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.cassandra.utils.TriFunction;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSource;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
public class AccordRoutingKeyByteSource
{
public static final ByteComparable.Version currentVersion = ByteComparable.Version.OSS50;
// TODO (review): I'm not sure MIN_MIN has a use, maybe try and remove?
private static final byte[] MIN_MIN_ORDER = { -2 };
private static final byte MIN_MIN_ORDER_BYTE = -2;
private static final byte[] MIN_MAX_ORDER = { -1 };
private static final byte MIN_MAX_ORDER_BYTE = -1;
private static final byte[] MIN_TOKEN_ORDER = { 0 };
private static final byte MIN_TOKEN_ORDER_BYTE = 0;
private static final byte[] TOKEN_ORDER = { 1 };
private static final byte TOKEN_ORDER_BYTE = 1;
private static final byte[] MAX_MIN_ORDER = { 2 };
private static final byte MAX_MIN_ORDER_BYTE = 2;
private static final byte[] MAX_MAX_ORDER = { 3 };
private static final byte MAX_MAX_ORDER_BYTE = 3;
private static ByteSource minMinPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MIN_MIN_ORDER);
}
private static ByteSource minMaxPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MIN_MAX_ORDER);
}
private static ByteSource minTokenPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MIN_TOKEN_ORDER);
}
private static ByteSource tokenPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, TOKEN_ORDER);
}
private static ByteSource maxMinPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MAX_MIN_ORDER);
}
private static ByteSource maxMaxPrefix()
{
return ByteSource.signedFixedLengthNumber(ByteArrayAccessor.instance, MAX_MAX_ORDER);
}
public static Serializer create(IPartitioner partitioner)
{
if (partitioner.isFixedLength())
return new FixedLength(partitioner, currentVersion);
return new VariableLength(partitioner, currentVersion);
}
public static FixedLength fixedLength(IPartitioner partitioner)
{
return new FixedLength(partitioner, currentVersion);
}
public static VariableLength variableLength(IPartitioner partitioner)
{
return new VariableLength(partitioner, currentVersion);
}
public static abstract class Serializer
{
protected final IPartitioner partitioner;
protected final ByteComparable.Version version;
protected final byte[] empty;
protected Serializer(IPartitioner partitioner, ByteComparable.Version version, byte[] empty)
{
this.partitioner = partitioner;
this.version = version;
this.empty = empty;
}
public ByteSource minMinAsComparableBytes()
{
return ByteSource.withTerminator(ByteSource.TERMINATOR, minMinPrefix(), ByteSource.fixedLength(empty));
}
public ByteSource minMaxAsComparableBytes()
{
return ByteSource.withTerminator(ByteSource.TERMINATOR, minMaxPrefix(), ByteSource.fixedLength(empty));
}
public ByteSource maxMinAsComparableBytes()
{
return ByteSource.withTerminator(ByteSource.TERMINATOR, maxMinPrefix(), ByteSource.fixedLength(empty));
}
public ByteSource maxMaxAsComparableBytes()
{
return ByteSource.withTerminator(ByteSource.TERMINATOR, maxMaxPrefix(), ByteSource.fixedLength(empty));
}
public ByteSource asComparableBytes(Token token, ByteSource prefix)
{
if (token.getPartitioner() != partitioner)
throw new IllegalArgumentException("Attempted to use the wrong partitioner: given " + token.getPartitioner() + " but expected " + partitioner);
return ByteSource.withTerminator(ByteSource.TERMINATOR, prefix, token.asComparableBytes(version));
}
public <V> Token tokenFromComparableBytes(ValueAccessor<V> accessor, V data) throws IOException
{
return tokenFromComparableBytes(ByteSource.peekable(ByteSource.fixedLength(accessor, data)));
}
public Token tokenFromComparableBytes(ByteSource.Peekable bs) throws IOException
{
if (bs.peek() == ByteSource.TERMINATOR)
throw new IOException("Unable to read prefix");
ByteSource.Peekable component = progress(bs);
byte[] prefix = ByteSourceInverse.getOptionalSignedFixedLength(ByteArrayAccessor.instance, component, 1);
if (prefix == null)
throw new IOException("Unable to read prefix; prefix was null");
switch (prefix[0])
{
case TOKEN_ORDER_BYTE:
case MIN_TOKEN_ORDER_BYTE:
break;
default:
String match = "unknown";
switch (prefix[0])
{
case MIN_MIN_ORDER_BYTE:
match = "minmin"; break;
case MIN_MAX_ORDER_BYTE:
match = "minmax"; break;
case MAX_MIN_ORDER_BYTE:
match = "maxmin"; break;
case MAX_MAX_ORDER_BYTE:
match = "maxmax"; break;
}
throw new IOException("Attempt to read token from non-token value: was " + match);
}
component = ByteSourceInverse.nextComponentSource(bs);
if (component == null)
throw new IOException("Unable to read token; component was not found");
return partitioner.getTokenFactory().fromComparableBytes(component, version);
}
public ByteSource asComparableBytes(AccordRoutingKey key)
{
UUID uuid = key.table().asUUID();
ByteSource[] srcs = { LongType.instance.asComparableBytes(LongType.instance.decompose(uuid.getMostSignificantBits()), ByteComparable.Version.OSS50),
LongType.instance.asComparableBytes(LongType.instance.decompose(uuid.getLeastSignificantBits()), ByteComparable.Version.OSS50),
asComparableBytesNoTable(key) };
return ByteSource.withTerminator(ByteSource.TERMINATOR, srcs);
}
public ByteSource asComparableBytesNoTable(AccordRoutingKey key)
{
switch (key.kindOfRoutingKey())
{
case SENTINEL:
SentinelKey sentinelKey = key.asSentinelKey();
if (sentinelKey.isMinSentinel)
{
if (sentinelKey.isMinMinSentinel)
return minMinAsComparableBytes();
else
return minMaxAsComparableBytes();
}
else
{
if (sentinelKey.isMinMinSentinel)
return maxMinAsComparableBytes();
else
return maxMaxAsComparableBytes();
}
case TOKEN:
return asComparableBytes(key.token(), tokenPrefix());
case MIN_TOKEN:
return asComparableBytes(key.token(), minTokenPrefix());
default:
throw new IllegalStateException("Unhandled routing key type " + key.kindOfRoutingKey());
}
}
public <V> AccordRoutingKey fromComparableBytes(ValueAccessor<V> accessor, V data) throws IOException
{
return fromComparableBytes(accessor, data, version, partitioner);
}
public static <V> AccordRoutingKey fromComparableBytes(ValueAccessor<V> accessor, V data, ByteComparable.Version version, @Nullable IPartitioner partitioner)
{
ByteSource.Peekable bs = ByteSource.peekable(ByteSource.fixedLength(accessor, data));
long[] uuidValues = new long[2];
for (int i = 0; i < 2; i++)
{
if (bs.peek() == ByteSource.TERMINATOR)
throw new IllegalArgumentException("Unable to parse bytes");
ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(bs);
long value = LongType.instance.compose(LongType.instance.fromComparableBytes(component, ByteComparable.Version.OSS50));
uuidValues[i] = value;
}
TableId tableId = TableId.fromUUID(new UUID(uuidValues[0], uuidValues[1]));
return fromComparableBytes(bs, tableId, version, partitioner);
}
public static <V> AccordRoutingKey fromComparableBytes(ValueAccessor<V> accessor, V data, TableId tableId, ByteComparable.Version version, @Nullable IPartitioner partitioner)
{
ByteSource.Peekable bs = ByteSource.peekable(ByteSource.fixedLength(accessor, data));
return fromComparableBytes(bs, tableId, version, partitioner);
}
public static <V> AccordRoutingKey fromComparableBytes(ByteSource.Peekable bs, TableId tableId, ByteComparable.Version version, @Nullable IPartitioner partitioner)
{
if (partitioner == null)
partitioner = AccordKeyspace.partitioner(tableId);
return fromComparableBytes(bs, tableId,
SentinelKey::new,
AccordRoutingKey.TokenKey::new,
AccordRoutingKey.MinTokenKey::new,
version, partitioner
);
}
public static AccordRoutingKey fromComparableBytes(ByteSource.Peekable bs, TableId tableId,
TriFunction<TableId, Boolean, Boolean, AccordRoutingKey> onSentinel,
BiFunction<TableId, Token, AccordRoutingKey> onToken,
BiFunction<TableId, Token, AccordRoutingKey> onMinToken,
ByteComparable.Version version, IPartitioner partitioner)
{
if (bs.peek() == ByteSource.TERMINATOR)
throw new IllegalStateException("Unable to read prefix");
ByteSource.Peekable component = progress(bs);
byte[] prefix = ByteSourceInverse.getOptionalSignedFixedLength(ByteArrayAccessor.instance, component, 1);
if (prefix == null)
throw new IllegalStateException("Unable to read prefix; prefix was null");
switch (prefix[0])
{
case TOKEN_ORDER_BYTE:
component = ByteSourceInverse.nextComponentSource(bs);
if (component == null)
throw new IllegalStateException("Unable to read token; component was not found");
return onToken.apply(tableId, partitioner.getTokenFactory().fromComparableBytes(component, version));
case MIN_TOKEN_ORDER_BYTE:
component = ByteSourceInverse.nextComponentSource(bs);
if (component == null)
throw new IllegalStateException("Unable to read token; component was not found");
return onMinToken.apply(tableId, partitioner.getTokenFactory().fromComparableBytes(component, version));
case MIN_MIN_ORDER_BYTE:
return onSentinel.apply(tableId, true, true);
case MIN_MAX_ORDER_BYTE:
return onSentinel.apply(tableId, true, false);
case MAX_MIN_ORDER_BYTE:
return onSentinel.apply(tableId, false, true);
case MAX_MAX_ORDER_BYTE:
return onSentinel.apply(tableId, false, false);
default:
throw new AssertionError("Unknown prefix");
}
}
private static ByteSource.Peekable progress(ByteSource.Peekable bs)
{
ByteSource.Peekable component = ByteSourceInverse.nextComponentSource(bs);
if (component == null)
throw new IllegalStateException("Unable to read prefix; component was not found");
if (component.peek() == ByteSource.NEXT_COMPONENT)
{
// this came from (table, token_or_sentinel)
component = ByteSourceInverse.nextComponentSource(bs);
if (component == null)
throw new IllegalStateException("Unable to read prefix; component was not found");
}
return component;
}
public byte[] serialize(Token token)
{
return ByteSourceInverse.readBytes(asComparableBytes(token, tokenPrefix()));
}
public byte[] serialize(AccordRoutingKey key)
{
return ByteSourceInverse.readBytes(asComparableBytes(key));
}
public byte[] serializeNoTable(AccordRoutingKey key)
{
return ByteSourceInverse.readBytes(asComparableBytesNoTable(key));
}
}
public static class VariableLength extends Serializer
{
public VariableLength(IPartitioner partitioner, ByteComparable.Version version)
{
super(partitioner, version, ByteArrayUtil.EMPTY_BYTE_ARRAY);
}
}
public static class FixedLength extends Serializer
{
public FixedLength(IPartitioner partitioner, ByteComparable.Version version)
{
super(partitioner, version, computeEmptyBytes(partitioner, version));
}
private static byte[] computeEmptyBytes(IPartitioner partitioner, ByteComparable.Version version)
{
if (!partitioner.isFixedLength())
throw new IllegalArgumentException("Unable to use partitioner " + partitioner.getClass() + "; it is not fixed-length");
int tokenSize = ByteSourceInverse.readBytes(partitioner.getMinimumToken().asComparableBytes(version)).length;
return new byte[tokenSize];
}
public int valueSize()
{
return 4 + empty.length;
}
}
}

8
src/java/org/apache/cassandra/service/accord/serializers/GetEphmrlReadDepsSerializers.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import accord.coordinate.ExecuteFlag.ExecuteFlags;
import accord.messages.GetEphemeralReadDeps;
import accord.messages.GetEphemeralReadDeps.GetEphemeralReadDepsOk;
import accord.primitives.Deps;
@ -61,6 +62,7 @@ public class GetEphmrlReadDepsSerializers
{
DepsSerializers.deps.serialize(reply.deps, out, version);
out.writeUnsignedVInt(reply.latestEpoch);
out.writeUnsignedVInt32(reply.flags.bits());
}
@Override
@ -68,14 +70,16 @@ public class GetEphmrlReadDepsSerializers
{
Deps deps = DepsSerializers.deps.deserialize(in, version);
long latestEpoch = in.readUnsignedVInt();
return new GetEphemeralReadDepsOk(deps, latestEpoch);
ExecuteFlags flags = ExecuteFlags.get(in.readUnsignedVInt32());
return new GetEphemeralReadDepsOk(deps, latestEpoch, flags);
}
@Override
public long serializedSize(GetEphemeralReadDepsOk reply, int version)
{
return DepsSerializers.deps.serializedSize(reply.deps, version)
+ TypeSizes.sizeofUnsignedVInt(reply.latestEpoch);
+ TypeSizes.sizeofUnsignedVInt(reply.latestEpoch)
+ TypeSizes.sizeofUnsignedVInt(reply.flags.bits());
}
};
}

384
src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.IntFunction;
@ -42,7 +43,9 @@ import accord.primitives.PartialRoute;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.RoutableKey;
import accord.primitives.Routables;
import accord.primitives.Route;
import accord.primitives.RoutingKeys;
import accord.primitives.Seekable;
@ -55,21 +58,24 @@ import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutableKey.AccordKeySerializer;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutableKey.AccordSearchableKeySerializer;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.utils.NullableSerializer;
import static accord.utils.ArrayBuffers.cachedInts;
public class KeySerializers
{
public static final AccordKeySerializer<Key> key;
public static final IVersionedSerializer<RoutingKey> routingKey;
public static final IVersionedSerializer<RoutingKey> nullableRoutingKey;
public static final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
public static final AbstractSearchableKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
public static final IVersionedSerializer<Keys> keys;
public static final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
public static final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
public static final AbstractSearchableKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
public static final AbstractSearchableKeysSerializer<?, FullKeyRoute> fullKeyRoute;
public static final IVersionedSerializer<Range> range;
public static final AbstractRangesSerializer<Ranges> ranges;
@ -120,14 +126,14 @@ public class KeySerializers
public static class Impl
{
final AccordKeySerializer<Key> key;
final IVersionedSerializer<RoutingKey> routingKey;
final AccordSearchableKeySerializer<RoutingKey> routingKey;
final IVersionedSerializer<RoutingKey> nullableRoutingKey;
final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final AbstractSearchableKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final IVersionedSerializer<Keys> keys;
final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final AbstractSearchableKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractSearchableKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final IVersionedSerializer<Range> range;
final AbstractRangesSerializer<Ranges> ranges;
@ -147,13 +153,13 @@ public class KeySerializers
private Impl()
{
this((AccordKeySerializer<Key>) (AccordKeySerializer<?>) PartitionKey.serializer,
(AccordKeySerializer<RoutingKey>) (AccordKeySerializer<?>) AccordRoutingKey.serializer,
(AccordSearchableKeySerializer<RoutingKey>) (AccordSearchableKeySerializer<?>) TokenKey.serializer,
(IVersionedSerializer<Range>) (IVersionedSerializer<?>) TokenRange.serializer);
}
@VisibleForTesting
public Impl(AccordKeySerializer<Key> key,
IVersionedSerializer<RoutingKey> routingKey,
AccordSearchableKeySerializer<RoutingKey> routingKey,
IVersionedSerializer<Range> range)
{
this.key = key;
@ -161,7 +167,7 @@ public class KeySerializers
this.range = range;
this.nullableRoutingKey = NullableSerializer.wrap(routingKey);
this.routingKeys = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
this.routingKeys = new AbstractSearchableKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override RoutingKeys deserialize(DataInputPlus in, int version, RoutingKey[] keys)
{
@ -177,8 +183,7 @@ public class KeySerializers
}
};
this.partialKeyRoute = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
this.partialKeyRoute = new AbstractSearchableKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override PartialKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
{
@ -194,14 +199,14 @@ public class KeySerializers
}
@Override
public long serializedSize(PartialKeyRoute keys, int version)
public long serializedSize(PartialKeyRoute routables, int version)
{
return super.serializedSize(keys, version)
+ routingKey.serializedSize(keys.homeKey, version);
return super.serializedSize(routables, version)
+ routingKey.serializedSize(routables.homeKey, version);
}
};
this.fullKeyRoute = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
this.fullKeyRoute = new AbstractSearchableKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override FullKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
{
@ -217,14 +222,14 @@ public class KeySerializers
}
@Override
public long serializedSize(FullKeyRoute route, int version)
public long serializedSize(FullKeyRoute routables, int version)
{
return super.serializedSize(route, version)
+ routingKey.serializedSize(route.homeKey, version);
return super.serializedSize(routables, version)
+ routingKey.serializedSize(routables.homeKey, version);
}
};
this.ranges = new AbstractRangesSerializer<Ranges>(range)
this.ranges = new AbstractRangesSerializer<>(routingKey)
{
@Override
public Ranges deserialize(DataInputPlus in, int version, Range[] ranges)
@ -234,7 +239,7 @@ public class KeySerializers
};
this.partialRangeRoute = new AbstractRangesSerializer<>(range)
this.partialRangeRoute = new AbstractRangesSerializer<>(routingKey)
{
@Override PartialRangeRoute deserialize(DataInputPlus in, int version, Range[] rs) throws IOException
{
@ -257,7 +262,7 @@ public class KeySerializers
}
};
this.fullRangeRoute = new AbstractRangesSerializer<>(range)
this.fullRangeRoute = new AbstractRangesSerializer<>(routingKey)
{
@Override FullRangeRoute deserialize(DataInputPlus in, int version, Range[] Ranges) throws IOException
{
@ -300,17 +305,17 @@ public class KeySerializers
public static class AbstractRoutablesSerializer<RS extends Unseekables<?>> implements IVersionedSerializer<RS>
{
final EnumSet<UnseekablesKind> permitted;
final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final AbstractSearchableKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final AbstractSearchableKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractSearchableKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final AbstractRangesSerializer<Ranges> ranges;
final AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute;
final AbstractRangesSerializer<FullRangeRoute> fullRangeRoute;
protected AbstractRoutablesSerializer(EnumSet<UnseekablesKind> permitted,
AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys,
AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute,
AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute,
AbstractSearchableKeysSerializer<RoutingKey, RoutingKeys> routingKeys,
AbstractSearchableKeysSerializer<?, PartialKeyRoute> partialKeyRoute,
AbstractSearchableKeysSerializer<?, FullKeyRoute> fullKeyRoute,
AbstractRangesSerializer<Ranges> ranges,
AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute,
AbstractRangesSerializer<FullRangeRoute> fullRangeRoute)
@ -518,12 +523,14 @@ public class KeySerializers
}
}
// this serializer is designed to permits using the collection in its serialized form with minimal in-memory state.
// it also saves some memory by avoiding duplicating prefixes (which happens to also assist faster lookups)
public abstract static class AbstractKeysSerializer<K extends RoutableKey, KS extends AbstractKeys<K>> implements IVersionedSerializer<KS>
{
final IVersionedSerializer<K> keySerializer;
final AccordKeySerializer<K> keySerializer;
final IntFunction<K[]> allocate;
public AbstractKeysSerializer(IVersionedSerializer<K> keySerializer, IntFunction<K[]> allocate)
public AbstractKeysSerializer(AccordKeySerializer<K> keySerializer, IntFunction<K[]> allocate)
{
this.keySerializer = keySerializer;
this.allocate = allocate;
@ -565,48 +572,313 @@ public class KeySerializers
}
}
public abstract static class AbstractRangesSerializer<RS extends AbstractRanges> implements IVersionedSerializer<RS>
// this serializer is designed to permits using the collection in its serialized form with minimal in-memory state.
// it also saves some memory by avoiding duplicating prefixes (which happens to also assist faster lookups)
public abstract static class AbstractSearchableSerializer<K extends RoutableKey, R extends Routable, RS extends Routables<R>> implements IVersionedSerializer<RS>
{
private final IVersionedSerializer<Range> rangeSerializer;
final AccordSearchableKeySerializer<K> keySerializer;
final IntFunction<R[]> allocate;
public AbstractRangesSerializer(IVersionedSerializer<Range> rangeSerializer)
public AbstractSearchableSerializer(AccordSearchableKeySerializer<K> keySerializer, IntFunction<R[]> allocate)
{
this.rangeSerializer = rangeSerializer;
this.keySerializer = keySerializer;
this.allocate = allocate;
}
private int serializedSizeOfPrefix(Object prefix)
{
return keySerializer.serializedSizeOfPrefix(prefix);
}
private void serializePrefix(Object prefix, DataOutputPlus out, int version) throws IOException
{
keySerializer.serializePrefix(prefix, out, version);
}
private Object deserializePrefix(DataInputPlus in, int version) throws IOException
{
return keySerializer.deserializePrefix(in, version);
}
// if we store Ranges, we have twice as many indexes
abstract int recordCountToLengthCount(int recordCount);
abstract int fixedKeyLengthForPrefix(Object prefix);
abstract int serializedSizeWithoutPrefix(R routable);
abstract void serializeWithoutPrefixOrLength(R routable, DataOutputPlus out, int version) throws IOException;
abstract void serializeOffsets(RS unseekables, int start, int end, DataOutputPlus out) throws IOException;
abstract R deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException;
abstract R deserializeWithPrefix(Object prefix, int lengthIndex, int[] lengths, DataInputPlus in, int version) throws IOException;
abstract RS deserialize(DataInputPlus in, int version, R[] keys) throws IOException;
@Override
public long serializedSize(RS routables, int version)
{
int count = routables.size();
long size = TypeSizes.sizeofUnsignedVInt(count);
if (count == 0)
return size;
Object prefix = routables.get(0).prefix();
int prefixStart = 0;
for (int i = 1 ; i <= count ; ++i)
{
Object nextPrefix = null;
if (i < count)
{
nextPrefix = routables.get(i).prefix();
if (Objects.equals(prefix, nextPrefix))
continue;
}
size += TypeSizes.sizeofUnsignedVInt(count - i);
size += serializedSizeOfPrefix(prefix);
int fixedLength = fixedKeyLengthForPrefix(prefix);
if (fixedLength < 0)
size += 4L * (i - prefixStart);
size += serializedSizeOfKeysWithoutPrefix(routables, prefixStart, i);
prefixStart = i;
prefix = nextPrefix;
}
return size;
}
@Override
public void serialize(RS ranges, DataOutputPlus out, int version) throws IOException
public void serialize(RS keys, DataOutputPlus out, int version) throws IOException
{
out.writeUnsignedVInt32(ranges.size());
for (int i=0, mi=ranges.size(); i<mi; i++)
rangeSerializer.serialize(ranges.get(i), out, version);
int size = keys.size();
out.writeUnsignedVInt32(size);
if (size == 0)
return;
Object prefix = keys.get(0).prefix();
int prefixStart = 0;
for (int i = 1 ; i <= size ; ++i)
{
Object nextPrefix = null;
if (i < size)
{
nextPrefix = keys.get(i).prefix();
if (Objects.equals(prefix, nextPrefix))
continue;
}
out.writeUnsignedVInt32(size - i);
serializePrefix(prefix, out, version);
int fixedLength = fixedKeyLengthForPrefix(prefix);
if (fixedLength < 0)
serializeOffsets(keys, prefixStart, i, out);
serializeKeysWithoutPrefix(keys, prefixStart, i, out, version);
prefixStart = i;
prefix = nextPrefix;
}
}
private long serializedSizeOfKeysWithoutPrefix(RS keys, int start, int end)
{
long size = 0;
for (int i = start; i < end; ++i)
size += serializedSizeWithoutPrefix(keys.get(i));
return size;
}
private void serializeKeysWithoutPrefix(RS keys, int start, int end, DataOutputPlus out, int version) throws IOException
{
for (int i = start; i < end; ++i)
serializeWithoutPrefixOrLength(keys.get(i), out, version);
}
public void skip(DataInputPlus in, int version) throws IOException
{
int count = in.readUnsignedVInt32();
for (int i = 0; i < count ; i++)
rangeSerializer.deserialize(in, version);
int remaining = in.readUnsignedVInt32();
if (remaining == 0)
return;
while (remaining > 0)
{
int count = remaining - in.readUnsignedVInt32();
remaining -= count;
Object prefix = deserializePrefix(in, version);
int fixedLength = fixedKeyLengthForPrefix(prefix);
if (fixedLength >= 0)
{
in.skipBytesFully(count * fixedLength);
}
else
{
in.skipBytesFully(4 * (recordCountToLengthCount(count) - 1));
int end = in.readInt();
in.skipBytesFully(end);
}
}
}
@Override
public RS deserialize(DataInputPlus in, int version) throws IOException
{
Range[] ranges = new Range[in.readUnsignedVInt32()];
for (int i=0; i<ranges.length; i++)
ranges[i] = rangeSerializer.deserialize(in, version);
return deserialize(in, version, ranges);
int remaining = in.readUnsignedVInt32();
R[] out = allocate.apply(remaining);
int outCount = 0;
while (remaining > 0)
{
int count = remaining - in.readUnsignedVInt32();
remaining -= count;
Object prefix = deserializePrefix(in, version);
int fixedLength = fixedKeyLengthForPrefix(prefix);
if (fixedLength >= 0)
{
for (int i = 0 ; i < count ; ++i)
out[outCount++] = deserializeWithPrefix(prefix, fixedLength, in, version);
}
else
{
int lengthCount = recordCountToLengthCount(count);
if (lengthCount == 1)
{
int end = in.readInt();
out[outCount++] = deserializeWithPrefix(prefix, end, in, version);
}
else
{
int[] lengths = cachedInts().getInts(lengthCount);
int prev = 0;
for (int i = 0 ; i < lengthCount ; ++i)
{
int end = in.readInt();
lengths[i] = end - prev;
prev = end;
}
for (int i = 0 ; i < count ; ++i)
out[outCount++] = deserializeWithPrefix(prefix, i, lengths, in, version);
cachedInts().forceDiscard(lengths);
}
}
}
return deserialize(in, version, out);
}
}
// this serializer is designed to permits using the collection in its serialized form with minimal in-memory state.
// it also saves some memory by avoiding duplicating prefixes (which happens to also assist faster lookups)
public abstract static class AbstractSearchableKeysSerializer<K extends RoutableKey, KS extends AbstractKeys<K>> extends AbstractSearchableSerializer<K, K, KS> implements IVersionedSerializer<KS>
{
public AbstractSearchableKeysSerializer(AccordSearchableKeySerializer<K> keySerializer, IntFunction<K[]> allocate)
{
super(keySerializer, allocate);
}
abstract RS deserialize(DataInputPlus in, int version, Range[] ranges) throws IOException;
@Override
final int fixedKeyLengthForPrefix(Object prefix)
{
return keySerializer.fixedKeyLengthForPrefix(prefix);
}
@Override
public long serializedSize(RS ranges, int version)
final int recordCountToLengthCount(int recordCount)
{
long size = TypeSizes.sizeofUnsignedVInt(ranges.size());
for (int i=0, mi=ranges.size(); i<mi; i++)
size += TokenRange.serializer.serializedSize((TokenRange) ranges.get(i), version);
return size;
return recordCount;
}
@Override
final int serializedSizeWithoutPrefix(K routable)
{
return keySerializer.serializedSizeWithoutPrefix(routable);
}
@Override
final void serializeWithoutPrefixOrLength(K routable, DataOutputPlus out, int version) throws IOException
{
keySerializer.serializeWithoutPrefixOrLength(routable, out, version);
}
@Override
final void serializeOffsets(KS keys, int startIndex, int endIndex, DataOutputPlus out) throws IOException
{
int endOffset = 0;
for (int i = startIndex; i < endIndex; ++i)
{
endOffset += serializedSizeWithoutPrefix(keys.get(i));
out.writeInt(endOffset);
}
}
@Override
final K deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException
{
return keySerializer.deserializeWithPrefix(prefix, length, in, version);
}
@Override
final K deserializeWithPrefix(Object prefix, int lengthIndex, int[] lengths, DataInputPlus in, int version) throws IOException
{
return keySerializer.deserializeWithPrefix(prefix, lengths[lengthIndex], in, version);
}
}
public abstract static class AbstractRangesSerializer<RS extends AbstractRanges> extends AbstractSearchableSerializer<RoutingKey, Range, RS> implements IVersionedSerializer<RS>
{
public AbstractRangesSerializer(AccordSearchableKeySerializer<RoutingKey> keySerializer)
{
super(keySerializer, Range[]::new);
}
@Override
int fixedKeyLengthForPrefix(Object prefix)
{
return keySerializer.fixedKeyLengthForPrefix(prefix) * 2;
}
@Override
int recordCountToLengthCount(int recordCount)
{
return recordCount * 2;
}
@Override
final int serializedSizeWithoutPrefix(Range range)
{
return keySerializer.serializedSizeWithoutPrefix(range.start())
+ keySerializer.serializedSizeWithoutPrefix(range.end());
}
@Override
final void serializeWithoutPrefixOrLength(Range key, DataOutputPlus out, int version) throws IOException
{
keySerializer.serializeWithoutPrefixOrLength(key.start(), out, version);
keySerializer.serializeWithoutPrefixOrLength(key.end(), out, version);
}
@Override
final void serializeOffsets(RS ranges, int startIndex, int endIndex, DataOutputPlus out) throws IOException
{
int endOffset = 0;
for (int i = startIndex; i < endIndex; ++i)
{
Range r = ranges.get(i);
endOffset += keySerializer.serializedSizeWithoutPrefix(r.start());
out.writeInt(endOffset);
endOffset += keySerializer.serializedSizeWithoutPrefix(r.end());
out.writeInt(endOffset);
}
}
@Override
final Range deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException
{
RoutingKey start = keySerializer.deserializeWithPrefix(prefix, length/2, in, version);
RoutingKey end = keySerializer.deserializeWithPrefix(prefix, length/2, in, version);
return start.rangeFactory().newRange(start, end);
}
@Override
final Range deserializeWithPrefix(Object prefix, int lengthIndex, int[] lengths, DataInputPlus in, int version) throws IOException
{
RoutingKey start = keySerializer.deserializeWithPrefix(prefix, lengths[lengthIndex * 2], in, version);
RoutingKey end = keySerializer.deserializeWithPrefix(prefix, lengths[lengthIndex * 2 + 1], in, version);
return start.rangeFactory().newRange(start, end);
}
}
@ -616,8 +888,8 @@ public class KeySerializers
TreeMap<ByteBuffer, ByteBuffer> result = new TreeMap<>();
for (Range range : ranges)
{
result.put(AccordRoutingKey.serializer.serialize((AccordRoutingKey) range.start()),
AccordRoutingKey.serializer.serialize((AccordRoutingKey) range.end()));
result.put(TokenKey.serializer.serialize((TokenKey) range.start()),
TokenKey.serializer.serialize((TokenKey) range.end()));
}
return result;
}
@ -628,8 +900,8 @@ public class KeySerializers
Range[] ranges = new Range[blobMap.size()];
for (Map.Entry<ByteBuffer, ByteBuffer> e : blobMap.entrySet())
{
ranges[i++] = TokenRange.create(AccordRoutingKey.serializer.deserialize(e.getKey()),
AccordRoutingKey.serializer.deserialize(e.getValue()));
ranges[i++] = TokenRange.create(TokenKey.serializer.deserialize(e.getKey()),
TokenKey.serializer.deserialize(e.getValue()));
}
return Ranges.of(ranges);
}

7
src/java/org/apache/cassandra/service/accord/serializers/PreacceptSerializers.java
View File

@ -21,6 +21,7 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import javax.annotation.Nullable;
import accord.coordinate.ExecuteFlag.ExecuteFlags;
import accord.messages.PreAccept;
import accord.messages.PreAccept.PreAcceptOk;
import accord.messages.PreAccept.PreAcceptReply;
@ -96,6 +97,7 @@ public class PreacceptSerializers
CommandSerializers.txnId.serialize(preAcceptOk.txnId, out, version);
ExecuteAtSerializer.serialize(preAcceptOk.txnId, preAcceptOk.witnessedAt, out);
DepsSerializers.deps.serialize(preAcceptOk.deps, out, version);
out.writeUnsignedVInt32(preAcceptOk.flags.bits());
}
@Override
@ -107,7 +109,8 @@ public class PreacceptSerializers
TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
return new PreAcceptOk(txnId,
ExecuteAtSerializer.deserialize(txnId, in),
DepsSerializers.deps.deserialize(in, version));
DepsSerializers.deps.deserialize(in, version),
ExecuteFlags.get(in.readUnsignedVInt32()));
}
@Override
@ -121,7 +124,7 @@ public class PreacceptSerializers
size += CommandSerializers.txnId.serializedSize(preAcceptOk.txnId, version);
size += ExecuteAtSerializer.serializedSize(preAcceptOk.txnId, preAcceptOk.witnessedAt);
size += DepsSerializers.deps.serializedSize(preAcceptOk.deps, version);
size += TypeSizes.sizeofUnsignedVInt(preAcceptOk.flags.bits());
return size;
}
};

58
src/java/org/apache/cassandra/service/accord/serializers/WaitingOnSerializer.java
View File

@ -20,12 +20,14 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import accord.impl.CommandChange;
import accord.impl.CommandChange.WaitingOnProvider;
import accord.local.Command;
import accord.local.Command.WaitingOn;
import accord.primitives.KeyDeps;
import accord.primitives.PartialDeps;
import accord.primitives.RangeDeps;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.ImmutableBitSet;
import accord.utils.Invariants;
@ -56,24 +58,22 @@ public class WaitingOnSerializer
}
}
public static CommandChange.WaitingOnProvider deserializeProvider(TxnId txnId, DataInputPlus in) throws IOException
public static final class Provider implements WaitingOnProvider
{
ImmutableBitSet waitingOn, appliedOrInvalidated;
final ImmutableBitSet waitingOn, appliedOrInvalidated;
final int waitingOnLength, appliedOrInvalidatedLength;
public Provider(ImmutableBitSet waitingOn, ImmutableBitSet appliedOrInvalidated, int waitingOnLength, int appliedOrInvalidatedLength)
{
int waitingOnLength = in.readUnsignedVInt32();
waitingOn = deserialize(waitingOnLength, in);
if (txnId.is(Range))
{
int appliedOrInvalidatedLength = waitingOnLength - in.readUnsignedVInt32();
appliedOrInvalidated = deserialize(appliedOrInvalidatedLength, in);
}
else
{
appliedOrInvalidated = null;
}
this.waitingOn = waitingOn;
this.appliedOrInvalidated = appliedOrInvalidated;
this.waitingOnLength = waitingOnLength;
this.appliedOrInvalidatedLength = appliedOrInvalidatedLength;
}
return (id, deps, executeAtLeast, uniqueHlc) -> {
@Override
public WaitingOn provide(TxnId txnId, PartialDeps deps, Timestamp executeAtLeast, long uniqueHlc)
{
RoutingKeys keys = deps.keyDeps.keys();
RangeDeps directRangeDeps = deps.rangeDeps;
KeyDeps directKeyDeps = deps.directKeyDeps;
@ -85,7 +85,33 @@ public class WaitingOnSerializer
if (executeAtLeast != null) return new Command.WaitingOnWithExecuteAt(result, executeAtLeast);
else if (uniqueHlc != 0) return new Command.WaitingOnWithMinUniqueHlc(result, uniqueHlc);
return result;
};
}
public void reserialize(DataOutputPlus out, int version) throws IOException
{
out.writeUnsignedVInt32(waitingOnLength);
serialize(waitingOnLength, waitingOn, out);
if (appliedOrInvalidated != null)
{
out.writeUnsignedVInt32(waitingOnLength - appliedOrInvalidatedLength);
serialize(appliedOrInvalidatedLength, appliedOrInvalidated, out);
}
}
}
public static WaitingOnProvider deserializeProvider(TxnId txnId, DataInputPlus in) throws IOException
{
ImmutableBitSet waitingOn, appliedOrInvalidated = null;
int waitingOnLength, appliedOrInvalidatedLength = 0;
waitingOnLength = in.readUnsignedVInt32();
waitingOn = deserialize(waitingOnLength, in);
if (txnId.is(Range))
{
appliedOrInvalidatedLength = waitingOnLength - in.readUnsignedVInt32();
appliedOrInvalidated = deserialize(appliedOrInvalidatedLength, in);
}
return new Provider(waitingOn, appliedOrInvalidated, waitingOnLength, appliedOrInvalidatedLength);
}
public static void skip(TxnId txnId, DataInputPlus in) throws IOException

31
src/java/org/apache/cassandra/service/accord/txn/TxnNamedRead.java
View File

@ -60,10 +60,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordObjectSizes;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.txn.TxnData.TxnDataNameKind;
@ -109,25 +106,25 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
boolean startIsMinKeyBound = startPP.getClass() == KeyBound.class ? ((KeyBound)startPP).isMinimumBound : false;
Token startToken = startPP.getToken();
Token stopToken = range.right.getToken();
AccordRoutingKey startAccordRoutingKey;
TokenKey startTokenKey;
if (startToken.isMinimum() && inclusiveLeft)
startAccordRoutingKey = SentinelKey.min(tableId);
startTokenKey = TokenKey.min(tableId, startToken.getPartitioner());
else if (inclusiveLeft || startIsMinKeyBound || startToken.equals(stopToken))
startAccordRoutingKey = new MinTokenKey(tableId, startToken);
startTokenKey = TokenKey.before(tableId, startToken);
else
startAccordRoutingKey = new TokenKey(tableId, startToken);
startTokenKey = new TokenKey(tableId, startToken);
boolean inclusiveRight = range.inclusiveRight();
PartitionPosition endPP = range.right;
boolean endIsMinKeyBound = endPP.getClass() == KeyBound.class ? ((KeyBound)endPP).isMinimumBound : false;
AccordRoutingKey stopAccordRoutingKey;
TokenKey stopTokenKey;
if (stopToken.isMinimum())
stopAccordRoutingKey = SentinelKey.max(tableId);
stopTokenKey = TokenKey.max(tableId, startToken.getPartitioner());
else if (inclusiveRight && !endIsMinKeyBound)
stopAccordRoutingKey = new TokenKey(tableId, stopToken);
stopTokenKey = new TokenKey(tableId, stopToken);
else
stopAccordRoutingKey = new MinTokenKey(tableId, stopToken);
return TokenRange.create(startAccordRoutingKey, stopAccordRoutingKey);
stopTokenKey = TokenKey.before(tableId, stopToken);
return TokenRange.create(startTokenKey, stopTokenKey);
}
public TxnNamedRead(int name, AbstractBounds<PartitionPosition> range, PartitionRangeReadCommand value)
@ -326,10 +323,10 @@ public class TxnNamedRead extends AbstractSerialized<ReadCommand>
AbstractBounds<PartitionPosition> bounds = command.dataRange().keyRange();
PartitionPosition startPP = bounds.left;
PartitionPosition endPP = bounds.right;
AccordRoutingKey startRoutingKey = ((AccordRoutingKey)r.start());
AccordRoutingKey endRoutingKey = ((AccordRoutingKey)r.end());
Token subRangeStartToken = startRoutingKey.getClass() == SentinelKey.class ? startPP.getToken() : ((AccordRoutingKey)r.start()).asTokenKey().token();
Token subRangeEndToken = endRoutingKey.getClass() == SentinelKey.class ? endPP.getToken() : ((AccordRoutingKey)r.end()).asTokenKey().token();
TokenKey startRoutingKey = ((TokenKey)r.start());
TokenKey endRoutingKey = ((TokenKey)r.end());
Token subRangeStartToken = startRoutingKey.isMin() ? startPP.getToken() : startRoutingKey.token();
Token subRangeEndToken = endRoutingKey.isMax() ? endPP.getToken() : endRoutingKey.token();
/*
* The way ranges/bounds work for range queries is that the beginning and ending bounds from the command

15
src/java/org/apache/cassandra/service/consensus/migration/ConsensusRequestRouter.java
View File

@ -57,9 +57,6 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState.KeyMigrationState;
import org.apache.cassandra.service.paxos.Paxos;
@ -323,14 +320,14 @@ public class ConsensusRequestRouter
// = token ends up as a min and max key bound in C* parlance and min and max token key in Accord parlance
// and the conversion to a C* range results in the unintentional creation of a wrap around range.
// Instead treat it like a key and do that check.
if (range.start().getClass() == MinTokenKey.class
&& range.end() instanceof TokenKey
if (range.start().isTokenSentinel()
&& !range.end().isSentinel()
&& range.start().token().equals(range.end().token()))
{
checkState(range.end().getClass() != MinTokenKey.class, "Unexpected empty range");
checkState(!range.end().isTokenSentinel(), "Unexpected empty range");
return isTokenManagedByAccordForReadAndWrite(metadata, tms, range.start().token());
}
else if (range.start().getClass() == MinTokenKey.class)
else if (range.start().isTokenSentinel())
{
// Start is particularly problematic because we use min MinTokenKey to make start inclusive and this is something
// that isn't possible to mimic at all with Range<Token>, for end it's less problematic because just the token
@ -339,9 +336,9 @@ public class ConsensusRequestRouter
// and use the bounds check
PartitionPosition startPP = range.start().token().minKeyBound();
PartitionPosition endPP;
if (range.end().getClass() == SentinelKey.class)
if (range.end().isTableSentinel())
endPP = DatabaseDescriptor.getPartitioner().getMinimumToken().maxKeyBound();
else if (range.end().getClass() == MinTokenKey.class)
else if (range.end().isTokenSentinel())
endPP = range.end().token().minKeyBound();
else
endPP = range.end().token().maxKeyBound();

19
src/java/org/apache/cassandra/utils/ByteBufferUtil.java
View File

@ -219,6 +219,25 @@ public class ByteBufferUtil
return bytes;
}
/**
* You should almost never use this. Instead, use the write* methods to avoid copies.
*/
public static byte[] getArrayUnsafe(ByteBuffer buffer)
{
return getArrayUnsafe(buffer, buffer.position(), buffer.remaining());
}
/**
* You should almost never use this. Instead, use the write* methods to avoid copies.
*/
public static byte[] getArrayUnsafe(ByteBuffer buffer, int position, int length)
{
if (buffer.hasArray() && position == 0 && buffer.arrayOffset() == 0 && length == buffer.capacity())
return buffer.array();
return getArray(buffer, position, length);
}
/**
* ByteBuffer adaptation of org.apache.commons.lang3.ArrayUtils.lastIndexOf method
*

1
test/distributed/org/apache/cassandra/distributed/test/accord/AccordCQLTestBase.java
View File

@ -2846,6 +2846,7 @@ public abstract class AccordCQLTestBase extends AccordTestBase
" UPDATE " + qualifiedAccordTableName + "1 SET v = row2.v WHERE k=1 AND c=2;\n" +
" END IF\n" +
"COMMIT TRANSACTION";
Object[][] result = SHARED_CLUSTER.coordinator(1).execute(query, ConsistencyLevel.ANY);
assertEquals(3, result[0][0]);

4
test/distributed/org/apache/cassandra/distributed/test/accord/AccordDropTableBase.java
View File

@ -45,6 +45,8 @@ import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.TokenRange;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
public class AccordDropTableBase extends TestBaseImpl
{
protected static void addChaos(Cluster cluster, int example)
@ -128,7 +130,7 @@ public class AccordDropTableBase extends TestBaseImpl
inst.runOnInstance(() -> {
TableId tableId = TableId.fromUUID(UUID.fromString(s));
AccordService accord = (AccordService) AccordService.instance();
PreLoadContext ctx = PreLoadContext.contextFor(Ranges.single(TokenRange.fullRange(tableId)), KeyHistory.SYNC);
PreLoadContext ctx = PreLoadContext.contextFor(Ranges.single(TokenRange.fullRange(tableId, getPartitioner())), KeyHistory.SYNC);
CommandStores stores = accord.node().commandStores();
for (int storeId : stores.ids())
{

2
test/distributed/org/apache/cassandra/distributed/test/accord/AccordIncrementalRepairTest.java
View File

@ -62,7 +62,7 @@ import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.DelegatingAccordService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.ByteBufferUtil;

2
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java
View File

@ -74,7 +74,7 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairResult;
import org.apache.cassandra.service.consensus.migration.ConsensusTableMigration;

5
test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationWriteRaceTestBase.java
View File

@ -91,8 +91,7 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.service.consensus.migration.ConsensusKeyMigrationState;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationRepairResult;
@ -476,7 +475,7 @@ public abstract class AccordMigrationWriteRaceTestBase extends AccordTestBase
if (route.domain() == Domain.Key)
for (RoutingKey key : (KeyRoute)route)
{
AccordRoutingKey routingKey = (AccordRoutingKey)key;
TokenKey routingKey = (TokenKey)key;
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(routingKey.table());
if (cfs.getKeyspaceName().equals(KEYSPACE))
return true;

4
test/distributed/org/apache/cassandra/distributed/test/accord/AccordTestBase.java
View File

@ -95,7 +95,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.exceptions.ReadPreemptedException;
import org.apache.cassandra.service.accord.exceptions.WritePreemptedException;
import org.apache.cassandra.service.consensus.TransactionalMode;
@ -671,7 +671,7 @@ public abstract class AccordTestBase extends TestBaseImpl
if (route.domain() == Domain.Key)
for (RoutingKey key : (KeyRoute)route)
{
AccordRoutingKey routingKey = (AccordRoutingKey)key;
TokenKey routingKey = (TokenKey)key;
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(routingKey.table());
if (cfs.getKeyspaceName().equals(KEYSPACE))
return true;

102
test/distributed/org/apache/cassandra/service/accord/BurnTestKeySerializers.java
View File

@ -45,7 +45,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.api.AccordRoutableKey;
import org.apache.cassandra.service.accord.api.AccordRoutableKey.AccordKeySerializer;
import org.apache.cassandra.service.accord.api.AccordRoutableKey.AccordSearchableKeySerializer;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.serializers.TopologySerializers;
@ -58,8 +58,8 @@ public class BurnTestKeySerializers
public static final AccordRoutableKey.AccordKeySerializer<Key> key =
(AccordRoutableKey.AccordKeySerializer<Key>)
(AccordRoutableKey.AccordKeySerializer<?>)
new AccordRoutableKey.AccordKeySerializer<PrefixedIntHashKey>()
(AccordSearchableKeySerializer<?>)
new AccordSearchableKeySerializer<PrefixedIntHashKey>()
{
public void serialize(PrefixedIntHashKey t, DataOutputPlus out, int version) throws IOException
{
@ -86,12 +86,57 @@ public class BurnTestKeySerializers
{
in.skipBytesFully(3 * Integer.BYTES);
}
@Override
public int fixedKeyLengthForPrefix(Object prefix)
{
return 8;
}
@Override
public int serializedSizeOfPrefix(Object prefix)
{
return 4;
}
@Override
public int serializedSizeWithoutPrefix(PrefixedIntHashKey key)
{
return 8;
}
@Override
public void serializePrefix(Object prefix, DataOutputPlus out, int version) throws IOException
{
out.writeInt((Integer) prefix);
}
@Override
public void serializeWithoutPrefixOrLength(PrefixedIntHashKey key, DataOutputPlus out, int version) throws IOException
{
out.writeInt(key.hash);
out.writeInt(key.key);
}
@Override
public Object deserializePrefix(DataInputPlus in, int version) throws IOException
{
return in.readInt();
}
@Override
public PrefixedIntHashKey deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException
{
int key = in.readInt();
int hash = in.readInt();
return PrefixedIntHashKey.key((Integer)prefix, key, hash);
}
};
public static final IVersionedSerializer<RoutingKey> routingKey =
(AccordKeySerializer<RoutingKey>)
(AccordKeySerializer<?>)
new AccordRoutableKey.AccordKeySerializer<PrefixedIntHashKey.Hash>()
public static final AccordSearchableKeySerializer<RoutingKey> routingKey =
(AccordSearchableKeySerializer<RoutingKey>)
(AccordSearchableKeySerializer<?>)
new AccordSearchableKeySerializer<PrefixedIntHashKey.Hash>()
{
public void serialize(PrefixedIntHashKey.Hash t, DataOutputPlus out, int version) throws IOException
{
@ -115,6 +160,49 @@ public class BurnTestKeySerializers
{
in.skipBytesFully(2 * Integer.BYTES);
}
@Override
public int fixedKeyLengthForPrefix(Object prefix)
{
return 4;
}
@Override
public int serializedSizeOfPrefix(Object prefix)
{
return 4;
}
@Override
public int serializedSizeWithoutPrefix(PrefixedIntHashKey.Hash key)
{
return 4;
}
@Override
public void serializePrefix(Object prefix, DataOutputPlus out, int version) throws IOException
{
out.writeInt((Integer) prefix);
}
@Override
public void serializeWithoutPrefixOrLength(PrefixedIntHashKey.Hash key, DataOutputPlus out, int version) throws IOException
{
out.writeInt(key.hash);
}
@Override
public Object deserializePrefix(DataInputPlus in, int version) throws IOException
{
return in.readInt();
}
@Override
public PrefixedIntHashKey.Hash deserializeWithPrefix(Object prefix, int length, DataInputPlus in, int version) throws IOException
{
int hash = in.readInt();
return PrefixedIntHashKey.forHash((Integer)prefix, hash);
}
};
public static final IVersionedSerializer<Range> range =

7
test/harry/main/org/apache/cassandra/harry/gen/Generators.java
View File

@ -135,6 +135,11 @@ public class Generators
}
public static Generator<ByteBuffer> bytes(int minSize, int maxSize)
{
return byteArrays(minSize, maxSize).map(ByteBuffer::wrap);
}
public static Generator<byte[]> byteArrays(int minSize, int maxSize)
{
return rng -> {
int size = rng.nextInt(minSize, maxSize);
@ -144,7 +149,7 @@ public class Generators
n = Math.min(size - i, Long.SIZE / Byte.SIZE);
n-- > 0; v >>= Byte.SIZE)
bytes[i++] = (byte) v;
return ByteBuffer.wrap(bytes);
return bytes;
};
}

23
test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java
View File

@ -41,7 +41,6 @@ import accord.api.Result;
import accord.local.CheckedCommands;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.RedundantBefore;
import accord.local.StoreParticipants;
@ -61,7 +60,6 @@ import accord.primitives.Txn;
import accord.primitives.Txn.Kind;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.cql3.QueryProcessor;
@ -86,7 +84,7 @@ import org.apache.cassandra.service.accord.AccordExecutor;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@ -99,7 +97,7 @@ import static org.apache.cassandra.Util.spinAssertEquals;
import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
import static org.apache.cassandra.service.accord.AccordKeyspace.COMMANDS_FOR_KEY;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeysAccessor;
import static org.apache.cassandra.service.accord.AccordKeyspace.CFKAccessor;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@ -174,7 +172,7 @@ public class CompactionAccordIteratorsTest
private void testAccordCommandsForKeyPurger(boolean singleCompaction) throws Throwable
{
this.singleCompaction = singleCompaction;
testAccordCommandsForKeyPurger(null, expectedAccordCommandsForKeyNoChange());
testAccordCommandsForKeyPurger(RedundantBefore.EMPTY, expectedAccordCommandsForKeyNoChange());
testAccordCommandsForKeyPurger(redundantBefore(LT_TXN_ID), expectedAccordCommandsForKeyNoChange());
// will erase one more than expected as converted to ExclusiveSyncPoint id which is > base id
testAccordCommandsForKeyPurger(redundantBefore(TXN_ID), expectedAccordCommandsForKeyEraseOne());
@ -188,7 +186,7 @@ public class CompactionAccordIteratorsTest
assertEquals(1, partitions.size());
Partition partition = partitions.get(0);
TokenKey partitionKey = new TokenKey(partition.metadata().id, partition.partitionKey().getToken());
CommandsForKey cfk = CommandsForKeysAccessor.getCommandsForKey(partitionKey, ((Row) partition.unfilteredIterator().next()));
CommandsForKey cfk = CFKAccessor.fromRow(partitionKey, ((Row) partition.unfilteredIterator().next()));
assertEquals(TXN_IDS.length, cfk.size());
for (int i = 0; i < TXN_IDS.length; ++i)
assertEquals(TXN_IDS[i], cfk.txnId(i));
@ -262,15 +260,10 @@ public class CompactionAccordIteratorsTest
private static IAccordService mockAccordService(CommandStore commandStore, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
IAccordService mockAccordService = mock(IAccordService.class);
Int2ObjectHashMap<RedundantBefore> redundantBefores = new Int2ObjectHashMap<>();
if (redundantBefore != null)
redundantBefores.put(commandStore.id(), redundantBefore);
Int2ObjectHashMap<DurableBefore> durableBefores = new Int2ObjectHashMap<>();
if (durableBefore != null)
durableBefores.put(commandStore.id(), durableBefore);
Int2ObjectHashMap<CommandStores.RangesForEpoch> rangesForEpochs = new Int2ObjectHashMap<>();
rangesForEpochs.put(commandStore.id(), commandStore.unsafeGetRangesForEpoch());
when(mockAccordService.getCompactionInfo()).thenReturn(new IAccordService.CompactionInfo(redundantBefores, rangesForEpochs, durableBefores));
IAccordService.AccordCompactionInfo compactionInfo = new IAccordService.AccordCompactionInfo(commandStore.id(), redundantBefore, commandStore.unsafeGetRangesForEpoch(), ((AccordCommandStore)commandStore).tableId());
IAccordService.AccordCompactionInfos compactionInfos = new IAccordService.AccordCompactionInfos(durableBefore);
compactionInfos.put(commandStore.id(), compactionInfo);
when(mockAccordService.getCompactionInfo()).thenReturn(compactionInfos);
return mockAccordService;
}

7
test/unit/org/apache/cassandra/db/virtual/AccordVirtualTablesTest.java
View File

@ -46,6 +46,7 @@ import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.TokenRange;
import org.mockito.Mockito;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
public class AccordVirtualTablesTest extends CQLTester
@ -126,13 +127,13 @@ public class AccordVirtualTablesTest extends CQLTester
row(e1, T1_META.keyspace, T1_META.name, FULL_RANGE, List.of(), List.of(), List.of(), FULL_RANGE));
// lets close e2
tm.onEpochClosed(Ranges.single(TokenRange.fullRange(T1)), e2);
tm.onEpochClosed(Ranges.single(TokenRange.fullRange(T1, getPartitioner())), e2);
assertRows(execute("SELECT * FROM " + VIRTUAL_VIEWS + "." + AccordVirtualTables.TABLE_EPOCHS),
row(e2, T1_META.keyspace, T1_META.name, List.of(), FULL_RANGE, List.of(), List.of(), FULL_RANGE),
row(e1, T1_META.keyspace, T1_META.name, FULL_RANGE, FULL_RANGE, List.of(), List.of(), FULL_RANGE));
// enjoy retirement!
tm.onEpochRetired(Ranges.single(TokenRange.fullRange(T1)), e2);
tm.onEpochRetired(Ranges.single(TokenRange.fullRange(T1, getPartitioner())), e2);
assertRows(execute("SELECT * FROM " + VIRTUAL_VIEWS + "." + AccordVirtualTables.TABLE_EPOCHS),
row(e2, T1_META.keyspace, T1_META.name, List.of(), FULL_RANGE, List.of(), FULL_RANGE, FULL_RANGE),
row(e1, T1_META.keyspace, T1_META.name, FULL_RANGE, FULL_RANGE, List.of(), FULL_RANGE, FULL_RANGE));
@ -145,7 +146,7 @@ public class AccordVirtualTablesTest extends CQLTester
private static Topology topology(long epoch, TableId tableId)
{
TokenRange all = TokenRange.fullRange(tableId);
TokenRange all = TokenRange.fullRange(tableId, getPartitioner());
return new Topology(epoch, Shard.create(all, ALL, FP));
}

20
test/unit/org/apache/cassandra/dht/AccordSplitterTest.java
View File

@ -31,7 +31,7 @@ import accord.primitives.Ranges;
import accord.utils.Gens;
import accord.utils.RandomSource;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.AccordGenerators;
import org.assertj.core.api.Assertions;
@ -50,8 +50,8 @@ public class AccordSplitterTest
public void split()
{
qt().forAll(AccordGenerators.range(), Gens.random()).check((range, rs) -> {
AccordRoutingKey startKey = (AccordRoutingKey) range.start();
AccordRoutingKey endKey = (AccordRoutingKey) range.end();
TokenKey startKey = (TokenKey) range.start();
TokenKey endKey = (TokenKey) range.end();
IPartitioner partitioner = getPartitioner(range, rs);
// this section is filtering out known bugs
// TODO (now): fix the fact accordSplitter returns AccordBytesSplitter which will fail for java.lang.ClassCastException: org.apache.cassandra.dht.LocalPartitioner$LocalToken cannot be cast to org.apache.cassandra.dht.ByteOrderedPartitioner$BytesToken
@ -59,13 +59,13 @@ public class AccordSplitterTest
if (partitioner instanceof LocalPartitioner)
return;
// TODO (now): java.lang.AssertionError: [size is not larger than 0 for partitioner org.apache.cassandra.dht.OrderPreservingPartitioner@54a67a45]
if (partitioner instanceof OrderPreservingPartitioner && endKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
if (partitioner instanceof OrderPreservingPartitioner && endKey.isTableSentinel())
return;
// TODO (now): [size is not larger than 0 for partitioner org.apache.cassandra.dht.ByteOrderedPartitioner@44e3a2b2]
if (partitioner instanceof ByteOrderedPartitioner && endKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
if (partitioner instanceof ByteOrderedPartitioner && endKey.isTableSentinel())
return;
// TODO (now): [num splits not as expected for partitioner org.apache.cassandra.dht.ByteOrderedPartitioner@4c550889]\nExpected size to be between: <47> and <48> but was:<62> in:
if (partitioner instanceof ByteOrderedPartitioner && startKey.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
if (partitioner instanceof ByteOrderedPartitioner && startKey.isTableSentinel())
return;
// TODO (now): [num splits not as expected for partitioner org.apache.cassandra.dht.ByteOrderedPartitioner@13518f37]\nExpected size to be between: <11> and <12> but was:<13> in:
if (partitioner instanceof ByteOrderedPartitioner)
@ -121,10 +121,10 @@ public class AccordSplitterTest
private static IPartitioner getPartitioner(Range range, RandomSource rs)
{
AccordRoutingKey key = (AccordRoutingKey) range.start();
if (key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
key = (AccordRoutingKey) range.end();
if (key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL)
TokenKey key = (TokenKey) range.start();
if (key.isTableSentinel())
key = (TokenKey) range.end();
if (key.isTableSentinel())
return AccordGenerators.partitioner().next(rs);
return key.token().getPartitioner();

115
test/unit/org/apache/cassandra/dht/LocalCompositePrefixPartitionerTest.java
View File

@ -1,115 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.dht;
import com.google.common.collect.Lists;
import org.apache.cassandra.CassandraTestBase;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.TableMetadata;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static java.lang.String.format;
import static org.apache.cassandra.cql3.QueryProcessor.executeOnceInternal;
import static org.apache.cassandra.utils.ByteBufferUtil.hexToBytes;
public class LocalCompositePrefixPartitionerTest extends CassandraTestBase
{
private static final String KEYSPACE = "ks";
@BeforeClass
public static void setupClass()
{
SchemaLoader.prepareServer();
}
private static TableMetadata.Builder parse(String keyspace, String name, String cql)
{
return CreateTableStatement.parse(format(cql, name), KEYSPACE)
.gcGraceSeconds((int) TimeUnit.DAYS.toSeconds(90));
}
private static LocalCompositePrefixPartitioner partitioner(AbstractType... types)
{
return new LocalCompositePrefixPartitioner(types);
}
private static void assertKeysMatch(Iterable<DecoratedKey> expected, Iterator<DecoratedKey> actual)
{
List<DecoratedKey> expectedList = Lists.newArrayList(expected);
List<DecoratedKey> actualList = Lists.newArrayList(actual);
Assert.assertEquals(expectedList, actualList);
}
@Test
public void keyIteratorTest() throws Throwable
{
String keyspaceName = "ks";
String tableName = "tbl";
LocalCompositePrefixPartitioner partitioner = partitioner(Int32Type.instance, BytesType.instance, Int32Type.instance);
TableMetadata metadata = parse(keyspaceName, tableName,
"CREATE TABLE %s (" +
"p1 int," +
"p2 blob," +
"p3 int," +
"v int," +
"PRIMARY KEY ((p1, p2, p3))" +
")").partitioner(partitioner).build();
SchemaLoader.createKeyspace(keyspaceName, KeyspaceParams.local(), metadata);
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (1, 0x00, 5, 0)", keyspaceName, tableName));
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (1, 0x0000, 5, 0)", keyspaceName, tableName));
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (2, 0x00, 5, 0)", keyspaceName, tableName));
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (2, 0x0100, 5, 0)", keyspaceName, tableName));
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (2, 0x02, 5, 0)", keyspaceName, tableName));
executeOnceInternal(String.format("INSERT INTO %s.%s (p1, p2, p3, v) VALUES (2, 0x02, 6, 0)", keyspaceName, tableName));
Token startToken = partitioner.createPrefixToken(1, hexToBytes("0000"));
Token endToken1 = partitioner.createPrefixToken(2, hexToBytes("0100"));
Token endToken2 = partitioner.createPrefixToken(2, hexToBytes("02"));
assertKeysMatch(List.of(partitioner.decoratedKey(2, hexToBytes("00"), 5),
partitioner.decoratedKey(2, hexToBytes("0100"), 5)
), partitioner.keyIterator(metadata, new Range<>(startToken.maxKeyBound(), endToken1.maxKeyBound())));
assertKeysMatch(List.of(partitioner.decoratedKey(1, hexToBytes("0000"), 5),
partitioner.decoratedKey(2, hexToBytes("00"), 5),
partitioner.decoratedKey(2, hexToBytes("0100"), 5),
partitioner.decoratedKey(2, hexToBytes("02"), 5),
partitioner.decoratedKey(2, hexToBytes("02"), 6)
), partitioner.keyIterator(metadata, new Bounds<>(startToken.minKeyBound(), endToken2.maxKeyBound())));
assertKeysMatch(List.of(partitioner.decoratedKey(1, hexToBytes("0000"), 5),
partitioner.decoratedKey(2, hexToBytes("00"), 5),
partitioner.decoratedKey(2, hexToBytes("0100"), 5)
), partitioner.keyIterator(metadata, new IncludingExcludingBounds<>(startToken.minKeyBound(), endToken2.minKeyBound())));
}
}

2
test/unit/org/apache/cassandra/dht/PartitionerTestCase.java
View File

@ -35,7 +35,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import static org.apache.cassandra.service.accord.AccordTestUtils.TABLE_ID1;
import static org.junit.Assert.assertEquals;

57
test/unit/org/apache/cassandra/dht/RandomPartitionerTest.java
View File

@ -19,9 +19,13 @@
package org.apache.cassandra.dht;
import java.math.BigInteger;
import java.util.Arrays;
import org.junit.Test;
import org.apache.cassandra.cql3.functions.types.utils.Bytes;
import org.apache.cassandra.harry.checker.TestHelper;
public class RandomPartitionerTest extends PartitionerTestCase
{
public void initPartitioner()
@ -55,4 +59,57 @@ public class RandomPartitionerTest extends PartitionerTestCase
assertSplit(left, tok("a"), 16);
}
@Test
public void testIncrement()
{
TestHelper.withRandom((rng) -> {
for (int i = 0; i < 10_000; i++)
{
BigInteger bi = BigInteger.valueOf(Math.abs(rng.next()));
byte[] bytes = bi.toByteArray();
byte[] copy = Arrays.copyOf(bytes, bytes.length);
BigInteger incremented = new BigInteger(RandomPartitioner.increment(bytes));
BigInteger expected = bi.add(BigInteger.valueOf(1));
if (!expected.equals(incremented))
{
throw new IllegalArgumentException(String.format("\nBefore increment: %s" +
"\n After increment: %s," +
"\n%s != %s",
Bytes.toHexString(copy),
Bytes.toHexString(bytes),
expected,
incremented));
}
}
});
}
@Test
public void testDecrement()
{
TestHelper.withRandom((rng) -> {
for (int i = 0; i < 10_000; i++)
{
BigInteger bi = BigInteger.valueOf(Math.abs(rng.next() + 1));
byte[] bytes = bi.toByteArray();
byte[] copy = Arrays.copyOf(bytes, bytes.length);
RandomPartitioner.decrement(bytes);
BigInteger incremented = new BigInteger(bytes);
BigInteger expected = bi.add(BigInteger.valueOf(-1));
if (!expected.equals(incremented))
{
throw new IllegalArgumentException(String.format("\nBefore increment: %s" +
"\n After increment: %s," +
"\n%s != %s",
Bytes.toHexString(copy),
Bytes.toHexString(bytes),
expected,
incremented));
}
}
});
}
}

2
test/unit/org/apache/cassandra/index/accord/CheckpointIntervalArrayIndexTest.java
View File

@ -71,7 +71,7 @@ public class CheckpointIntervalArrayIndexTest
static
{
DatabaseDescriptor.clientInitialization();
DatabaseDescriptor.toolInitialization();
}
private static final byte[] EMPTY = new byte[0];

50
test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
View File

@ -86,9 +86,9 @@ import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.AccordTopology;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.IAccordService.AccordCompactionInfo;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
@ -177,8 +177,8 @@ public class RouteIndexTest extends CQLTester.InMemory
TokenRange range = selectExistingRange(rs, ranges);
// have a key, so find a key within the range
long start = range.start().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL ? Long.MIN_VALUE : ((LongToken) range.start().token()).token;
long end = range.end().kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL ? Long.MAX_VALUE : ((LongToken) range.end().token()).token;
long start = range.start().isMin() ? Long.MIN_VALUE : ((LongToken) range.start().token()).token;
long end = range.end().isMax() ? Long.MAX_VALUE : ((LongToken) range.end().token()).token;
long token = 1 + rs.nextLong(start, end);
return new KeySearch(storeId, new TokenKey(tableId, new LongToken(token)));
}
@ -265,9 +265,9 @@ public class RouteIndexTest extends CQLTester.InMemory
private static class KeySearch implements Command<State, Sut, Set<TxnId>>
{
private final int storeId;
private final AccordRoutingKey key;
private final TokenKey key;
private KeySearch(int storeId, AccordRoutingKey key)
private KeySearch(int storeId, TokenKey key)
{
this.storeId = storeId;
this.key = key;
@ -448,7 +448,7 @@ public class RouteIndexTest extends CQLTester.InMemory
private static class State implements AutoCloseable
{
private final Int2ObjectHashMap<Map<TableId, Long2ObjectHashMap<List<TxnId>>>> storeToTableToRoutingKeysToTxns = new Int2ObjectHashMap<>();
private final Int2ObjectHashMap<Map<TableId, RangeTree<AccordRoutingKey, TokenRange, TxnId>>> storeToTableToRangesToTxns = new Int2ObjectHashMap<>();
private final Int2ObjectHashMap<Map<TableId, RangeTree<TokenKey, TokenRange, TxnId>>> storeToTableToRangesToTxns = new Int2ObjectHashMap<>();
private final int numStores;
private final List<TableId> tables;
@ -484,7 +484,7 @@ public class RouteIndexTest extends CQLTester.InMemory
// the reason for the mocking is to speed up compaction. Collecting the info from the stores has been slow and its always empty in this test... so stub it out to speed up the test
AccordService mock = Mockito.spy(as);
Mockito.doReturn(emptyCompactionInfo()).when(mock).getCompactionInfo();
Mockito.doReturn(emptyCompactionInfo(tableId)).when(mock).getCompactionInfo();
AccordService.unsafeSetNewAccordService(mock);
AccordService.replayJournal(as);
@ -504,7 +504,7 @@ public class RouteIndexTest extends CQLTester.InMemory
{
case Key:
{
AccordRoutingKey key = (AccordRoutingKey) u;
TokenKey key = (TokenKey) u;
var table = key.table();
var token = key.token().getLongValue();
storeToTableToRoutingKeysToTxns.computeIfAbsent(storeId, ignore -> new HashMap<>())
@ -596,53 +596,47 @@ public class RouteIndexTest extends CQLTester.InMemory
}
}
private static RangeTree<AccordRoutingKey, TokenRange, TxnId> rangeTree()
private static RangeTree<TokenKey, TokenRange, TxnId> rangeTree()
{
return RTree.create(ACCESSOR);
}
private static final RangeTree.Accessor<AccordRoutingKey, TokenRange> ACCESSOR = new RangeTree.Accessor<>()
private static final RangeTree.Accessor<TokenKey, TokenRange> ACCESSOR = new RangeTree.Accessor<>()
{
@Override
public AccordRoutingKey start(TokenRange tokenRange)
public TokenKey start(TokenRange tokenRange)
{
return tokenRange.start();
}
@Override
public AccordRoutingKey end(TokenRange tokenRange)
public TokenKey end(TokenRange tokenRange)
{
return tokenRange.end();
}
@Override
public boolean contains(AccordRoutingKey start, AccordRoutingKey end, AccordRoutingKey accordRoutingKey)
public boolean contains(TokenKey start, TokenKey end, TokenKey tokenKey)
{
return TokenRange.create(start, end).contains(accordRoutingKey);
return TokenRange.create(start, end).contains(tokenKey);
}
@Override
public boolean intersects(TokenRange tokenRange, AccordRoutingKey start, AccordRoutingKey end)
public boolean intersects(TokenRange tokenRange, TokenKey start, TokenKey end)
{
return tokenRange.compareIntersecting(TokenRange.create(start, end)) == 0;
}
};
private static IAccordService.CompactionInfo emptyCompactionInfo()
private static IAccordService.AccordCompactionInfos emptyCompactionInfo(TableId tableId)
{
Int2ObjectHashMap<RedundantBefore> redundantBefores = new Int2ObjectHashMap<>();
Int2ObjectHashMap<DurableBefore> durableBefores = new Int2ObjectHashMap<>();
Int2ObjectHashMap<CommandStores.RangesForEpoch> ranges = new Int2ObjectHashMap<>();
IAccordService.AccordCompactionInfos compactionInfos = new IAccordService.AccordCompactionInfos(DurableBefore.EMPTY);
RedundantBefore redundantBefore = Mockito.spy(RedundantBefore.EMPTY);
Mockito.doReturn(RedundantStatus.NONE).when(redundantBefore).status(Mockito.any(), Mockito.any(), (Participants<?>) Mockito.any());
Mockito.doReturn(RedundantStatus.NONE).when(redundantBefore).status(Mockito.any(), Mockito.any(), (RoutingKey) Mockito.any());
for (int i = 0; i < MAX_STORES; i++)
{
redundantBefores.put(i, redundantBefore);
durableBefores.put(i, DurableBefore.EMPTY);
ranges.put(i, new CommandStores.RangesForEpoch(1, Ranges.EMPTY));
}
return new IAccordService.CompactionInfo(redundantBefores, ranges, durableBefores);
compactionInfos.put(i, new AccordCompactionInfo(i, redundantBefore, new CommandStores.RangesForEpoch(1, Ranges.EMPTY), tableId));
return compactionInfos;
}
private static ColumnFamilyStore cfs()
@ -707,7 +701,7 @@ public class RouteIndexTest extends CQLTester.InMemory
{
case Key:
{
TreeSet<AccordRoutingKey> keys = new TreeSet<>();
TreeSet<TokenKey> keys = new TreeSet<>();
while (keys.size() < numKeys)
{
var table = rs.pick(state.tables);
@ -728,7 +722,7 @@ public class RouteIndexTest extends CQLTester.InMemory
}
}
private static TokenRange selectExistingRange(RandomSource rs, RangeTree<AccordRoutingKey, TokenRange, TxnId> ranges)
private static TokenRange selectExistingRange(RandomSource rs, RangeTree<TokenKey, TokenRange, TxnId> ranges)
{
TreeSet<TokenRange> distinctRanges = ranges.stream().map(Map.Entry::getKey).collect(Collectors.toCollection(() -> new TreeSet<>(TokenRange::compareTo)));
TokenRange range;

7
test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
View File

@ -59,7 +59,8 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializerTest.TestSafeCommandStore;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
@ -166,9 +167,9 @@ public class AccordCommandStoreTest
cfk.set(cfk.current().update(new TestSafeCommandStore(command1.txnId()), command1).cfk());
cfk.set(cfk.current().update(new TestSafeCommandStore(command1.txnId()), command2).cfk());
AccordKeyspace.getCommandsForKeyUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run();
CommandsForKeyAccessor.systemTableUpdater(commandStore.id(), (TokenKey)cfk.key(), cfk.current(), null, commandStore.nextSystemTimestampMicros()).run();
logger.info("E: {}", cfk);
CommandsForKey actual = AccordKeyspace.loadCommandsForKey(commandStore.id(), key);
CommandsForKey actual = CommandsForKeyAccessor.load(commandStore.id(), key);
logger.info("A: {}", actual);
Assert.assertEquals(cfk.current(), actual);

4
test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java
View File

@ -50,7 +50,7 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.utils.StorageCompatibilityMode;
@ -87,7 +87,7 @@ public class AccordJournalOrderTest
TxnId txnId = randomSource.nextBoolean() ? id1 : id2;
JournalKey key = new JournalKey(txnId, JournalKey.Type.COMMAND_DIFF, randomSource.nextInt(5));
res.compute(key, (k, prev) -> prev == null ? 1 : prev + 1);
Participants<?> participants = RoutingKeys.of(new AccordRoutingKey.TokenKey(TableId.generate(), new ByteOrderedPartitioner.BytesToken(new byte[1])));
Participants<?> participants = RoutingKeys.of(new TokenKey(TableId.generate(), new ByteOrderedPartitioner.BytesToken(new byte[1])));
Command command = Command.NotDefined.notDefined(txnId, SaveStatus.NotDefined, Status.Durability.NotDurable, StoreParticipants.create(null, participants, null, participants, participants), Ballot.ZERO);
accordJournal.saveCommand(key.commandStoreId,
new Journal.CommandUpdate(null, command),

44
test/unit/org/apache/cassandra/service/accord/AccordKeyspaceTest.java
View File

@ -29,6 +29,7 @@ import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import com.google.common.collect.Iterables;
import org.junit.Test;
import accord.api.RoutingKey;
@ -62,8 +63,8 @@ import org.apache.cassandra.schema.MemtableParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.CassandraGenerators;
import org.assertj.core.api.Assertions;
import org.mockito.Mockito;
@ -71,10 +72,13 @@ import org.mockito.stubbing.Answer;
import static accord.local.Command.Committed.committed;
import static accord.utils.Property.qt;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
import static org.apache.cassandra.config.DatabaseDescriptor.setSelectedSSTableFormat;
import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.setMemtable;
import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.findAllKeysBetween;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor.makeSystemTableKeyBytes;
import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
import static org.apache.cassandra.utils.AbstractTypeGenerators.getTypeSupport;
import static org.apache.cassandra.utils.AccordGenerators.fromQT;
@ -95,7 +99,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
String tableName = createTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode = 'full'");
TableId tableId = Schema.instance.getTableMetadata(KEYSPACE, tableName).id;
Ranges scope = Ranges.of(TokenRange.create(AccordRoutingKey.SentinelKey.min(tableId), AccordRoutingKey.SentinelKey.max(tableId)));
Ranges scope = Ranges.of(TokenRange.create(TokenKey.min(tableId, getPartitioner()), TokenKey.max(tableId, getPartitioner())));
AccordCommandStore store = AccordTestUtils.createAccordCommandStore(now::incrementAndGet, KEYSPACE, tableName);
@ -127,7 +131,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
public void findOverlappingKeys()
{
var tableIdGen = fromQT(CassandraGenerators.TABLE_ID_GEN);
var partitionGen = fromQT(CassandraGenerators.partitioners());
var partitionGen = fromQT(CassandraGenerators.partitioners().assuming(IPartitioner::accordSupported));
var sstableFormats = DatabaseDescriptor.getSSTableFormats();
List<String> sstableFormatNames = new ArrayList<>(sstableFormats.keySet());
@ -148,7 +152,8 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
// define the tables w/ partitioners for the test
// this uses the ability to override the SchemaProvider for the keyspace and only defines the single API call expected: getTablePartitioner
TreeMap<TableId, IPartitioner> tables = new TreeMap<>();
int numTables = rs.nextInt(1, 3);
int numStores = rs.nextInt(1, 3);
int numTables = numStores == 1 ? 1 : rs.nextInt(1, numStores);
for (int i = 0; i < numTables; i++)
{
var tableId = tableIdGen.next(rs);
@ -156,11 +161,16 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
tableId = tableIdGen.next(rs);
tables.put(tableId, partitionGen.next(rs));
}
TreeMap<Integer, TableId> storeTableIds = new TreeMap<>();
for (int i = 0; i < numStores; i++)
{
int tableIdx = rs.nextInt(tables.size());
TableId tableId = Iterables.get(tables.keySet(), tableIdx);
storeTableIds.put(i, tableId);
}
SchemaProvider schema = Mockito.mock(SchemaProvider.class);
Mockito.when(schema.getTablePartitioner(Mockito.any())).thenAnswer((Answer<IPartitioner>) invocationOnMock -> tables.get(invocationOnMock.getArgument(0)));
AccordKeyspace.unsafeSetSchema(schema);
int numStores = rs.nextInt(1, 3);
// The model of the DB
TreeMap<Integer, SortedSet<TokenKey>> storesToKeys = new TreeMap<>();
@ -174,7 +184,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
// else this will loop forever...
for (int attempt = 0; attempt < 10; attempt++)
{
TableId tableId = rs.pickOrderedSet(tables.navigableKeySet());
TableId tableId = storeTableIds.get(store);
IPartitioner partitioner = tables.get(tableId);
ByteBuffer data;
if (partitioner instanceof ReversedLongLocalPartitioner)
@ -196,10 +206,8 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
{
// using Mutation directly (what we do in Accord) can break when user data is too large; leading to data loss
// The memtable will allow the write, but it will be dropped when writing to the SSTable...
//TODO (now, correctness): since we store the user token + user key, if a key is close to the PK limits then we could tip over and loose our CFK
// new Mutation(AccordKeyspace.getCommandsForKeyPartitionUpdate(store, pk, 42, ByteBufferUtil.EMPTY_BYTE_BUFFER)).apply();
execute("INSERT INTO system_accord.commands_for_key (store_id, table_id, key_token) VALUES (?, ?, ?)",
store, pk.table().asUUID(), AccordKeyspace.serializeRoutingKeyNoTable(pk));
execute("INSERT INTO system_accord.commands_for_key (key) VALUES (?)",
makeSystemTableKeyBytes(store, pk));
}
catch (IllegalArgumentException | InvalidRequestException e)
{
@ -236,17 +244,17 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
SortedSet<TokenKey> keys = e.getValue();
if (keys.isEmpty())
continue;
expectedCqlStoresToKeys.put(store, new TreeSet<>(keys.stream().map(AccordKeyspace::serializeRoutingKeyNoTable).collect(Collectors.toList())));
expectedCqlStoresToKeys.put(store, new TreeSet<>(keys.stream().map(key -> makeSystemTableKeyBytes(store, key)).collect(Collectors.toList())));
}
// make sure no data loss... when this test was written sstable had all the rows but the sstable didn't... this
// is mostly a santity check to detect that case early
var resultSet = execute("SELECT store_id, table_id, key_token FROM system_accord.commands_for_key ALLOW FILTERING");
var resultSet = execute("SELECT key FROM system_accord.commands_for_key ALLOW FILTERING");
TreeMap<Integer, SortedSet<ByteBuffer>> cqlStoresToKeys = new TreeMap<>();
for (var row : resultSet)
{
int storeId = row.getInt("store_id");
ByteBuffer bb = row.getBytes("key_token");
ByteBuffer bb = row.getBytes("key");
int storeId = CommandsForKeyAccessor.getCommandStoreId(bb);
cqlStoresToKeys.computeIfAbsent(storeId, ignore -> new TreeSet<>()).add(bb);
}
Assertions.assertThat(cqlStoresToKeys).isEqualTo(expectedCqlStoresToKeys);
@ -276,7 +284,7 @@ public class AccordKeyspaceTest extends CQLTester.InMemory
TokenKey end = expected.get(expected.size() - 1);
List<TokenKey> actual = new ArrayList<>();
AccordKeyspace.findAllKeysBetween(store, start, true, end, true, actual::add);
findAllKeysBetween(store, storeTableIds.get(store), start.token().getPartitioner(), start, true, end, true, actual::add);
Assertions.assertThat(actual).isEqualTo(expected);
}

6
test/unit/org/apache/cassandra/service/accord/AccordStaleReplicasTest.java
View File

@ -27,6 +27,7 @@ import accord.local.Node;
import accord.utils.AccordGens;
import accord.utils.Gen;
import accord.utils.Gens;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializers;
@ -36,6 +37,11 @@ import static accord.utils.Property.qt;
public class AccordStaleReplicasTest
{
static
{
DatabaseDescriptor.toolInitialization();
}
@Test
public void serde()
{

5
test/unit/org/apache/cassandra/service/accord/AccordTaskTest.java
View File

@ -76,7 +76,8 @@ import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
import org.apache.cassandra.service.accord.AccordExecutor.ExclusiveGlobalCaches;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.utils.AssertionUtils;
import org.apache.cassandra.utils.FBUtilities;
@ -160,7 +161,7 @@ public class AccordTaskTest
}));
long nowInSeconds = FBUtilities.nowInSeconds();
SinglePartitionReadCommand command = AccordKeyspace.getCommandsForKeyRead(commandStore.id(), key, (int) nowInSeconds);
SinglePartitionReadCommand command = CommandsForKeyAccessor.makeRead(commandStore.id(), key, (int) nowInSeconds);
try(ReadExecutionController controller = command.executionController();
FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
{

24
test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
View File

@ -313,7 +313,7 @@ public class AccordTestUtils
public static Ranges fullRange(Seekables<?, ?> keys)
{
PartitionKey key = (PartitionKey) keys.get(0);
return Ranges.of(TokenRange.fullRange(key.table()));
return Ranges.of(TokenRange.fullRange(key.table(), DatabaseDescriptor.getPartitioner()));
}
public static PartialTxn createPartialTxn(int key)
@ -323,21 +323,6 @@ public class AccordTestUtils
return new PartialTxn.InMemory(txn.kind(), txn.keys(), txn.read(), txn.query(), txn.update());
}
private static class SingleEpochRanges extends CommandStore.EpochUpdateHolder
{
private final Ranges ranges;
public SingleEpochRanges(Ranges ranges)
{
this.ranges = ranges;
}
private void set()
{
add(1, new CommandStores.RangesForEpoch(1, ranges), ranges);
}
}
public static AccordCommandStore createAccordCommandStore(
Node.Id node, LongSupplier now, Topology topology, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor)
{
@ -372,12 +357,13 @@ public class AccordTestUtils
AccordJournal journal = new AccordJournal(spec, agent);
journal.start(null);
SingleEpochRanges holder = new SingleEpochRanges(topology.rangesForNode(node));
CommandStore.EpochUpdateHolder holder = new CommandStore.EpochUpdateHolder();
Ranges ranges = topology.rangesForNode(node);
holder.add(1, new CommandStores.RangesForEpoch(1, ranges), ranges);
AccordCommandStore result = new AccordCommandStore(0, time, agent, null,
cs -> new NoOpProgressLog(),
cs -> new DefaultLocalListeners(new NoOpRemoteListeners(), new NoOpNotifySink()),
holder, journal, executor);
holder.set();
result.unsafeUpdateRangesForEpoch();
return result;
}
@ -391,7 +377,7 @@ public class AccordTestUtils
LongSupplier now, String keyspace, String table, ExecutorPlus loadExecutor, ExecutorPlus saveExecutor)
{
TableMetadata metadata = Schema.instance.getTableMetadata(keyspace, table);
TokenRange range = TokenRange.fullRange(metadata.id);
TokenRange range = TokenRange.fullRange(metadata.id, metadata.partitioner);
Node.Id node = new Id(1);
Topology topology = new Topology(1, Shard.create(range, new SortedArrayList<>(new Id[] { node }), Sets.newHashSet(node), Collections.emptySet()));
AccordCommandStore store = createAccordCommandStore(node, now, topology, loadExecutor, saveExecutor);

12
test/unit/org/apache/cassandra/service/accord/AccordTopologyTest.java
View File

@ -84,7 +84,7 @@ public class AccordTopologyTest
Topology topology = AccordTopology.createAccordTopology(metadata);
Topology expected = new Topology(1,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, NODE_SET));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, NODE_SET));
Assert.assertEquals(expected, topology);
}
@ -99,7 +99,7 @@ public class AccordTopologyTest
ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
Topology topology = AccordTopology.createAccordTopology(metadata);
Topology expected = new Topology(1,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, NODE_SET));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, NODE_SET));
Assert.assertEquals(expected, topology);
}
@ -113,7 +113,7 @@ public class AccordTopologyTest
ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
Topology topology = AccordTopology.createAccordTopology(metadata);
Topology expected = new Topology(1,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, NODE_SET));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, NODE_SET));
Assert.assertEquals(expected, topology);
topology = AccordTopology.createAccordTopology(metadata.transformer().withFastPathStatusSince(new Id(1), AccordFastPath.Status.UNAVAILABLE, 1, 1).build().metadata);
@ -122,7 +122,7 @@ public class AccordTopologyTest
fastPath.remove(new Node.Id(1));
expected = new Topology(2,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, fastPath));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, fastPath));
Assert.assertEquals(expected, topology);
}
@ -138,7 +138,7 @@ public class AccordTopologyTest
ClusterMetadata metadata = configureCluster(ranges, Keyspaces.of(keyspace));
Topology topology = AccordTopology.createAccordTopology(metadata);
Topology expected = new Topology(1,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, NODE_SET));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, NODE_SET));
Assert.assertEquals(expected, topology);
metadata = metadata.transformer()
@ -151,7 +151,7 @@ public class AccordTopologyTest
fastPath.remove(new Node.Id(1));
expected = new Topology(2,
Shard.create(AccordTopology.fullRange(tableId), NODE_LIST, fastPath));
Shard.create(AccordTopology.fullRange(tableId, partitioner), NODE_LIST, fastPath));
Assert.assertEquals(expected, topology);
}
}

2
test/unit/org/apache/cassandra/service/accord/SimpleSimulatedAccordCommandStoreTest.java
View File

@ -35,7 +35,7 @@ public class SimpleSimulatedAccordCommandStoreTest extends SimulatedAccordComman
{
qt().withExamples(10).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(reverseTokenTbl.id, rs))
{
for (int i = 0, examples = 100; i < examples; i++)
{

35
test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
View File

@ -30,6 +30,8 @@ import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.ToLongFunction;
import javax.annotation.Nullable;
import accord.api.Agent;
import accord.api.Journal;
import accord.api.LocalListeners;
@ -87,12 +89,15 @@ import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.memtable.Memtable;
import org.apache.cassandra.metrics.AccordCacheMetrics;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Generators;
import org.apache.cassandra.utils.Pair;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS;
import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
import static org.apache.cassandra.utils.AccordGenerators.fromQT;
@ -125,10 +130,20 @@ public class SimulatedAccordCommandStore implements AutoCloseable
public SimulatedAccordCommandStore(RandomSource rs)
{
this(rs, FunctionWrapper.identity());
this(null, rs, FunctionWrapper.identity());
}
public SimulatedAccordCommandStore(TableId tableId, RandomSource rs)
{
this(tableId, rs, FunctionWrapper.identity());
}
public SimulatedAccordCommandStore(RandomSource rs, FunctionWrapper loadFunctionWrapper)
{
this(null, rs, loadFunctionWrapper);
}
public SimulatedAccordCommandStore(@Nullable TableId tableId, RandomSource rs, FunctionWrapper loadFunctionWrapper)
{
globalExecutor = new SimulatedExecutorFactory(rs.fork(), fromQT(Generators.TIMESTAMP_GEN.map(java.sql.Timestamp::getTime)).mapToLong(TimeUnit.MILLISECONDS::toNanos).next(rs), failures::add);
this.unorderedScheduled = globalExecutor.scheduled("ignored");
@ -138,8 +153,16 @@ public class SimulatedAccordCommandStore implements AutoCloseable
for (Stage stage : Arrays.asList(Stage.MISC, Stage.ACCORD_MIGRATION, Stage.READ, Stage.MUTATION))
stage.unsafeSetExecutor(globalExecutor.configureSequential("ignore").build());
this.updateHolder = new CommandStore.EpochUpdateHolder();
this.nodeId = AccordTopology.tcmIdToAccord(ClusterMetadata.currentNullable().myNodeId());
this.updateHolder = new CommandStore.EpochUpdateHolder();
this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
Ranges ranges = topology.ranges();
if (tableId != null)
ranges = ranges.slice(Ranges.of(TokenRange.create(TokenKey.min(tableId, getPartitioner()), TokenKey.max(tableId, getPartitioner()))));
CommandStores.RangesForEpoch rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), ranges);
updateHolder.add(topology.epoch(), rangesForEpoch, ranges);
this.storeService = new NodeCommandStoreService()
{
private final ToLongFunction<TimeUnit> elapsed = TimeService.elapsedWrapperFromNonMonotonicSource(TimeUnit.NANOSECONDS, this::now);
@ -229,10 +252,6 @@ public class SimulatedAccordCommandStore implements AutoCloseable
commandStore.executor().setCapacity(8 << 20);
commandStore.executor().setWorkingSetSize(4 << 20);
});
this.topology = AccordTopology.createAccordTopology(ClusterMetadata.current());
this.topologies = new Topologies.Single(SizeOfIntersectionSorter.SUPPLIER, topology);
CommandStores.RangesForEpoch rangesForEpoch = new CommandStores.RangesForEpoch(topology.epoch(), topology.ranges());
updateHolder.add(topology.epoch(), rangesForEpoch, topology.ranges());
commandStore.unsafeUpdateRangesForEpoch();
shouldEvict = boolSource(rs.fork());
@ -342,7 +361,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
}
}
private static boolean intersects(ColumnFamilyStore store, Memtable memtable, Unseekables<RoutingKey> keys, Ranges ranges)
private boolean intersects(ColumnFamilyStore store, Memtable memtable, Unseekables<RoutingKey> keys, Ranges ranges)
{
if (keys.isEmpty() && ranges.isEmpty()) // shouldn't happen, but just in case...
return false;
@ -355,7 +374,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
{
while (it.hasNext())
{
var key = AccordKeyspace.CommandsForKeysAccessor.getKey(it.next().partitionKey());
var key = AccordKeyspace.CommandsForKeyAccessor.getUserTableKey(commandStore.tableId(), it.next().partitionKey());
if (keys.contains(key) || ranges.intersects(key))
return true;
}

21
test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStoreTestBase.java
View File

@ -63,11 +63,12 @@ import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.Pair;
@ -141,19 +142,19 @@ public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester
ServerTestUtils.markCMS();
}
protected static TokenRange fullRange(TableId id)
protected static TokenRange fullRange(TableId id, IPartitioner partitioner)
{
return TokenRange.create(AccordRoutingKey.SentinelKey.min(id), AccordRoutingKey.SentinelKey.max(id));
return TokenRange.create(TokenKey.min(id, partitioner), TokenKey.max(id, partitioner));
}
protected static TokenRange tokenRange(TableId id, long start, long end)
protected static TokenRange tokenRange(TableId id, IPartitioner partitioner, long start, long end)
{
return TokenRange.create(start == Long.MIN_VALUE ? AccordRoutingKey.SentinelKey.min(id) : tokenKey(id, start), tokenKey(id, end));
return TokenRange.create(start == Long.MIN_VALUE ? TokenKey.min(id, partitioner) : tokenKey(id, start), tokenKey(id, end));
}
protected static AccordRoutingKey.TokenKey tokenKey(TableId id, long token)
protected static TokenKey tokenKey(TableId id, long token)
{
return new AccordRoutingKey.TokenKey(id, new Murmur3Partitioner.LongToken(token));
return new TokenKey(id, new Murmur3Partitioner.LongToken(token));
}
protected static Map<RoutingKey, List<TxnId>> keyConflicts(List<TxnId> list, Unseekables<RoutingKey> keys)
@ -348,9 +349,9 @@ public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester
}
}
protected static Gen<Pair<Txn, FullRoute<?>>> randomTxn(Gen<Routable.Domain> domainGen, Gen.LongGen tokenGen)
protected static Gen<Pair<Txn, FullRoute<?>>> randomTxn(TableMetadata tbl, Gen<Routable.Domain> domainGen, Gen.LongGen tokenGen)
{
TableMetadata tbl = reverseTokenTbl;
Invariants.require(tbl == reverseTokenTbl);
Invariants.requireArgument(tbl.partitioner == Murmur3Partitioner.instance, "Only murmur partitioner is supported; given %s", tbl.partitioner.getClass());
Gen<PartitionKey> keyGen = rs -> new PartitionKey(tbl.id, tbl.partitioner.decorateKey(Murmur3Partitioner.LongToken.keyForToken(tokenGen.nextLong(rs))));
Gen<Range> rangeGen = rs -> {
@ -364,7 +365,7 @@ public abstract class SimulatedAccordCommandStoreTestBase extends CQLTester
a = b;
b = tmp;
}
return tokenRange(tbl.id, a, b);
return tokenRange(tbl.id, tbl.partitioner, a, b);
};
return rs -> {
Routable.Domain domain = domainGen.next(rs);

10
test/unit/org/apache/cassandra/service/accord/SimulatedAccordTaskTest.java
View File

@ -49,7 +49,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.SimulatedAccordCommandStore.FunctionWrapper;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.Pair;
import org.assertj.core.api.Assertions;
@ -67,7 +67,7 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase
@Test
public void happyPath()
{
qt().withExamples(100).check(rs -> test(rs, 100, intTbl, ignore -> Action.SUCCESS, ignore -> 0L));
qt().withExamples(100).check(rs -> test(rs, 100, reverseTokenTbl, ignore -> Action.SUCCESS, ignore -> 0L));
}
@Test
@ -75,7 +75,7 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase
{
Gen<Action> actionGen = Gens.enums().allWithWeights(Action.class, 10, 1, 1);
Gen.LongGen delaysNanos = Gens.longs().between(0, TimeUnit.MILLISECONDS.toNanos(10));
qt().withExamples(100).check(rs -> test(rs, 100, intTbl, actionGen, delaysNanos));
qt().withExamples(100).check(rs -> test(rs, 100, reverseTokenTbl, actionGen, delaysNanos));
}
enum Operation { Task, PreAccept }
@ -93,9 +93,9 @@ public class SimulatedAccordTaskTest extends SimulatedAccordCommandStoreTestBase
Gen<RoutingKeys> keysGen = Gens.lists(keyGen).unique().ofSizeBetween(1, 10).map(l -> RoutingKeys.of(l));
Gen<Ranges> rangesGen = Gens.lists(rangeInsideRange(tbl.id, minToken, maxToken)).uniqueBestEffort().ofSizeBetween(1, 10).map(l -> Ranges.of(l.toArray(Range[]::new)));
Gen<Unseekables<?>> unseekablesGen = Gens.oneOf(keysGen, rangesGen);
Gen<Pair<Txn, FullRoute<?>>> txnGen = randomTxn(mixedDomainGen.next(rs), mixedTokenGen.next(rs));
Gen<Pair<Txn, FullRoute<?>>> txnGen = randomTxn(tbl, mixedDomainGen.next(rs), mixedTokenGen.next(rs));
try (var instance = new SimulatedAccordCommandStore(rs, new SimulatedLoadFunctionWrapper(actionGen.asSupplier(rs), delaysNanos.asLongSupplier(rs))))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs, new SimulatedLoadFunctionWrapper(actionGen.asSupplier(rs), delaysNanos.asLongSupplier(rs))))
{
instance.ignoreExceptions = t -> t instanceof SimulatedFault;
Counter counter = new Counter();

26
test/unit/org/apache/cassandra/service/accord/SimulatedDepsTest.java
View File

@ -32,7 +32,7 @@ import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.utils.Generators;
import org.junit.Test;
@ -63,7 +63,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
Keys keys = Keys.of(pk);
FullKeyRoute route = keys.toRoute(pk.toUnseekable());
Txn txn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
List<TxnId> conflicts = new ArrayList<>(numSamples);
for (int i = 0; i < numSamples; i++)
@ -96,7 +96,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
Keys keysTokenConflict = Keys.of(pkTokenConflict);
FullKeyRoute routeTokenConflict = keysTokenConflict.toRoute(pkTokenConflict.toUnseekable());
Txn txnTokenConflict = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(tokenConflictKey, 42));
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
List<TxnId> conflicts = new ArrayList<>(numSamples);
for (int i = 0; i < numSamples; i++)
@ -118,10 +118,10 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
qt().withExamples(10).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
long token = rs.nextLong(Long.MIN_VALUE + 1, Long.MAX_VALUE);
Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - 1, token));
Ranges partialRange = Ranges.of(tokenRange(tbl.id, tbl.partitioner, token - 1, token));
long outOfRangeToken = token - 10;
if (outOfRangeToken == Long.MIN_VALUE) // if this wraps around that is fine, just can't be min
@ -166,12 +166,12 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
public void simpleRangeConflicts()
{
var tbl = reverseTokenTbl;
Ranges wholeRange = Ranges.of(fullRange(tbl.id));
Ranges wholeRange = Ranges.of(fullRange(tbl.id, tbl.partitioner));
int numSamples = 100;
qt().withExamples(10).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
long token = rs.nextLong(Long.MIN_VALUE + 1, Long.MAX_VALUE);
ByteBuffer key = LongToken.keyForToken(token);
@ -180,7 +180,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
FullKeyRoute keyRoute = keys.toRoute(pk.toUnseekable());
Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - 1, token));
Ranges partialRange = Ranges.of(tokenRange(tbl.id, tbl.partitioner, token - 1, token));
boolean useWholeRange = rs.nextBoolean();
Ranges ranges = useWholeRange ? wholeRange : partialRange;
FullRangeRoute rangeRoute = ranges.toRoute(pk.toUnseekable());
@ -205,7 +205,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
qt().withExamples(10).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
long token = rs.nextLong(Long.MIN_VALUE + numSamples + 1, Long.MAX_VALUE - numSamples);
ByteBuffer key = LongToken.keyForToken(token);
@ -218,7 +218,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
Map<Range, List<TxnId>> rangeConflicts = new HashMap<>();
for (int i = 0; i < numSamples; i++)
{
Ranges partialRange = Ranges.of(tokenRange(tbl.id, token - i - 1, token + i));
Ranges partialRange = Ranges.of(tokenRange(tbl.id, tbl.partitioner, token - i - 1, token + i));
FullRangeRoute rangeRoute = partialRange.toRoute(pk.toUnseekable());
Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, partialRange);
try
@ -246,7 +246,7 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
qt().withExamples(10).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
long token = rs.nextLong(Long.MIN_VALUE + numSamples + 1, Long.MAX_VALUE - numSamples);
ByteBuffer key = LongToken.keyForToken(token);
@ -255,8 +255,8 @@ public class SimulatedDepsTest extends SimulatedAccordCommandStoreTestBase
FullKeyRoute keyRoute = keys.toRoute(pk.toUnseekable());
Txn keyTxn = createTxn(wrapInTxn("INSERT INTO " + tbl + "(pk, value) VALUES (?, ?)"), Arrays.asList(key, 42));
Range left = tokenRange(tbl.id, token - 10, token + 5);
Range right = tokenRange(tbl.id, token - 5, token + 10);
Range left = tokenRange(tbl.id, tbl.partitioner, token - 10, token + 5);
Range right = tokenRange(tbl.id, tbl.partitioner, token - 5, token + 10);
DepsModel model = new DepsModel(instance.commandStore.unsafeGetRangesForEpoch().currentRanges());
for (int i = 0; i < numSamples; i++)

4
test/unit/org/apache/cassandra/service/accord/SimulatedMultiKeyAndRangeTest.java
View File

@ -62,7 +62,7 @@ public class SimulatedMultiKeyAndRangeTest extends SimulatedAccordCommandStoreTe
qt().withExamples(100).check(rs -> {
AccordKeyspace.unsafeClear();
try (var instance = new SimulatedAccordCommandStore(rs))
try (var instance = new SimulatedAccordCommandStore(tbl.id, rs))
{
Gen.LongGen tokenGen = tokenDistribution.next(rs);
Gen<Domain> domainGen = domainDistribution.next(rs);
@ -115,7 +115,7 @@ public class SimulatedMultiKeyAndRangeTest extends SimulatedAccordCommandStoreTe
start = token;
end = start + offset;
}
set.add(tokenRange(tbl.id, start, end));
set.add(tokenRange(tbl.id, tbl.partitioner, start, end));
}
// The property ranges.size() == numRanges is not true as this logic will sort + deoverlap
// so if the ranges were overlapped, we could have more or less than numRanges

4
test/unit/org/apache/cassandra/service/accord/SimulatedRandomKeysWithRangeConflictTest.java
View File

@ -78,7 +78,7 @@ public class SimulatedRandomKeysWithRangeConflictTest extends SimulatedAccordCom
final SimulatedAccordCommandStore instance;
final TableMetadata tbl = reverseTokenTbl;
final Ranges wholeRange = Ranges.of(fullRange(tbl.id));
final Ranges wholeRange = Ranges.of(fullRange(tbl.id, tbl.partitioner));
final FullRangeRoute rangeRoute = wholeRange.toRoute(wholeRange.get(0).end());
final Txn rangeTxn = createTxn(Txn.Kind.ExclusiveSyncPoint, wholeRange);
final DepsModel model;
@ -86,7 +86,7 @@ public class SimulatedRandomKeysWithRangeConflictTest extends SimulatedAccordCom
public State(RandomSource rs)
{
AccordKeyspace.unsafeClear();
this.instance = new SimulatedAccordCommandStore(rs);
this.instance = new SimulatedAccordCommandStore(tbl.id, rs);
this.instance.commandStore.executor().cacheUnsafe().setShrinkingOn(false);
this.model = new DepsModel(instance.commandStore.unsafeGetRangesForEpoch().currentRanges());
}

6
test/unit/org/apache/cassandra/service/accord/api/AccordKeyTest.java
View File

@ -28,8 +28,6 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.SentinelKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.SerializerTestUtils;
@ -109,8 +107,8 @@ public class AccordKeyTest
DecoratedKey dk2 = partitioner(TABLE2).decorateKey(ByteBufferUtil.bytes(5));
PartitionKey pk2 = new PartitionKey(TABLE2, dk2);
SentinelKey loSentinel = SentinelKey.min(TABLE1);
SentinelKey hiSentinel = SentinelKey.max(TABLE1);
TokenKey loSentinel = TokenKey.min(TABLE1, partitioner(TABLE1));
TokenKey hiSentinel = TokenKey.max(TABLE1, partitioner(TABLE1));
Assert.assertTrue(loSentinel.compareTo(hiSentinel) < 0);
Assert.assertTrue(pk1.compareTo(loSentinel) > 0);
Assert.assertTrue(loSentinel.compareTo(pk1) < 0);

94
test/unit/org/apache/cassandra/service/accord/serializers/AccordRoutingKeyByteSourceTest.java
View File

@ -1,94 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.ByteArrayAccessor;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.serializers.AccordRoutingKeyByteSource.FixedLength;
import org.apache.cassandra.utils.AccordGenerators;
import org.apache.cassandra.utils.ByteArrayUtil;
import org.apache.cassandra.utils.CassandraGenerators;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
import org.assertj.core.api.Assertions;
import static accord.utils.Property.qt;
import static org.apache.cassandra.utils.AccordGenerators.fromQT;
import static org.apache.cassandra.utils.CassandraGenerators.token;
public class AccordRoutingKeyByteSourceTest
{
static
{
DatabaseDescriptor.clientInitialization();
// AccordRoutingKey$TokenKey reaches into DD to get partitioner, so need to set that up...
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@Test
public void tokenSerde()
{
qt().forAll(fromQT(CassandraGenerators.token())).check(token -> {
var serializer = AccordRoutingKeyByteSource.create(token.getPartitioner());
byte[] minMin = ByteSourceInverse.readBytes(serializer.minMinAsComparableBytes());
byte[] minMax = ByteSourceInverse.readBytes(serializer.minMaxAsComparableBytes());
byte[] maxMin = ByteSourceInverse.readBytes(serializer.maxMinAsComparableBytes());
byte[] maxMax = ByteSourceInverse.readBytes(serializer.maxMaxAsComparableBytes());
var bytes = serializer.serialize(token);
if (serializer instanceof FixedLength)
{
FixedLength fl = (FixedLength) serializer;
Assertions.assertThat(bytes)
.hasSize(fl.valueSize())
.hasSize(minMin.length)
.hasSize(minMax.length)
.hasSize(maxMin.length)
.hasSize(maxMax.length);
}
Assertions.assertThat(ByteArrayUtil.compareUnsigned(minMin, 0, bytes, 0, bytes.length)).isLessThan(0);
Assertions.assertThat(ByteArrayUtil.compareUnsigned(minMax, 0, bytes, 0, bytes.length)).isLessThan(0);
Assertions.assertThat(ByteArrayUtil.compareUnsigned(maxMin, 0, bytes, 0, bytes.length)).isGreaterThan(0);
Assertions.assertThat(ByteArrayUtil.compareUnsigned(maxMax, 0, bytes, 0, bytes.length)).isGreaterThan(0);
var read = serializer.tokenFromComparableBytes(ByteArrayAccessor.instance, bytes);
Assertions.assertThat(read).isEqualTo(token);
});
}
@Test
public void accordRoutingKeySerde()
{
qt().forAll(AccordGenerators.routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(token()))).check(key -> {
AccordRoutingKeyByteSource.Serializer serializer = key.kindOfRoutingKey() == AccordRoutingKey.RoutingKeyKind.SENTINEL ?
// doesn't really matter...
new AccordRoutingKeyByteSource.VariableLength(ByteOrderedPartitioner.instance, ByteComparable.Version.OSS50)
: AccordRoutingKeyByteSource.create(key.asTokenKey().token().getPartitioner());
var read = serializer.fromComparableBytes(ByteArrayAccessor.instance, serializer.serialize(key));
Assertions.assertThat(read).isEqualTo(key);
});
}
}

8
test/unit/org/apache/cassandra/service/accord/serializers/CheckStatusSerializersTest.java
View File

@ -40,7 +40,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.AccordGenerators;
import org.apache.cassandra.utils.CassandraGenerators;
import org.assertj.core.api.Assertions;
@ -86,9 +86,9 @@ public class CheckStatusSerializersTest
{
case Key:
// TODO (coverage): don't hard code murmur
Gen<AccordRoutingKey> keyGen = AccordGenerators.routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), Gens.constant(AccordRoutingKey.RoutingKeyKind.TOKEN), fromQT(CassandraGenerators.murmurToken()));
AccordRoutingKey homeKey = keyGen.next(rs);
List<AccordRoutingKey> forOrdering = Gens.lists(keyGen).unique().ofSizeBetween(1, 10).next(rs);
Gen<TokenKey> keyGen = AccordGenerators.routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), Gens.constant(TokenKey.RoutingKeyKind.TOKEN), fromQT(CassandraGenerators.murmurToken()), Murmur3Partitioner.instance);
TokenKey homeKey = keyGen.next(rs);
List<TokenKey> forOrdering = Gens.lists(keyGen).unique().ofSizeBetween(1, 10).next(rs);
forOrdering.sort(Comparator.naturalOrder());
// TODO (coverage): don't hard code keys type
keysOrRanges = new FullKeyRoute(homeKey, forOrdering.toArray(RoutingKey[]::new));

3
test/unit/org/apache/cassandra/service/accord/serializers/CommandSerializersTest.java
View File

@ -31,6 +31,7 @@ import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.utils.SerializerTestUtils;
import static org.apache.cassandra.config.DatabaseDescriptor.getPartitioner;
import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
public class CommandSerializersTest
@ -55,7 +56,7 @@ public class CommandSerializersTest
" END IF\n" +
"COMMIT TRANSACTION");
PartitionKey key = (PartitionKey) txn.keys().get(0);
PartialTxn expected = txn.slice(Ranges.of(TokenRange.fullRange(key.table())), true);
PartialTxn expected = txn.slice(Ranges.of(TokenRange.fullRange(key.table(), getPartitioner())), true);
SerializerTestUtils.assertSerializerIOEquality(expected, CommandSerializers.partialTxn);
}
}

2
test/unit/org/apache/cassandra/service/accord/serializers/CommandsForKeySerializerTest.java
View File

@ -105,7 +105,7 @@ import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.txn.TxnData;
import org.apache.cassandra.service.accord.txn.TxnWrite;
import org.apache.cassandra.simulator.RandomSource.Choices;

299
test/unit/org/apache/cassandra/service/accord/serializers/TokenKeyTest.java Normal file
View File

@ -0,0 +1,299 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.Test;
import accord.utils.Gen;
import accord.utils.Gens;
import accord.utils.Invariants;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Murmur3Partitioner.LongToken;
import org.apache.cassandra.dht.RandomPartitioner;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.utils.AccordGenerators;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraGenerators;
import org.assertj.core.api.Assertions;
import static accord.utils.Property.qt;
import static org.apache.cassandra.service.accord.api.TokenKey.serializer;
import static org.apache.cassandra.utils.AccordGenerators.fromQT;
import static org.apache.cassandra.utils.CassandraGenerators.partitioners;
import static org.apache.cassandra.utils.CassandraGenerators.token;
public class TokenKeyTest
{
static
{
DatabaseDescriptor.clientInitialization();
// AccordRoutingKey$TokenKey reaches into DD to get partitioner, so need to set that up...
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@Test
public void serde()
{
qt().forAll(fromQT(partitioners().assuming(IPartitioner::accordSupported)).flatMap(partitioner -> routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(token(partitioner)), partitioner)))
.check(key -> {
IPartitioner partitioner = key.token().getPartitioner();
{
ByteBuffer buffer = serializer.serialize(key);
TokenKey roundTrip = serializer.deserialize(buffer, partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
}
{
TokenKey roundTrip = serializer.deserializeWithPrefixAndImpliedLength(key.prefix(), serializer.serializeWithoutPrefixOrLength(key), partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
}
{
TokenKey roundTrip = serializer.deserializeWithPrefixAndImpliedLength(key.prefix(), serializer.serializeWithoutPrefixOrLength(key), ByteBufferAccessor.instance, 0, partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
}
{
TokenKey roundTrip = serializer.deserializeWithPrefix(key.prefix(), serializer.serializedSizeWithoutPrefix(key), serializer.serializeWithoutPrefixOrLength(key), partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
}
{
TokenKey roundTrip = serializer.deserializeWithPrefix(key.prefix(), serializer.serializedSizeWithoutPrefix(key), serializer.serializeWithoutPrefixOrLength(key), ByteBufferAccessor.instance, 0, partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
}
try (DataOutputBuffer buf = new DataOutputBuffer())
{
serializer.serialize(key, buf, 0);
byte[] bytes = buf.toByteArray();
Assertions.assertThat(bytes.length).isEqualTo(serializer.serializedSize(key, 0));
try (DataInputBuffer in = new DataInputBuffer(bytes))
{
TokenKey roundTrip = serializer.deserialize(in, 0, partitioner);
Assertions.assertThat(roundTrip).isEqualTo(key);
Invariants.require(0 == in.available());
}
try (DataInputBuffer in = new DataInputBuffer(bytes))
{
serializer.skip(in, 0, partitioner);
Invariants.require(0 == in.available());
}
}
});
}
@Test
public void compare()
{
qt().withSeed(0L).forAll(fromQT(partitioners().assuming(IPartitioner::accordSupported)).flatMap(partitioner -> routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(token(partitioner)), partitioner)))
.check(key -> {
ByteBuffer keyBytes = serializer.serialize(key);
for (TokenKey test : mutateAfter(key))
{
ByteBuffer testBytes = serializer.serialize(test);
Invariants.require(test.compareTo(key) > 0);
Invariants.require(ByteBufferUtil.compareUnsigned(testBytes, keyBytes) > 0);
}
for (TokenKey test : mutateBefore(key))
{
ByteBuffer testBytes = serializer.serialize(test);
Invariants.require(test.compareTo(key) < 0);
Invariants.require(ByteBufferUtil.compareUnsigned(testBytes, keyBytes) < 0);
}
});
}
private List<TokenKey> mutateAfter(TokenKey mutate)
{
List<TokenKey> results = new ArrayList<>();
if (!mutate.isTableSentinel())
{
Token token = mutate.token();
if (token instanceof ByteOrderedPartitioner.BytesToken)
{
byte[] bytes = (byte[]) token.getTokenValue();
bytes = bytes.clone();
for (int i = 0 ; i < bytes.length ; ++i)
{
if ((bytes[i] & 0xff) != 0xff)
{
++bytes[i];
add(results, mutate.withToken(new ByteOrderedPartitioner.BytesToken(bytes.clone())));
--bytes[i];
}
}
add(results, mutate.withToken(new ByteOrderedPartitioner.BytesToken(Arrays.copyOf(bytes, bytes.length + 1))));
}
else if (token instanceof LongToken)
{
long value = token.getLongValue();
if (value < Long.MAX_VALUE)
add(results, mutate.withToken(new LongToken(value + 1)));
for (long v = 2L; v >= 0 ; v <<= 1)
{
if ((value & v) == 0)
add(results, mutate.withToken(new LongToken(value | v)));
}
if (value >= 0)
{
long higher = value;
while ((higher <<= 8) > value)
add(results, mutate.withToken(new LongToken(higher)));
}
else
{
for (int i = 1 ; i < 8 ; ++i)
add(results, mutate.withToken(new LongToken(value >> (i * 8))));
}
}
else if (token instanceof BigIntegerToken)
{
BigInteger value = (BigInteger) token.getTokenValue();
if (value.compareTo(RandomPartitioner.MAXIMUM) < 0)
add(results, mutate.withToken(new BigIntegerToken(value.add(BigInteger.ONE))));
for (long v = 1L; v >= 0 ; v <<= 1)
{
BigInteger i = BigInteger.valueOf(v);
if (value.and(i).equals(BigInteger.ZERO))
add(results, mutate.withToken(new BigIntegerToken(value.or(i))));
}
BigInteger higher = value;
while ((higher = higher.shiftLeft(8)).compareTo(RandomPartitioner.MAXIMUM) <= 0)
add(results, mutate.withToken(new BigIntegerToken(higher)));
}
else throw new UnsupportedOperationException();
}
TableId tableId = mutate.table();
if (tableId.msb() != Long.MAX_VALUE)
add(results, mutate.withTable(TableId.fromRaw(tableId.msb() + 1, tableId.lsb())));
if (tableId.lsb() != Long.MAX_VALUE)
add(results, mutate.withTable(TableId.fromRaw(tableId.msb(), tableId.lsb() + 1)));
return results;
}
private List<TokenKey> mutateBefore(TokenKey mutate)
{
List<TokenKey> results = new ArrayList<>();
if (!mutate.isTableSentinel())
{
Token token = mutate.token();
if (token instanceof ByteOrderedPartitioner.BytesToken)
{
byte[] bytes = (byte[]) token.getTokenValue();
bytes = bytes.clone();
for (int i = 0 ; i < bytes.length ; ++i)
{
add(results, mutate.withToken(new ByteOrderedPartitioner.BytesToken(Arrays.copyOf(bytes, i))));
if ((bytes[i] & 0xff) != 0)
{
--bytes[i];
add(results, mutate.withToken(new ByteOrderedPartitioner.BytesToken(bytes.clone())));
++bytes[i];
}
}
}
else if (token instanceof LongToken)
{
long value = token.getLongValue();
if (value > Long.MIN_VALUE)
add(results, mutate.withToken(new LongToken(value - 1)));
for (long v = 2L; v >= 0 ; v <<= 1)
{
if ((value & v) != 0)
add(results, mutate.withToken(new LongToken(value & ~v)));
}
if (value >= 0)
{
for (int i = 1 ; i < 8 ; ++i)
add(results, mutate.withToken(new LongToken(value >>> (i * 8))));
}
else
{
for (int i = 0 ; i < 7 ; ++i)
{
long next = value & (-1L << (i * 8));
if (next != value)
add(results, mutate.withToken(new LongToken(next)));
}
}
}
else if (token instanceof BigIntegerToken)
{
BigInteger value = (BigInteger) token.getTokenValue();
if (value.compareTo(RandomPartitioner.MINIMUM.getTokenValue()) > 0)
add(results, mutate.withToken(new BigIntegerToken(value.subtract(BigInteger.ONE))));
for (long v = 1L; v >= 0 ; v <<= 1)
{
BigInteger i = BigInteger.valueOf(v);
if (!value.and(i).equals(BigInteger.ZERO))
add(results, mutate.withToken(new BigIntegerToken(value.andNot(i))));
}
for (int i = 1 ; i < 8 ; ++i)
add(results, mutate.withToken(new BigIntegerToken(value.shiftRight(i * 16))));
}
else throw new UnsupportedOperationException();
}
TableId tableId = mutate.table();
if (tableId.msb() != Long.MIN_VALUE)
add(results, mutate.withTable(TableId.fromRaw(tableId.msb() - 1, tableId.lsb())));
if (tableId.lsb() != Long.MIN_VALUE)
add(results, mutate.withTable(TableId.fromRaw(tableId.msb(), tableId.lsb() -1)));
return results;
}
private static void add(List<TokenKey> to, TokenKey vary)
{
to.add(vary);
to.add(vary.before());
to.add(vary.after());
}
private static Gen<TokenKey> routingKeyGen(Gen<TableId> tableIdGen, Gen<Token> tokenGen, IPartitioner partitioner)
{
Gen<TokenKey> result = AccordGenerators.routingKeyGen(tableIdGen, Gens.enums().all(TokenKey.RoutingKeyKind.class), tokenGen, partitioner);
if (!(partitioner instanceof ByteOrderedPartitioner))
return result;
return result.map((rs, k) -> {
byte[] bytes = (byte[]) k.token().getTokenValue();
if (bytes.length >= 3)
{
while (rs.nextFloat() < 0.25f)
{
int i = rs.nextInt(bytes.length - 2);
bytes[i] = 0;
bytes[i + 1] = (byte) rs.nextInt(0, TokenKey.Serializer.ESCAPE_BYTE);
}
}
return k;
});
}
}

6
test/unit/org/apache/cassandra/service/accord/serializers/TopologySerializersTest.java
View File

@ -21,11 +21,17 @@ package org.apache.cassandra.service.accord.serializers;
import org.junit.Test;
import accord.local.Node;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.utils.SerializerTestUtils;
public class TopologySerializersTest
{
static
{
DatabaseDescriptor.toolInitialization();
}
@Test
public void nodeId()
{

8
test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java
View File

@ -24,8 +24,10 @@ import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import com.google.common.collect.Iterables;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
@ -61,6 +63,12 @@ import static org.junit.Assert.assertTrue;
public class ClusterMetadataTransformationTest
{
@BeforeClass
public static void init()
{
DatabaseDescriptor.toolInitialization();
}
long seed = System.nanoTime();
Random random = new Random(seed);

64
test/unit/org/apache/cassandra/utils/AccordGenerators.java
View File

@ -68,8 +68,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.MinTokenKey;
import org.apache.cassandra.service.accord.api.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.txn.TxnData;
import org.apache.cassandra.service.accord.txn.TxnWrite;
@ -91,7 +90,7 @@ public class AccordGenerators
public static Gen<IPartitioner> partitioner()
{
return PARTITIONER_GEN;
return PARTITIONER_GEN.filter(IPartitioner::accordSupported);
}
private enum SupportedCommandTypes
@ -319,36 +318,31 @@ public class AccordGenerators
return rs -> new PartitionKey(tableIdGen.next(rs), key.next(rs));
}
public static Gen<AccordRoutingKey> routingKeys()
public static Gen<TokenKey> routingKeys(IPartitioner partitioner)
{
return routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN),
fromQT(CassandraGenerators.token()));
fromQT(CassandraGenerators.token(partitioner)),
partitioner);
}
public static Gen<AccordRoutingKey> routingKeys(IPartitioner partitioner)
public static Gen<TokenKey> routingKeyGen(Gen<TableId> tableIdGen, Gen<Token> tokenGen, IPartitioner partitioner)
{
return routingKeyGen(fromQT(CassandraGenerators.TABLE_ID_GEN),
fromQT(CassandraGenerators.token(partitioner)));
return routingKeyGen(tableIdGen, Gens.enums().all(TokenKey.RoutingKeyKind.class), tokenGen, partitioner);
}
public static Gen<AccordRoutingKey> routingKeyGen(Gen<TableId> tableIdGen, Gen<Token> tokenGen)
{
return routingKeyGen(tableIdGen, Gens.enums().all(AccordRoutingKey.RoutingKeyKind.class), tokenGen);
}
public static Gen<AccordRoutingKey> routingKeyGen(Gen<TableId> tableIdGen, Gen<AccordRoutingKey.RoutingKeyKind> kindGen, Gen<Token> tokenGen)
public static Gen<TokenKey> routingKeyGen(Gen<TableId> tableIdGen, Gen<TokenKey.RoutingKeyKind> kindGen, Gen<Token> tokenGen, IPartitioner partitioner)
{
return rs -> {
TableId tableId = tableIdGen.next(rs);
AccordRoutingKey.RoutingKeyKind kind = kindGen.next(rs);
TokenKey.RoutingKeyKind kind = kindGen.next(rs);
switch (kind)
{
case TOKEN:
return new AccordRoutingKey.TokenKey(tableId, tokenGen.next(rs));
return new TokenKey(tableId, tokenGen.next(rs));
case MIN_TOKEN:
return new MinTokenKey(tableId, tokenGen.next(rs));
return TokenKey.before(tableId, tokenGen.next(rs));
case SENTINEL:
return rs.nextBoolean() ? AccordRoutingKey.SentinelKey.min(tableId) : AccordRoutingKey.SentinelKey.max(tableId);
return rs.nextBoolean() ? TokenKey.min(tableId, partitioner) : TokenKey.max(tableId, partitioner);
default:
throw new AssertionError("Unknown kind: " + kind);
}
@ -357,20 +351,20 @@ public class AccordGenerators
public static Gen<Range> range()
{
return PARTITIONER_GEN.flatMap(partitioner -> range(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(CassandraGenerators.token(partitioner))));
return PARTITIONER_GEN.flatMap(partitioner -> range(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(CassandraGenerators.token(partitioner)), partitioner));
}
public static Gen<Range> range(IPartitioner partitioner)
{
return range(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(CassandraGenerators.token(partitioner)));
return range(fromQT(CassandraGenerators.TABLE_ID_GEN), fromQT(CassandraGenerators.token(partitioner)), partitioner);
}
public static Gen<Range> range(Gen<TableId> tables, Gen<Token> tokenGen)
public static Gen<Range> range(Gen<TableId> tables, Gen<Token> tokenGen, IPartitioner partitioner)
{
return rs -> {
Gen<AccordRoutingKey> gen = routingKeyGen(Gens.constant(tables.next(rs)), tokenGen);
AccordRoutingKey a = gen.next(rs);
AccordRoutingKey b = gen.next(rs);
Gen<TokenKey> gen = routingKeyGen(Gens.constant(tables.next(rs)), tokenGen, partitioner);
TokenKey a = gen.next(rs);
TokenKey b = gen.next(rs);
while (a.equals(b))
b = gen.next(rs);
if (a.compareTo(b) < 0) return TokenRange.create(a, b);
@ -391,7 +385,7 @@ public class AccordGenerators
IPartitioner partitioner = partitionerGen.next(rs);
List<Range> ranges = new ArrayList<>();
int numSplits = rs.nextInt(10, 100);
TokenRange range = TokenRange.create(AccordRoutingKey.SentinelKey.min(TABLE_ID1), AccordRoutingKey.SentinelKey.max(TABLE_ID1));
TokenRange range = TokenRange.create(TokenKey.min(TABLE_ID1, partitioner), TokenKey.max(TABLE_ID1, partitioner));
AccordSplitter splitter = partitioner.accordSplitter().apply(Ranges.of(range));
BigInteger size = splitter.sizeOf(range);
BigInteger update = splitter.divide(size, numSplits);
@ -428,41 +422,21 @@ public class AccordGenerators
};
}
public static Gen<KeyDeps> keyDepsGen()
{
return AccordGens.keyDeps(AccordGenerators.routingKeys());
}
public static Gen<KeyDeps> keyDepsGen(IPartitioner partitioner)
{
return AccordGens.keyDeps(AccordGenerators.routingKeys(partitioner));
}
public static Gen<KeyDeps> directKeyDepsGen()
{
return AccordGens.directKeyDeps(AccordGenerators.routingKeys());
}
public static Gen<KeyDeps> directKeyDepsGen(IPartitioner partitioner)
{
return AccordGens.directKeyDeps(AccordGenerators.routingKeys(partitioner));
}
public static Gen<RangeDeps> rangeDepsGen()
{
return AccordGens.rangeDeps(AccordGenerators.range());
}
public static Gen<RangeDeps> rangeDepsGen(IPartitioner partitioner)
{
return AccordGens.rangeDeps(AccordGenerators.range(partitioner));
}
public static Gen<Deps> depsGen()
{
return AccordGens.deps(keyDepsGen(), rangeDepsGen(), directKeyDepsGen());
}
public static Gen<Deps> depsGen(IPartitioner partitioner)
{
return AccordGens.deps(keyDepsGen(partitioner), rangeDepsGen(partitioner), directKeyDepsGen(partitioner));