natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Semi integrate burn test: 

* Move serializers that previously were in static field into corresponding swappable accessors
  * Add a (no-op in Cassandra impl) Result serializer
  * Allow more than one Accord Journal table
  * Move some previously-Cassandra implementation details (such as FieldUpdates) to Accord
  * Fix a bug with sorting of Journal Key (incorrect non-identity flags)
  * Accord side: Make sure not to re-sort collection of events scheduled for application
  * Implement serializers for Accord/BurnTest List* items

Patch by Alex Petrov; reviewed by Benedict Elliott Smith for CASSANDRA-20112.
This commit is contained in:
Alex Petrov 2024-11-15 13:35:05 +01:00 committed by David Capwell
parent 4c0f9198de
commit 74817bc83e
48 changed files with 1688 additions and 1265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit 61f4ebd21ff34b0807081d9df442236d8c334b52
Subproject commit bd0761c567d153995a3db8da686ffdc940247200

7
src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
View File

@ -244,7 +244,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
if (isAccordTimestampsForKey(cfs))
return new AccordTimestampsForKeyPurger(accordService);
if (isAccordJournal(cfs))
return new AccordJournalPurger(accordService);
return new AccordJournalPurger(accordService, cfs);
if (isAccordCommandsForKey(cfs))
return new AccordCommandsForKeyPurger(AccordKeyspace.CommandsForKeysAccessor, accordService);
@ -1035,7 +1035,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
long lastDescriptor = -1;
int lastOffset = -1;
public AccordJournalPurger(Supplier<IAccordService> serviceSupplier)
public AccordJournalPurger(Supplier<IAccordService> serviceSupplier, ColumnFamilyStore cfs)
{
service = (AccordService) serviceSupplier.get();
// TODO: test serialization version logic
@ -1046,7 +1046,6 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
this.redundantBefores = compactionInfo.redundantBefores;
this.ranges = compactionInfo.ranges;
this.durableBefores = compactionInfo.durableBefores;
ColumnFamilyStore cfs = Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL);
this.recordColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("record", false));
this.versionColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("user_version", false));
}
@ -1240,7 +1239,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
private static boolean isAccordJournal(ColumnFamilyStore cfs)
{
return isAccordTable(cfs, AccordKeyspace.JOURNAL);
return cfs.getKeyspaceName().equals(SchemaConstants.ACCORD_KEYSPACE_NAME) && cfs.name.startsWith(AccordKeyspace.JOURNAL);
}
private static boolean isAccordCommands(ColumnFamilyStore cfs)

10
src/java/org/apache/cassandra/journal/Journal.java
View File

@ -97,8 +97,11 @@ public class Journal<K, V> implements Shutdownable
final ValueSerializer<K, V> valueSerializer;
final Metrics<K, V> metrics;
final Flusher<K, V> flusher;
final Compactor<K, V> compactor;
Interruptible allocator;
SequentialExecutorPlus closer, releaser;
volatile long replayLimit;
final AtomicLong nextSegmentId = new AtomicLong();
@ -112,15 +115,14 @@ public class Journal<K, V> implements Shutdownable
final AtomicReference<State> state = new AtomicReference<>(State.UNINITIALIZED);
Interruptible allocator;
// TODO (required): we do not need wait queues here, we can just wait on a signal on a segment while its byte buffer is being allocated
private final WaitQueue segmentPrepared = newWaitQueue();
private final WaitQueue allocatorThreadWaitQueue = newWaitQueue();
private final BooleanSupplier allocatorThreadWaitCondition = () -> (availableSegment == null);
private final FlusherCallbacks flusherCallbacks;
final OpOrder readOrder = new OpOrder();
SequentialExecutorPlus closer, releaser;
private final FlusherCallbacks flusherCallbacks;
final OpOrder readOrder = new OpOrder();
private class FlusherCallbacks implements Flusher.Callbacks
{

157
src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
View File

@ -38,9 +38,9 @@ import accord.api.DataStore;
import accord.api.LocalListeners;
import accord.api.ProgressLog;
import accord.api.RoutingKey;
import accord.impl.AbstractLoader;
import accord.impl.AbstractSafeCommandStore.CommandStoreCaches;
import accord.impl.TimestampsForKey;
import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
@ -49,7 +49,6 @@ import accord.local.KeyHistory;
import accord.local.NodeCommandStoreService;
import accord.local.PreLoadContext;
import accord.local.RedundantBefore;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.cfk.CommandsForKey;
import accord.primitives.PartialTxn;
@ -69,15 +68,14 @@ import org.apache.cassandra.service.accord.SavedCommand.MinimalCommand;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.txn.TxnRead;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Promise;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static accord.api.Journal.CommandUpdate;
import static accord.api.Journal.FieldUpdates;
import static accord.api.Journal.Loader;
import static accord.api.Journal.OnDone;
import static accord.local.KeyHistory.SYNC;
import static accord.primitives.SaveStatus.Applying;
import static accord.primitives.Status.Committed;
import static accord.primitives.Status.Invalidated;
import static accord.primitives.Status.Truncated;
import static accord.utils.Invariants.checkState;
import static org.apache.cassandra.service.accord.SavedCommand.Load.MINIMAL;
@ -167,6 +165,8 @@ public class AccordCommandStore extends CommandStore
private AccordSafeCommandStore current;
private Thread currentThread;
private final CommandStoreLoader loader;
public AccordCommandStore(int id,
NodeCommandStoreService node,
Agent agent,
@ -195,6 +195,8 @@ public class AccordCommandStore extends CommandStore
this.taskExecutor = executor.executor(this);
this.commandsForRanges = new CommandsForRanges.Manager(this);
this.loader = new CommandStoreLoader(this);
loadRedundantBefore(journal.loadRedundantBefore(id()));
loadBootstrapBeganAt(journal.loadBootstrapBeganAt(id()));
loadSafeToRead(journal.loadSafeToRead(id()));
@ -292,16 +294,16 @@ public class AccordCommandStore extends CommandStore
return null;
}
public void persistFieldUpdates(AccordSafeCommandStore.FieldUpdates fieldUpdates, Runnable onFlush)
public void persistFieldUpdates(FieldUpdates fieldUpdates, Runnable onFlush)
{
journal.persistStoreState(id, fieldUpdates, onFlush);
journal.saveStoreState(id, fieldUpdates, onFlush);
}
@Nullable
@VisibleForTesting
public void appendToLog(Command before, Command after, Runnable onFlush)
{
journal.appendCommand(id, SavedCommand.diff(before, after), onFlush);
journal.saveCommand(id, new CommandUpdate(before, after), onFlush);
}
boolean validateCommand(TxnId txnId, Command evicting)
@ -490,13 +492,13 @@ public class AccordCommandStore extends CommandStore
}
}
public void appendCommands(List<SavedCommand.Writer> diffs, Runnable onFlush)
public void appendCommands(List<CommandUpdate> diffs, Runnable onFlush)
{
for (int i = 0; i < diffs.size(); i++)
{
boolean isLast = i == diffs.size() - 1;
SavedCommand.Writer writer = diffs.get(i);
journal.appendCommand(id, writer, isLast ? onFlush : null);
CommandUpdate change = diffs.get(i);
journal.saveCommand(id, change, isLast ? onFlush : null);
}
}
@ -526,92 +528,65 @@ public class AccordCommandStore extends CommandStore
return journal.loadMinimal(id, txnId, MINIMAL, unsafeGetRedundantBefore(), durableBefore());
}
public interface Loader
{
Promise<?> load(Command next);
Promise<?> apply(Command next);
}
public Loader loader()
{
return new Loader()
return loader;
}
private static class CommandStoreLoader extends AbstractLoader
{
private final AccordCommandStore store;
private CommandStoreLoader(AccordCommandStore store)
{
private PreLoadContext context(Command command, KeyHistory keyHistory)
{
TxnId txnId = command.txnId();
Participants<?> keys = null;
if (CommandsForKey.manages(txnId))
keys = command.hasBeen(Committed) ? command.participants().hasTouched() : command.participants().touches();
else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
keys = command.asCommitted().waitingOn.keys;
this.store = store;
}
if (keys != null)
return PreLoadContext.contextFor(txnId, keys, keyHistory);
private PreLoadContext context(Command command, KeyHistory keyHistory)
{
TxnId txnId = command.txnId();
Participants<?> keys = null;
if (CommandsForKey.manages(txnId))
keys = command.hasBeen(Committed) ? command.participants().hasTouched() : command.participants().touches();
else if (!CommandsForKey.managesExecution(txnId) && command.hasBeen(Status.Stable) && !command.hasBeen(Status.Truncated))
keys = command.asCommitted().waitingOn.keys;
return PreLoadContext.contextFor(txnId);
}
if (keys != null)
return PreLoadContext.contextFor(txnId, keys, keyHistory);
public Promise<?> load(Command command)
{
TxnId txnId = command.txnId();
return PreLoadContext.contextFor(txnId);
}
AsyncPromise<?> future = new AsyncPromise<>();
execute(context(command, SYNC),
safeStore -> {
Command local = command;
if (local.status() != Truncated && local.status() != Invalidated)
{
Cleanup cleanup = Cleanup.shouldCleanup(agent, local, unsafeGetRedundantBefore(), durableBefore());
switch (cleanup)
{
case NO:
break;
case INVALIDATE:
case TRUNCATE_WITH_OUTCOME:
case TRUNCATE:
case ERASE:
local = Commands.purge(local, local.participants(), cleanup);
}
}
@Override
public void load(Command command, OnDone onDone)
{
store.execute(context(command, SYNC),
safeStore -> loadInternal(command, safeStore))
.begin((unused, throwable) -> {
if (throwable != null)
onDone.failure(throwable);
else
onDone.success();
});
}
local = safeStore.unsafeGet(txnId).update(safeStore, local);
if (local.status() == Truncated)
safeStore.progressLog().clear(local.txnId());
})
.begin((unused, throwable) -> {
if (throwable != null)
future.setFailure(throwable);
else
future.setSuccess(null);
});
return future;
}
public Promise<?> apply(Command command)
{
TxnId txnId = command.txnId();
AsyncPromise<?> future = new AsyncPromise<>();
PreLoadContext context = context(command, KeyHistory.TIMESTAMPS);
execute(context,
safeStore -> {
SafeCommand safeCommand = safeStore.unsafeGet(txnId);
Command local = safeCommand.current();
if (local.hasBeen(Truncated))
return;
if (local.saveStatus().compareTo(Applying) >= 0) Commands.applyWrites(safeStore, context, local).begin(agent);
else Commands.maybeExecute(safeStore, safeCommand, local, true, true);
})
.begin((unused, throwable) -> {
if (throwable != null)
future.setFailure(throwable);
else
future.setSuccess(null);
});
return future;
}
};
@Override
public void apply(Command command, OnDone onDone)
{
PreLoadContext context = context(command, KeyHistory.TIMESTAMPS);
store.execute(context,
safeStore -> {
applyWrites(command, safeStore, (safeCommand, cmd) -> {
Commands.applyWrites(safeStore, context, cmd).begin(store.agent);
});
})
.begin((unused, throwable) -> {
if (throwable != null)
onDone.failure(throwable);
else
onDone.success();
});
}
}
/**

8
src/java/org/apache/cassandra/service/accord/AccordFetchCoordinator.java
View File

@ -90,7 +90,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
this.hasData = hasData;
}
static final IVersionedSerializer<SessionInfo> serializer = new IVersionedSerializer<SessionInfo>()
static final IVersionedSerializer<SessionInfo> serializer = new IVersionedSerializer<>()
{
public void serialize(SessionInfo info, DataOutputPlus out, int version) throws IOException
{
@ -110,7 +110,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
}
};
}
public static final IVersionedSerializer<StreamData> serializer = new IVersionedSerializer<StreamData>()
public static final IVersionedSerializer<StreamData> serializer = new IVersionedSerializer<>()
{
@Override
public void serialize(StreamData data, DataOutputPlus out, int version) throws IOException
@ -303,7 +303,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
private static final IVersionedSerializer<Read> read = new CastingSerializer<>(StreamingRead.class,
StreamingRead.serializer);
private static final IVersionedSerializer<Query> query = new IVersionedSerializer<Query>()
private static final IVersionedSerializer<Query> query = new IVersionedSerializer<>()
{
@Override
public void serialize(Query t, DataOutputPlus out, int version)
@ -325,7 +325,7 @@ public class AccordFetchCoordinator extends AbstractFetchCoordinator implements
}
};
private static final IVersionedSerializer<Update> update = new IVersionedSerializer<Update>()
private static final IVersionedSerializer<Update> update = new IVersionedSerializer<>()
{
@Override
public void serialize(Update t, DataOutputPlus out, int version)

89
src/java/org/apache/cassandra/service/accord/AccordJournal.java
View File

@ -25,21 +25,21 @@ import java.util.NavigableMap;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.BiConsumer;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.impl.ErasedSafeCommand;
import accord.impl.TimestampsForKey;
import accord.local.Cleanup;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.CommandStores;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.DurableBefore;
import accord.local.Node;
import accord.local.RedundantBefore;
import accord.local.cfk.CommandsForKey;
import accord.primitives.Ranges;
import accord.primitives.SaveStatus;
import accord.primitives.Timestamp;
@ -50,6 +50,8 @@ import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.apache.cassandra.concurrent.Shutdownable;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -64,11 +66,11 @@ import org.apache.cassandra.service.accord.AccordJournalValueSerializers.Identit
import org.apache.cassandra.service.accord.JournalKey.JournalKeySupport;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.utils.ExecutorUtils;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import static accord.primitives.SaveStatus.ErasedOrVestigial;
import static accord.primitives.Status.Truncated;
import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.DurableBeforeAccumulator;
import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.RedundantBeforeAccumulator;
public class AccordJournal implements IJournal, Shutdownable
{
@ -93,11 +95,15 @@ public class AccordJournal implements IJournal, Shutdownable
enum Status { INITIALIZED, STARTING, REPLAY, STARTED, TERMINATING, TERMINATED }
private volatile Status status = Status.INITIALIZED;
@VisibleForTesting
public AccordJournal(Params params, AccordAgent agent)
{
this(params, agent, new File(DatabaseDescriptor.getAccordJournalDirectory()), Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL));
}
@VisibleForTesting
public AccordJournal(Params params, AccordAgent agent, File directory, ColumnFamilyStore cfs)
{
this.agent = agent;
File directory = new File(DatabaseDescriptor.getAccordJournalDirectory());
this.journal = new Journal<>("AccordJournal", directory, params, JournalKey.SUPPORT,
// In Accord, we are using streaming serialization, i.e. Reader/Writer interfaces instead of materializing objects
new ValueSerializer<>()
@ -114,8 +120,8 @@ public class AccordJournal implements IJournal, Shutdownable
throw new UnsupportedOperationException();
}
},
new AccordSegmentCompactor<>(params.userVersion()));
this.journalTable = new AccordJournalTable<>(journal, JournalKey.SUPPORT, params.userVersion());
new AccordSegmentCompactor<>(params.userVersion(), cfs));
this.journalTable = new AccordJournalTable<>(journal, JournalKey.SUPPORT, cfs, params.userVersion());
this.params = params;
}
@ -216,7 +222,7 @@ public class AccordJournal implements IJournal, Shutdownable
@VisibleForTesting
public RedundantBefore loadRedundantBefore(int store)
{
RedundantBeforeAccumulator accumulator = readAll(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, store));
IdentityAccumulator<RedundantBefore> accumulator = readAll(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, store));
return accumulator.get();
}
@ -242,18 +248,18 @@ public class AccordJournal implements IJournal, Shutdownable
}
@Override
public void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush)
public void saveCommand(int store, CommandUpdate update, Runnable onFlush)
{
if (value == null || status == Status.REPLAY)
SavedCommand.Writer diff = SavedCommand.diff(update.before, update.after);
if (diff == null || status == Status.REPLAY)
{
if (onFlush != null)
onFlush.run();
return;
}
// TODO: use same API for commands as for the other states?
JournalKey key = new JournalKey(value.key(), JournalKey.Type.COMMAND_DIFF, store);
RecordPointer pointer = journal.asyncWrite(key, value, SENTINEL_HOSTS);
JournalKey key = new JournalKey(update.txnId, JournalKey.Type.COMMAND_DIFF, store);
RecordPointer pointer = journal.asyncWrite(key, diff, SENTINEL_HOSTS);
if (onFlush != null)
journal.onDurable(pointer, onFlush);
}
@ -287,12 +293,12 @@ public class AccordJournal implements IJournal, Shutdownable
}
@Override
public void persistStoreState(int store, AccordSafeCommandStore.FieldUpdates fieldUpdates, Runnable onFlush)
public void saveStoreState(int store, FieldUpdates fieldUpdates, Runnable onFlush)
{
RecordPointer pointer = null;
// TODO: avoid allocating keys
if (fieldUpdates.addRedundantBefore != null)
pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, store), fieldUpdates.addRedundantBefore);
if (fieldUpdates.newRedundantBefore != null)
pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.REDUNDANT_BEFORE, store), fieldUpdates.newRedundantBefore);
if (fieldUpdates.newBootstrapBeganAt != null)
pointer = appendInternal(new JournalKey(TxnId.NONE, JournalKey.Type.BOOTSTRAP_BEGAN_AT, store), fieldUpdates.newBootstrapBeganAt);
if (fieldUpdates.newSafeToRead != null)
@ -370,17 +376,23 @@ public class AccordJournal implements IJournal, Shutdownable
journal.runCompactorForTesting();
}
public void replay()
@Override
public void purge(CommandStores commandStores)
{
logger.info("Starting journal replay.");
TimestampsForKey.unsafeSetReplay(true);
CommandsForKey.disableLinearizabilityViolationsReporting();
AccordKeyspace.truncateAllCaches();
journal.closeCurrentSegmentForTestingIfNonEmpty();
journal.runCompactorForTesting();
journalTable.forceCompaction();
}
@Override
public void replay(CommandStores commandStores)
{
journal.closeCurrentSegmentForTestingIfNonEmpty();
try (AccordJournalTable.KeyOrderIterator<JournalKey> iter = journalTable.readAll())
{
JournalKey key;
SavedCommand.Builder builder = new SavedCommand.Builder();
while ((key = iter.key()) != null)
{
builder.reset(key.id);
@ -408,20 +420,15 @@ public class AccordJournal implements IJournal, Shutdownable
if (builder.nextCalled)
{
Command command = builder.construct();
AccordCommandStore commandStore = (AccordCommandStore) node.commandStores().forId(key.commandStoreId);
AccordCommandStore.Loader loader = commandStore.loader();
loader.load(command).get();
Invariants.checkState(command.saveStatus() != SaveStatus.Uninitialised,
"Found uninitialized command in the log: %s %s", command.toString(), builder.toString());
CommandStore commandStore = commandStores.forId(key.commandStoreId);
Loader loader = commandStore.loader();
async(loader::load, command).get();
if (command.saveStatus().compareTo(SaveStatus.Stable) >= 0 && !command.hasBeen(Truncated))
loader.apply(command).get();
async(loader::apply, command).get();
}
}
logger.info("Waiting for command stores to quiesce.");
((AccordCommandStores)node.commandStores()).waitForQuiescense();
CommandsForKey.enableLinearizabilityViolationsReporting();
TimestampsForKey.unsafeSetReplay(false);
logger.info("Finished journal replay.");
status = Status.STARTED;
}
catch (Throwable t)
{
@ -429,6 +436,24 @@ public class AccordJournal implements IJournal, Shutdownable
}
}
private AsyncPromise<?> async(BiConsumer<Command, OnDone> consumer, Command command)
{
AsyncPromise<?> future = new AsyncPromise<>();
consumer.accept(command, new OnDone()
{
public void success()
{
future.setSuccess(null);
}
public void failure(Throwable t)
{
future.setFailure(t);
}
});
return future;
}
// TODO: this is here temporarily; for debugging purposes
@VisibleForTesting
public void checkAllCommands()

10
src/java/org/apache/cassandra/service/accord/AccordJournalTable.java
View File

@ -31,7 +31,6 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.ColumnFamilyStore.RefViewFragment;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.EmptyIterators;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Slices;
import org.apache.cassandra.db.StorageHook;
import org.apache.cassandra.db.filter.ColumnFilter;
@ -70,16 +69,21 @@ public class AccordJournalTable<K extends JournalKey, V>
private final KeySupport<K> keySupport;
private final int accordJournalVersion;
public AccordJournalTable(Journal<K, V> journal, KeySupport<K> keySupport, int accordJournalVersion)
public AccordJournalTable(Journal<K, V> journal, KeySupport<K> keySupport, ColumnFamilyStore cfs, int accordJournalVersion)
{
this.journal = journal;
this.cfs = Keyspace.open(AccordKeyspace.metadata().name).getColumnFamilyStore(AccordKeyspace.JOURNAL);
this.cfs = cfs;
this.recordColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("record", false));
this.versionColumn = cfs.metadata().getColumn(ColumnIdentifier.getInterned("user_version", false));
this.keySupport = keySupport;
this.accordJournalVersion = accordJournalVersion;
}
public void forceCompaction()
{
cfs.forceMajorCompaction();
}
public interface Reader
{
void read(DataInputPlus input, int userVersion) throws IOException;

24
src/java/org/apache/cassandra/service/accord/AccordJournalValueSerializers.java
View File

@ -129,27 +129,13 @@ public class AccordJournalValueSerializers
}
}
public static class RedundantBeforeAccumulator extends Accumulator<RedundantBefore, RedundantBefore>
{
public RedundantBeforeAccumulator()
{
super(RedundantBefore.EMPTY);
}
@Override
protected RedundantBefore accumulate(RedundantBefore oldValue, RedundantBefore newValue)
{
return RedundantBefore.merge(oldValue, newValue);
}
}
public static class RedundantBeforeSerializer
implements FlyweightSerializer<RedundantBefore, RedundantBeforeAccumulator>
implements FlyweightSerializer<RedundantBefore, IdentityAccumulator<RedundantBefore>>
{
@Override
public RedundantBeforeAccumulator mergerFor(JournalKey journalKey)
public IdentityAccumulator<RedundantBefore> mergerFor(JournalKey journalKey)
{
return new RedundantBeforeAccumulator();
return new IdentityAccumulator<>(RedundantBefore.EMPTY);
}
@Override
@ -172,13 +158,13 @@ public class AccordJournalValueSerializers
}
@Override
public void reserialize(JournalKey key, RedundantBeforeAccumulator from, DataOutputPlus out, int userVersion) throws IOException
public void reserialize(JournalKey key, IdentityAccumulator<RedundantBefore> from, DataOutputPlus out, int userVersion) throws IOException
{
serialize(key, from.get(), out, userVersion);
}
@Override
public void deserialize(JournalKey journalKey, RedundantBeforeAccumulator into, DataInputPlus in, int userVersion) throws IOException
public void deserialize(JournalKey journalKey, IdentityAccumulator<RedundantBefore> into, DataInputPlus in, int userVersion) throws IOException
{
if (in.readInt() == 0)
{

50
src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
View File

@ -228,25 +228,28 @@ public class AccordKeyspace
}
}
public static final TableMetadata Journal =
parse(JOURNAL,
"accord journal",
"CREATE TABLE %s ("
+ "store_id int,"
+ "type tinyint,"
+ "id blob,"
+ "descriptor bigint,"
+ "offset int,"
+ "user_version int,"
+ "record blob,"
+ "PRIMARY KEY((store_id, type, id), descriptor, offset)"
+ ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC)" +
" WITH compression = {'class':'NoopCompressor'};")
.compaction(CompactionParams.lcs(emptyMap()))
.bloomFilterFpChance(0.01)
.partitioner(new LocalPartitioner(BytesType.instance))
.build();
public static TableMetadata journalMetadata(String tableName)
{
return parse(tableName,
"accord journal",
"CREATE TABLE %s ("
+ "store_id int,"
+ "type tinyint,"
+ "id blob,"
+ "descriptor bigint,"
+ "offset int,"
+ "user_version int,"
+ "record blob,"
+ "PRIMARY KEY((store_id, type, id), descriptor, offset)"
+ ") WITH CLUSTERING ORDER BY (descriptor DESC, offset DESC)" +
" WITH compression = {'class':'NoopCompressor'};")
.compaction(CompactionParams.lcs(emptyMap()))
.bloomFilterFpChance(0.01)
.partitioner(new LocalPartitioner(BytesType.instance))
.build();
}
public static final TableMetadata Journal = journalMetadata(JOURNAL);
// TODO: store timestamps as blobs (confirm there are no negative numbers, or offset)
public static final TableMetadata Commands =
@ -470,7 +473,7 @@ public class AccordKeyspace
}
}
private static final TableMetadata TimestampsForKeys =
public static final TableMetadata TimestampsForKeys =
parse(TIMESTAMPS_FOR_KEY,
"accord timestamps per key",
"CREATE TABLE %s ("
@ -596,7 +599,7 @@ public class AccordKeyspace
}
private static final LocalCompositePrefixPartitioner CFKPartitioner = new LocalCompositePrefixPartitioner(Int32Type.instance, UUIDType.instance, BytesType.instance);
private static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
public static final TableMetadata CommandsForKeys = commandsForKeysTable(COMMANDS_FOR_KEY);
private static TableMetadata commandsForKeysTable(String tableName)
{
@ -723,7 +726,7 @@ public class AccordKeyspace
public static final CommandsForKeyAccessor CommandsForKeysAccessor = new CommandsForKeyAccessor(CommandsForKeys);
private static final TableMetadata Topologies =
public static final TableMetadata Topologies =
parse(TOPOLOGIES,
"accord topologies",
"CREATE TABLE %s (" +
@ -735,7 +738,7 @@ public class AccordKeyspace
"redundant map<blob, blob>" +
')').build();
private static final TableMetadata EpochMetadata =
public static final TableMetadata EpochMetadata =
parse(EPOCH_METADATA,
"global epoch info",
"CREATE TABLE %s (" +
@ -762,9 +765,10 @@ public class AccordKeyspace
return KeyspaceMetadata.create(ACCORD_KEYSPACE_NAME, KeyspaceParams.local(), tables(), Views.none(), Types.none(), UserFunctions.none());
}
public static Tables tables = Tables.of(Commands, TimestampsForKeys, CommandsForKeys, Topologies, EpochMetadata, Journal);
public static Tables tables()
{
return Tables.of(Commands, TimestampsForKeys, CommandsForKeys, Topologies, EpochMetadata, Journal);
return tables;
}
public static void truncateAllCaches()

4
src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
View File

@ -62,7 +62,7 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.accord.api.AccordRoutingKey;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
import org.apache.cassandra.service.accord.txn.AccordUpdate;
import org.apache.cassandra.service.accord.txn.TxnQuery;
@ -296,7 +296,7 @@ public class AccordObjectSizes
final static long PREACCEPTED = measure(Command.SerializerSupport.preaccepted(attrs(false, true), EMPTY_TXNID, Ballot.ZERO));;
final static long ACCEPTED = measure(Command.SerializerSupport.accepted(attrs(true, false), SaveStatus.Accepted, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO));
final static long COMMITTED = measure(Command.SerializerSupport.committed(attrs(true, true), SaveStatus.Committed, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, null));
final static long EXECUTED = measure(Command.SerializerSupport.executed(attrs(true, true), SaveStatus.Applied, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.empty(Domain.Key), EMPTY_WRITES, CommandSerializers.APPLIED));
final static long EXECUTED = measure(Command.SerializerSupport.executed(attrs(true, true), SaveStatus.Applied, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.empty(Domain.Key), EMPTY_WRITES, ResultSerializers.APPLIED));
final static long TRUNCATED = measure(Command.SerializerSupport.truncatedApply(attrs(false, false), SaveStatus.TruncatedApply, EMPTY_TXNID, null, null));
final static long INVALIDATED = measure(Command.SerializerSupport.invalidated(EMPTY_TXNID));

5
src/java/org/apache/cassandra/service/accord/AccordSafeCommand.java
View File

@ -22,6 +22,7 @@ import java.util.Objects;
import com.google.common.annotations.VisibleForTesting;
import accord.api.Journal;
import accord.local.Command;
import accord.local.SafeCommand;
import accord.primitives.TxnId;
@ -120,9 +121,9 @@ public class AccordSafeCommand extends SafeCommand implements AccordSafeState<Tx
return original;
}
public SavedCommand.Writer diff()
public Journal.CommandUpdate update()
{
return SavedCommand.diff(original, current);
return new Journal.CommandUpdate(original, current);
}
@Override

13
src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java
View File

@ -48,6 +48,7 @@ import accord.primitives.Unseekables;
import accord.utils.Invariants;
import org.apache.cassandra.service.accord.AccordCommandStore.ExclusiveCaches;
import static accord.api.Journal.*;
import static accord.utils.Invariants.illegalState;
public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey, AccordCommandStore.ExclusiveCaches>
@ -283,7 +284,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
{
// TODO (required): this is a temporary measure, see comment on AccordJournalValueSerializers; upsert instead
// when modifying, only modify together with AccordJournalValueSerializers
ensureFieldUpdates().newRedundantBefore = ensureFieldUpdates().addRedundantBefore = RedundantBefore.merge(redundantBefore(), addRedundantBefore);
ensureFieldUpdates().newRedundantBefore = RedundantBefore.merge(redundantBefore(), addRedundantBefore);
}
@Override
@ -360,12 +361,4 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
if (fieldUpdates.newRangesForEpoch != null)
super.setRangesForEpoch(ranges);
}
public static class FieldUpdates
{
public RedundantBefore addRedundantBefore, newRedundantBefore;
public NavigableMap<TxnId, Ranges> newBootstrapBeganAt;
public NavigableMap<Timestamp, Ranges> newSafeToRead;
public RangesForEpoch.Snapshot newRangesForEpoch;
}
}
}

13
src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
View File

@ -47,10 +47,12 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
{
private static final Logger logger = LoggerFactory.getLogger(AccordSegmentCompactor.class);
private final int userVersion;
private final ColumnFamilyStore cfs;
public AccordSegmentCompactor(int userVersion)
public AccordSegmentCompactor(int userVersion, ColumnFamilyStore cfs)
{
this.userVersion = userVersion;
this.cfs = cfs;
}
@Override
@ -72,7 +74,6 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
if (readers.isEmpty())
return Collections.emptyList();
ColumnFamilyStore cfs = AccordKeyspace.AccordColumnFamilyStores.journal;
Descriptor descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables());
SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
@ -90,7 +91,7 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
{
if (key == null || !reader.key().equals(key))
{
maybeWritePartition(cfs, writer, key, builder, serializer, firstDescriptor, firstOffset);
maybeWritePartition(writer, key, builder, serializer, firstDescriptor, firstOffset);
key = reader.key();
serializer = (FlyweightSerializer<Object, Object>) key.type.serializer;
@ -127,7 +128,7 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
if (advanced) readers.offer(reader); // there is more to this reader, but not with this key
}
maybeWritePartition(cfs, writer, key, builder, serializer, firstDescriptor, firstOffset);
maybeWritePartition(writer, key, builder, serializer, firstDescriptor, firstOffset);
}
catch (Throwable t)
{
@ -140,11 +141,11 @@ public class AccordSegmentCompactor<V> implements SegmentCompactor<JournalKey, V
}
}
private void maybeWritePartition(ColumnFamilyStore cfs, SSTableTxnWriter writer, JournalKey key, Object builder, FlyweightSerializer<Object, Object> serializer, long descriptor, int offset) throws IOException
private void maybeWritePartition(SSTableTxnWriter writer, JournalKey key, Object builder, FlyweightSerializer<Object, Object> serializer, long descriptor, int offset) throws IOException
{
if (builder != null)
{
SimpleBuilder partitionBuilder = PartitionUpdate.simpleBuilder(AccordKeyspace.Journal, AccordKeyspace.JournalColumns.decorate(key));
SimpleBuilder partitionBuilder = PartitionUpdate.simpleBuilder(cfs.metadata(), AccordKeyspace.JournalColumns.decorate(key));
try (DataOutputBuffer out = DataOutputBuffer.scratchBuffer.get())
{
serializer.reserialize(key, builder, out, userVersion);

20
src/java/org/apache/cassandra/service/accord/AccordService.java
View File

@ -72,6 +72,7 @@ import accord.impl.DefaultRemoteListeners;
import accord.impl.DurabilityScheduling;
import accord.impl.RequestCallbacks;
import accord.impl.SizeOfIntersectionSorter;
import accord.impl.TimestampsForKey;
import accord.impl.progresslog.DefaultProgressLogs;
import accord.local.Command;
import accord.local.CommandStore;
@ -264,9 +265,26 @@ public class AccordService implements IAccordService, Shutdownable
}
instance = as;
as.journal().replay();
replayJournal(as);
}
private static void replayJournal(AccordService as)
{
logger.info("Starting journal replay.");
TimestampsForKey.unsafeSetReplay(true);
CommandsForKey.disableLinearizabilityViolationsReporting();
AccordKeyspace.truncateAllCaches();
as.journal().replay(as.node().commandStores());
logger.info("Waiting for command stores to quiesce.");
((AccordCommandStores)as.node.commandStores()).waitForQuiescense();
CommandsForKey.enableLinearizabilityViolationsReporting();
TimestampsForKey.unsafeSetReplay(false);
as.journal.unsafeSetStarted();
logger.info("Finished journal replay.");
}
public static void shutdownServiceAndWait(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException
{
IAccordService i = instance;

21
src/java/org/apache/cassandra/service/accord/AccordTask.java
View File

@ -36,6 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import accord.api.Journal;
import accord.api.RoutingKey;
import accord.local.Command;
import accord.local.CommandStore;
@ -649,7 +650,7 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
}
}
private void save(List<SavedCommand.Writer> diffs, Runnable onFlush)
private void save(List<Journal.CommandUpdate> diffs, Runnable onFlush)
{
if (sanityCheck != null)
{
@ -701,7 +702,7 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
R result = apply(safeStore);
// TODO (required): currently, we are not very efficient about ensuring that we persist the absolute minimum amount of state. Improve that.
List<SavedCommand.Writer> diffs = null;
List<Journal.CommandUpdate> changes = null;
if (commands != null)
{
for (AccordSafeCommand safeCommand : commands.values())
@ -709,27 +710,27 @@ public abstract class AccordTask<R> extends Task implements Runnable, Function<S
if (safeCommand.txnId().is(EphemeralRead))
continue;
SavedCommand.Writer diff = safeCommand.diff();
Journal.CommandUpdate diff = safeCommand.update();
if (diff == null)
continue;
if (diffs == null)
diffs = new ArrayList<>(commands.size());
diffs.add(diff);
if (changes == null)
changes = new ArrayList<>(commands.size());
changes.add(diff);
maybeSanityCheck(safeCommand);
}
}
boolean flush = diffs != null || safeStore.fieldUpdates() != null;
boolean flush = changes != null || safeStore.fieldUpdates() != null;
if (flush)
{
state(PERSISTING);
Runnable onFlush = () -> finish(result, null);
if (safeStore.fieldUpdates() != null)
commandStore.persistFieldUpdates(safeStore.fieldUpdates(), diffs == null ? onFlush : null);
if (diffs != null)
save(diffs, onFlush);
commandStore.persistFieldUpdates(safeStore.fieldUpdates(), changes == null ? onFlush : null);
if (changes != null)
save(changes, onFlush);
}
commandStore.complete(safeStore);

26
src/java/org/apache/cassandra/service/accord/IJournal.java
View File

@ -18,31 +18,21 @@
package org.apache.cassandra.service.accord;
import java.util.NavigableMap;
import accord.api.Journal;
import accord.local.Command;
import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.RedundantBefore;
import accord.primitives.Ranges;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.PersistentField.Persister;
public interface IJournal
public interface IJournal extends Journal
{
Command loadCommand(int commandStoreId, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore);
SavedCommand.MinimalCommand loadMinimal(int commandStoreId, TxnId txnId, SavedCommand.Load load, RedundantBefore redundantBefore, DurableBefore durableBefore);
// TODO (required): migrate to accord.api.Journal
default SavedCommand.MinimalCommand loadMinimal(int commandStoreId, TxnId txnId, SavedCommand.Load load, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
Command command = loadCommand(commandStoreId, txnId, redundantBefore, durableBefore);
return new SavedCommand.MinimalCommand(command.txnId(), command.saveStatus(), command.participants(), command.durability(), command.executeAt(), command.writes());
}
RedundantBefore loadRedundantBefore(int commandStoreId);
NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int commandStoreId);
NavigableMap<Timestamp, Ranges> loadSafeToRead(int commandStoreId);
CommandStores.RangesForEpoch.Snapshot loadRangesForEpoch(int commandStoreId);
void appendCommand(int store, SavedCommand.Writer value, Runnable onFlush);
Persister<DurableBefore, DurableBefore> durableBeforePersister();
void persistStoreState(int store,
// TODO: this class should not live under ASCS
AccordSafeCommandStore.FieldUpdates fieldUpdates,
Runnable onFlush);
}

3
src/java/org/apache/cassandra/service/accord/JournalKey.java
View File

@ -51,6 +51,7 @@ public final class JournalKey
public JournalKey(TxnId id, Type type, int commandStoreId)
{
Invariants.checkState((id.lsb & (0xffff & ~TxnId.IDENTITY_FLAGS)) == 0);
Invariants.nonNull(type);
Invariants.nonNull(id);
this.type = type;
@ -252,7 +253,7 @@ public final class JournalKey
{
return "Key{" +
"id=" + id +
"type=" + type +
", type=" + type +
", commandStoreId=" + commandStoreId +
'}';
}

95
src/java/org/apache/cassandra/service/accord/SavedCommand.java
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.function.Function;
import javax.annotation.Nullable;
@ -37,7 +36,6 @@ import accord.local.StoreParticipants;
import accord.primitives.Ballot;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Route;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
import accord.primitives.Timestamp;
@ -49,7 +47,8 @@ import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.journal.Journal;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.DepsSerializer;
import org.apache.cassandra.service.accord.serializers.DepsSerializers;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
import org.apache.cassandra.utils.Throwables;
@ -64,6 +63,7 @@ import static accord.utils.Invariants.illegalState;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.DURABILITY;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.EXECUTE_AT;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.PARTICIPANTS;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.RESULT;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.SAVE_STATUS;
import static org.apache.cassandra.service.accord.SavedCommand.Fields.WRITES;
import static org.apache.cassandra.service.accord.SavedCommand.Load.ALL;
@ -85,6 +85,7 @@ public class SavedCommand
PARTIAL_TXN,
WRITES,
CLEANUP,
RESULT,
;
public static final Fields[] FIELDS = values();
@ -142,18 +143,18 @@ public class SavedCommand
}
@Nullable
public static Writer diff(Command original, Command current)
public static Writer diff(Command before, Command after)
{
if (original == current
|| current == null
|| current.saveStatus() == SaveStatus.Uninitialised)
if (before == after
|| after == null
|| after.saveStatus() == SaveStatus.Uninitialised)
return null;
int flags = validateFlags(getFlags(original, current));
int flags = validateFlags(getFlags(before, after));
if (!anyFieldChanged(flags))
return null;
return new Writer(current, flags);
return new Writer(after, flags);
}
// TODO (required): calculate flags once
@ -210,7 +211,7 @@ public class SavedCommand
CommandSerializers.partialTxn.serialize(after.partialTxn(), out, userVersion);
break;
case PARTIAL_DEPS:
DepsSerializer.partialDeps.serialize(after.partialDeps(), out, userVersion);
DepsSerializers.partialDeps.serialize(after.partialDeps(), out, userVersion);
break;
case WAITING_ON:
Command.WaitingOn waitingOn = getWaitingOn(after);
@ -223,6 +224,9 @@ public class SavedCommand
case WRITES:
CommandSerializers.writes.serialize(after.writes(), out, userVersion);
break;
case RESULT:
ResultSerializers.result.serialize(after.result(), out, userVersion);
break;
case CLEANUP:
throw new IllegalStateException();
}
@ -248,10 +252,18 @@ public class SavedCommand
flags = collectFlags(before, after, Command::partialTxn, false, Fields.PARTIAL_TXN, flags);
flags = collectFlags(before, after, Command::partialDeps, false, Fields.PARTIAL_DEPS, flags);
flags = collectFlags(before, after, SavedCommand::getWaitingOn, false, Fields.WAITING_ON, flags);
// TODO: waitingOn vs WaitingOnWithExecutedAt?
flags = collectFlags(before, after, SavedCommand::getWaitingOn, true, Fields.WAITING_ON, flags);
flags = collectFlags(before, after, Command::writes, false, WRITES, flags);
// Special-cased for Journal BurnTest integration
if ((before != null && before.result() != null && before.result() != ResultSerializers.APPLIED) ||
(after.result() != null && after.result() != ResultSerializers.APPLIED))
{
flags = collectFlags(before, after, Command::writes, false, RESULT, flags);
}
return flags;
}
@ -298,17 +310,6 @@ public class SavedCommand
return (oldFlags & (0x10000 << field.ordinal())) != 0;
}
static EnumSet<Fields> getFieldsChanged(int flags)
{
EnumSet<Fields> fields = EnumSet.noneOf(Fields.class);
for (Fields field : Fields.FIELDS)
{
if ((flags & (0x10000 << field.ordinal())) != 0)
fields.add(field);
}
return fields;
}
static int toIterableSetFields(int flags)
{
return flags >>> 16;
@ -336,11 +337,6 @@ public class SavedCommand
return oldFlags | (1 << field.ordinal());
}
private static int unsetFieldIsNull(Fields field, int oldFlags)
{
return oldFlags & ~(1 << field.ordinal());
}
public enum Load
{
ALL(0),
@ -536,7 +532,7 @@ public class SavedCommand
durability = NotDurable;
acceptedOrCommitted = promised = Ballot.ZERO;
waitingOn = (txn, deps) -> null;
result = CommandSerializers.APPLIED;
result = ResultSerializers.APPLIED;
}
public boolean isEmpty()
@ -592,7 +588,7 @@ public class SavedCommand
throw new IllegalStateException("Unknown cleanup: " + cleanup);}
}
public Builder expungePartial(Cleanup cleanup, SaveStatus saveStatus, boolean includeOutcome)
private Builder expungePartial(Cleanup cleanup, SaveStatus saveStatus, boolean includeOutcome)
{
Invariants.checkState(txnId != null);
Builder builder = new Builder(txnId, ALL);
@ -629,7 +625,7 @@ public class SavedCommand
return builder;
}
public Builder saveStatusOnly()
private Builder saveStatusOnly()
{
Invariants.checkState(txnId != null);
Builder builder = new Builder(txnId, ALL);
@ -661,16 +657,6 @@ public class SavedCommand
return new MinimalCommand(txnId, saveStatus, participants, durability, executeAt, writes);
}
public static Route<?> deserializeRouteOrNull(DataInputPlus in, int userVersion) throws IOException
{
int flags = in.readInt();
if (!getFieldChanged(PARTICIPANTS, flags) || getFieldIsNull(PARTICIPANTS, flags))
return null;
return CommandSerializers.participants.deserializeRouteOnly(in, userVersion);
}
public void serialize(DataOutputPlus out, int userVersion) throws IOException
{
Invariants.checkState(mask == 0);
@ -714,7 +700,7 @@ public class SavedCommand
CommandSerializers.partialTxn.serialize(partialTxn(), out, userVersion);
break;
case PARTIAL_DEPS:
DepsSerializer.partialDeps.serialize(partialDeps(), out, userVersion);
DepsSerializers.partialDeps.serialize(partialDeps(), out, userVersion);
break;
case WAITING_ON:
out.writeInt(waitingOnBytes.length);
@ -726,6 +712,9 @@ public class SavedCommand
case CLEANUP:
out.writeByte(cleanup.ordinal());
break;
case RESULT:
ResultSerializers.result.serialize(result(), out, userVersion);
break;
}
iterable = unsetIterableFields(field, iterable);
@ -796,7 +785,7 @@ public class SavedCommand
partialTxn = CommandSerializers.partialTxn.deserialize(in, userVersion);
break;
case PARTIAL_DEPS:
partialDeps = DepsSerializer.partialDeps.deserialize(in, userVersion);
partialDeps = DepsSerializers.partialDeps.deserialize(in, userVersion);
break;
case WAITING_ON:
int size = in.readInt();
@ -823,6 +812,9 @@ public class SavedCommand
if (cleanup == null || newCleanup.compareTo(cleanup) > 0)
cleanup = newCleanup;
break;
case RESULT:
result = ResultSerializers.result.deserialize(in, userVersion);
break;
}
}
@ -845,13 +837,15 @@ public class SavedCommand
CommandSerializers.ballot.skip(in, userVersion);
break;
case PARTICIPANTS:
// TODO (expected): skip
CommandSerializers.participants.deserialize(in, userVersion);
break;
case PARTIAL_TXN:
CommandSerializers.partialTxn.deserialize(in, userVersion);
break;
case PARTIAL_DEPS:
DepsSerializer.partialDeps.deserialize(in, userVersion);
// TODO (expected): skip
DepsSerializers.partialDeps.deserialize(in, userVersion);
break;
case WAITING_ON:
int size = in.readInt();
@ -864,6 +858,10 @@ public class SavedCommand
case CLEANUP:
in.readByte();
break;
case RESULT:
// TODO (expected): skip
result = ResultSerializers.result.deserialize(in, userVersion);
break;
}
}
@ -885,12 +883,23 @@ public class SavedCommand
attrs.durability(durability);
if (participants != null)
attrs.setParticipants(participants);
else
attrs.setParticipants(StoreParticipants.empty(txnId));
if (partialDeps != null &&
(saveStatus.known.deps != NoDeps &&
saveStatus.known.deps != DepsErased &&
saveStatus.known.deps != DepsUnknown))
attrs.partialDeps(partialDeps);
switch (saveStatus.known.outcome)
{
case Erased:
case WasApply:
writes = null;
result = null;
break;
}
Command.WaitingOn waitingOn = null;
if (this.waitingOn != null)
waitingOn = this.waitingOn.provide(txnId, partialDeps);

2
src/java/org/apache/cassandra/service/accord/interop/AccordInteropRead.java
View File

@ -91,7 +91,7 @@ public class AccordInteropRead extends ReadData
private static class LocalReadData implements Data
{
static final IVersionedSerializer<LocalReadData> serializer = new IVersionedSerializer<LocalReadData>()
static final IVersionedSerializer<LocalReadData> serializer = new IVersionedSerializer<>()
{
@Override
public void serialize(LocalReadData data, DataOutputPlus out, int version) throws IOException

14
src/java/org/apache/cassandra/service/accord/serializers/AcceptSerializers.java
View File

@ -45,7 +45,7 @@ public class AcceptSerializers
{
CommandSerializers.ballot.serialize(accept.ballot, out, version);
CommandSerializers.timestamp.serialize(accept.executeAt, out, version);
DepsSerializer.partialDeps.serialize(accept.partialDeps, out, version);
DepsSerializers.partialDeps.serialize(accept.partialDeps, out, version);
}
@Override
@ -54,7 +54,7 @@ public class AcceptSerializers
return create(txnId, scope, waitForEpoch, minEpoch,
CommandSerializers.ballot.deserialize(in, version),
CommandSerializers.timestamp.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version));
DepsSerializers.partialDeps.deserialize(in, version));
}
@Override
@ -62,7 +62,7 @@ public class AcceptSerializers
{
return CommandSerializers.ballot.serializedSize(accept.ballot, version)
+ CommandSerializers.timestamp.serializedSize(accept.executeAt, version)
+ DepsSerializer.partialDeps.serializedSize(accept.partialDeps, version);
+ DepsSerializers.partialDeps.serializedSize(accept.partialDeps, version);
}
};
@ -93,7 +93,7 @@ public class AcceptSerializers
}
};
public static final IVersionedSerializer<AcceptReply> reply = new IVersionedSerializer<AcceptReply>()
public static final IVersionedSerializer<AcceptReply> reply = new IVersionedSerializer<>()
{
@Override
public void serialize(AcceptReply reply, DataOutputPlus out, int version) throws IOException
@ -105,7 +105,7 @@ public class AcceptSerializers
if (reply.deps != null)
{
out.writeByte(1);
DepsSerializer.deps.serialize(reply.deps, out, version);
DepsSerializers.deps.serialize(reply.deps, out, version);
}
else
{
@ -138,7 +138,7 @@ public class AcceptSerializers
{
default: throw new IllegalStateException("Unexpected AcceptNack type: " + (flags & 0x7));
case 1:
return new AcceptReply(DepsSerializer.deps.deserialize(in, version));
return new AcceptReply(DepsSerializers.deps.deserialize(in, version));
case 2:
return AcceptReply.ACCEPT_INVALIDATE;
case 3:
@ -161,7 +161,7 @@ public class AcceptSerializers
default: throw new AssertionError();
case Success:
if (reply.deps != null)
size += DepsSerializer.deps.serializedSize(reply.deps, version);
size += DepsSerializers.deps.serializedSize(reply.deps, version);
break;
case Truncated:
break;

12
src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java
View File

@ -38,7 +38,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
public class ApplySerializers
{
private static final IVersionedSerializer<Apply.Kind> kind = new IVersionedSerializer<Apply.Kind>()
private static final IVersionedSerializer<Apply.Kind> kind = new IVersionedSerializer<>()
{
public void serialize(Apply.Kind kind, DataOutputPlus out, int version) throws IOException
{
@ -64,7 +64,7 @@ public class ApplySerializers
{
kind.serialize(apply.kind, out, version);
CommandSerializers.timestamp.serialize(apply.executeAt, out, version);
DepsSerializer.partialDeps.serialize(apply.deps, out, version);
DepsSerializers.partialDeps.serialize(apply.deps, out, version);
CommandSerializers.nullablePartialTxn.serialize(apply.txn, out, version);
KeySerializers.nullableFullRoute.serialize(apply.fullRoute, out, version);
CommandSerializers.writes.serialize(apply.writes, out, version);
@ -79,19 +79,19 @@ public class ApplySerializers
return deserializeApply(txnId, scope, waitForEpoch,
kind.deserialize(in, version),
CommandSerializers.timestamp.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version),
DepsSerializers.partialDeps.deserialize(in, version),
CommandSerializers.nullablePartialTxn.deserialize(in, version),
KeySerializers.nullableFullRoute.deserialize(in, version),
CommandSerializers.writes.deserialize(in, version),
CommandSerializers.APPLIED);
ResultSerializers.APPLIED);
}
@Override
public long serializedBodySize(A apply, int version)
{
return kind.serializedSize(apply.kind, version)
return kind.serializedSize(apply.kind, version)
+ CommandSerializers.timestamp.serializedSize(apply.executeAt, version)
+ DepsSerializer.partialDeps.serializedSize(apply.deps, version)
+ DepsSerializers.partialDeps.serializedSize(apply.deps, version)
+ CommandSerializers.nullablePartialTxn.serializedSize(apply.txn, version)
+ KeySerializers.nullableFullRoute.serializedSize(apply.fullRoute, version)
+ CommandSerializers.writes.serializedSize(apply.writes, version);

4
src/java/org/apache/cassandra/service/accord/serializers/BeginInvalidationSerializers.java
View File

@ -34,7 +34,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
public class BeginInvalidationSerializers
{
public static final IVersionedSerializer<BeginInvalidation> request = new IVersionedSerializer<BeginInvalidation>()
public static final IVersionedSerializer<BeginInvalidation> request = new IVersionedSerializer<>()
{
@Override
public void serialize(BeginInvalidation begin, DataOutputPlus out, int version) throws IOException
@ -61,7 +61,7 @@ public class BeginInvalidationSerializers
}
};
public static final IVersionedSerializer<InvalidateReply> reply = new IVersionedSerializer<InvalidateReply>()
public static final IVersionedSerializer<InvalidateReply> reply = new IVersionedSerializer<>()
{
@Override
public void serialize(InvalidateReply reply, DataOutputPlus out, int version) throws IOException

8
src/java/org/apache/cassandra/service/accord/serializers/CalculateDepsSerializers.java
View File

@ -53,24 +53,24 @@ public class CalculateDepsSerializers
}
};
public static final IVersionedSerializer<CalculateDepsOk> reply = new IVersionedSerializer<CalculateDepsOk>()
public static final IVersionedSerializer<CalculateDepsOk> reply = new IVersionedSerializer<>()
{
@Override
public void serialize(CalculateDepsOk reply, DataOutputPlus out, int version) throws IOException
{
DepsSerializer.deps.serialize(reply.deps, out, version);
DepsSerializers.deps.serialize(reply.deps, out, version);
}
@Override
public CalculateDepsOk deserialize(DataInputPlus in, int version) throws IOException
{
return new CalculateDepsOk(DepsSerializer.deps.deserialize(in, version));
return new CalculateDepsOk(DepsSerializers.deps.deserialize(in, version));
}
@Override
public long serializedSize(CalculateDepsOk reply, int version)
{
return DepsSerializer.deps.serializedSize(reply.deps, version);
return DepsSerializers.deps.serializedSize(reply.deps, version);
}
};
}

8
src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java
View File

@ -157,7 +157,7 @@ public class CheckStatusSerializers
CheckStatusOkFull okFull = (CheckStatusOkFull) ok;
CommandSerializers.nullablePartialTxn.serialize(okFull.partialTxn, out, version);
DepsSerializer.nullablePartialDeps.serialize(okFull.stableDeps, out, version);
DepsSerializers.nullablePartialDeps.serialize(okFull.stableDeps, out, version);
CommandSerializers.nullableWrites.serialize(okFull.writes, out, version);
}
@ -190,12 +190,12 @@ public class CheckStatusSerializers
isCoordinating, durability, route, homeKey, invalidIf);
PartialTxn partialTxn = CommandSerializers.nullablePartialTxn.deserialize(in, version);
PartialDeps committedDeps = DepsSerializer.nullablePartialDeps.deserialize(in, version);
PartialDeps committedDeps = DepsSerializers.nullablePartialDeps.deserialize(in, version);
Writes writes = CommandSerializers.nullableWrites.deserialize(in, version);
Result result = null;
if (maxKnowledgeStatus.known.outcome.isOrWasApply())
result = CommandSerializers.APPLIED;
result = ResultSerializers.APPLIED;
return createOk(map, maxKnowledgeStatus, maxStatus, maxPromised, maxAcceptedOrCommitted, acceptedOrCommitted, executeAt,
isCoordinating, durability, route, homeKey, invalidIf, partialTxn, committedDeps, writes, result);
@ -229,7 +229,7 @@ public class CheckStatusSerializers
CheckStatusOkFull okFull = (CheckStatusOkFull) ok;
size += CommandSerializers.nullablePartialTxn.serializedSize(okFull.partialTxn, version);
size += DepsSerializer.nullablePartialDeps.serializedSize(okFull.stableDeps, version);
size += DepsSerializers.nullablePartialDeps.serializedSize(okFull.stableDeps, version);
size += CommandSerializers.nullableWrites.serializedSize(okFull.writes, version);
return size;
}

95
src/java/org/apache/cassandra/service/accord/serializers/CommandSerializers.java
View File

@ -21,30 +21,30 @@ package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import java.nio.ByteBuffer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import accord.api.Query;
import accord.api.Read;
import accord.api.Result;
import accord.api.Update;
import accord.api.Write;
import accord.coordinate.Infer;
import accord.local.Node;
import accord.local.StoreParticipants;
import accord.primitives.Ballot;
import accord.primitives.Known;
import accord.primitives.Known.Definition;
import accord.primitives.Known.KnownDeps;
import accord.primitives.Known.KnownExecuteAt;
import accord.primitives.Known.KnownRoute;
import accord.primitives.Known.Outcome;
import accord.primitives.PartialTxn;
import accord.primitives.Participants;
import accord.primitives.Route;
import accord.primitives.SaveStatus;
import accord.primitives.Seekables;
import accord.primitives.Status;
import accord.primitives.Status.Durability;
import accord.primitives.Known;
import accord.primitives.Ballot;
import accord.primitives.PartialTxn;
import accord.primitives.ProgressToken;
import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@ -65,17 +65,9 @@ import org.apache.cassandra.utils.NullableSerializer;
public class CommandSerializers
{
private CommandSerializers() {}
// TODO (expected): this is meant to encode e.g. whether the transaction's condition met or not
public static final Result APPLIED = new Result()
private CommandSerializers()
{
@Override
public ProgressToken asProgressToken()
{
return ProgressToken.APPLIED;
}
};
}
public static final TimestampSerializer<TxnId> txnId = new TimestampSerializer<>(TxnId::fromBits);
public static final IVersionedSerializer<TxnId> nullableTxnId = NullableSerializer.wrap(txnId);
@ -93,6 +85,7 @@ public class CommandSerializers
static final int HAS_TOUCHED_EQUALS_ROUTE = 0x2;
static final int TOUCHES_EQUALS_HAS_TOUCHED = 0x4;
static final int OWNS_EQUALS_TOUCHES = 0x8;
@Override
public void serialize(StoreParticipants t, DataOutputPlus out, int version) throws IOException
{
@ -255,7 +248,8 @@ public class CommandSerializers
}
}
public static class PartialTxnSerializer extends AbstractWithKeysSerializer implements IVersionedSerializer<PartialTxn>
public static class PartialTxnSerializer extends AbstractWithKeysSerializer
implements IVersionedSerializer<PartialTxn>
{
private final IVersionedSerializer<Read> readSerializer;
private final IVersionedSerializer<Query> querySerializer;
@ -321,12 +315,60 @@ public class CommandSerializers
}
}
private static final IVersionedSerializer<Read> read = new CastingSerializer<>(TxnRead.class, TxnRead.serializer);
private static final IVersionedSerializer<Query> query = new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer);
private static final IVersionedSerializer<Update> update = new CastingSerializer<>(AccordUpdate.class, AccordUpdate.serializer);
public static final IVersionedSerializer<Read> read;
public static final IVersionedSerializer<Query> query;
public static final IVersionedSerializer<Update> update;
public static final IVersionedSerializer<Write> write;
public static final IVersionedSerializer<PartialTxn> partialTxn = new PartialTxnSerializer(read, query, update);
public static final IVersionedSerializer<PartialTxn> nullablePartialTxn = NullableSerializer.wrap(partialTxn);
public static final IVersionedSerializer<PartialTxn> partialTxn;
public static final IVersionedSerializer<PartialTxn> nullablePartialTxn;
static
{
// We use a separate class for initialization to make it easier for BurnTest to plug its own serializers.
QuerySerializers querySerializers = new QuerySerializers();
read = querySerializers.read;
query = querySerializers.query;
update = querySerializers.update;
write = querySerializers.write;
partialTxn = querySerializers.partialTxn;
nullablePartialTxn = querySerializers.nullablePartialTxn;
}
@VisibleForTesting
public static class QuerySerializers
{
public final IVersionedSerializer<Read> read;
public final IVersionedSerializer<Query> query;
public final IVersionedSerializer<Update> update;
public final IVersionedSerializer<Write> write;
public final IVersionedSerializer<PartialTxn> partialTxn;
public final IVersionedSerializer<PartialTxn> nullablePartialTxn;
private QuerySerializers()
{
this(new CastingSerializer<>(TxnRead.class, TxnRead.serializer),
new CastingSerializer<>(TxnQuery.class, TxnQuery.serializer),
new CastingSerializer<>(AccordUpdate.class, AccordUpdate.serializer),
new CastingSerializer<>(TxnWrite.class, TxnWrite.serializer));
}
public QuerySerializers(IVersionedSerializer<Read> read,
IVersionedSerializer<Query> query,
IVersionedSerializer<Update> update,
IVersionedSerializer<Write> write)
{
this.read = read;
this.query = query;
this.update = update;
this.write = write;
this.partialTxn = new PartialTxnSerializer(read, query, update);
this.nullablePartialTxn = NullableSerializer.wrap(partialTxn);
}
}
public static final EnumSerializer<SaveStatus> saveStatus = new EnumSerializer<>(SaveStatus.class);
public static final EnumSerializer<Status> status = new EnumSerializer<>(Status.class);
@ -342,8 +384,9 @@ public class CommandSerializers
KeySerializers.seekables.serialize(writes.keys, out, version);
boolean hasWrites = writes.write != null;
out.writeBoolean(hasWrites);
if (hasWrites)
TxnWrite.serializer.serialize((TxnWrite) writes.write, out, version);
CommandSerializers.write.serialize(writes.write, out, version);
}
@Override
@ -351,7 +394,7 @@ public class CommandSerializers
{
return new Writes(txnId.deserialize(in, version), timestamp.deserialize(in, version),
KeySerializers.seekables.deserialize(in, version),
in.readBoolean() ? TxnWrite.serializer.deserialize(in, version) : null);
in.readBoolean() ? CommandSerializers.write.deserialize(in, version) : null);
}
@Override
@ -363,7 +406,7 @@ public class CommandSerializers
boolean hasWrites = writes.write != null;
size += TypeSizes.sizeof(hasWrites);
if (hasWrites)
size += TxnWrite.serializer.serializedSize((TxnWrite) writes.write, version);
size += CommandSerializers.write.serializedSize(writes.write, version);
return size;
}
};
@ -412,4 +455,4 @@ public class CommandSerializers
};
public static final IVersionedSerializer<Known> nullableKnown = NullableSerializer.wrap(known);
}
}

6
src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java
View File

@ -61,7 +61,7 @@ public class CommitSerializers
CommandSerializers.ballot.serialize(msg.ballot, out, version);
CommandSerializers.timestamp.serialize(msg.executeAt, out, version);
CommandSerializers.nullablePartialTxn.serialize(msg.partialTxn, out, version);
DepsSerializer.partialDeps.serialize(msg.scope, msg.partialDeps, out, version);
DepsSerializers.partialDeps.serialize(msg.scope, msg.partialDeps, out, version);
serializeNullable(msg.route, out, version, KeySerializers.fullRoute);
serializeNullable(msg.readData, out, version, read);
}
@ -78,7 +78,7 @@ public class CommitSerializers
Ballot ballot = CommandSerializers.ballot.deserialize(in, version);
Timestamp executeAt = CommandSerializers.timestamp.deserialize(in, version);
PartialTxn txn = CommandSerializers.nullablePartialTxn.deserialize(in, version);
PartialDeps deps = DepsSerializer.partialDeps.deserialize(scope, in, version);
PartialDeps deps = DepsSerializers.partialDeps.deserialize(scope, in, version);
FullRoute<?> route = deserializeNullable(in, version, KeySerializers.fullRoute);
ReadData read = deserializeNullable(in, version, this.read);
return deserializeCommit(txnId, scope, waitForEpoch, minEpoch, kind, ballot, executeAt, txn, deps, route, read);
@ -91,7 +91,7 @@ public class CommitSerializers
+ CommandSerializers.ballot.serializedSize(msg.ballot, version)
+ CommandSerializers.timestamp.serializedSize(msg.executeAt, version)
+ CommandSerializers.nullablePartialTxn.serializedSize(msg.partialTxn, version)
+ DepsSerializer.partialDeps.serializedSize(msg.scope, msg.partialDeps, version)
+ DepsSerializers.partialDeps.serializedSize(msg.scope, msg.partialDeps, version)
+ serializedNullableSize(msg.route, version, KeySerializers.fullRoute)
+ serializedNullableSize(msg.readData, version, read);
}

295
src/java/org/apache/cassandra/service/accord/serializers/DepsSerializer.java
View File

@ -1,295 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import com.google.common.primitives.Ints;
import accord.primitives.AbstractUnseekableKeys;
import accord.primitives.Deps;
import accord.primitives.KeyDeps;
import accord.primitives.PartialDeps;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.RangeDeps;
import accord.primitives.RoutingKeys;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.utils.NullableSerializer;
import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIds;
import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIdsCount;
import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIds;
import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIdsCount;
import static accord.primitives.Routable.Domain.Key;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
public abstract class DepsSerializer<D extends Deps> extends IVersionedWithKeysSerializer.AbstractWithKeysSerializer implements IVersionedWithKeysSerializer<Unseekables<?>, D>
{
public static final DepsSerializer<Deps> deps = new DepsSerializer<>()
{
@Override
Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version)
{
return new Deps(keyDeps, rangeDeps, directKeyDeps);
}
};
public static final IVersionedSerializer<Deps> nullableDeps = NullableSerializer.wrap(deps);
public static final DepsSerializer<PartialDeps> partialDeps = new DepsSerializer<>()
{
@Override
PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException
{
Participants<?> covering = KeySerializers.participants.deserialize(in, version);
return new PartialDeps(covering, keyDeps, rangeDeps, directKeyDeps);
}
@Override
public void serialize(PartialDeps partialDeps, DataOutputPlus out, int version) throws IOException
{
super.serialize(partialDeps, out, version);
KeySerializers.participants.serialize(partialDeps.covering, out, version);
}
@Override
public void serialize(Unseekables<?> superset, PartialDeps partialDeps, DataOutputPlus out, int version) throws IOException
{
super.serialize(superset, partialDeps, out, version);
KeySerializers.participants.serialize(partialDeps.covering, out, version);
}
@Override
public long serializedSize(PartialDeps partialDeps, int version)
{
return super.serializedSize(partialDeps, version)
+ KeySerializers.participants.serializedSize(partialDeps.covering, version);
}
@Override
public long serializedSize(Unseekables<?> keys, PartialDeps partialDeps, int version)
{
return super.serializedSize(keys, partialDeps, version)
+ KeySerializers.participants.serializedSize(partialDeps.covering, version);
}
};
public static final IVersionedSerializer<PartialDeps> nullablePartialDeps = NullableSerializer.wrap(partialDeps);
abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException;
@Override
public void serialize(D deps, DataOutputPlus out, int version) throws IOException
{
KeySerializers.routingKeys.serialize(deps.keyDeps.keys(), out, version);
serializeWithoutKeys(deps, out, version);
}
@Override
public void serialize(Unseekables<?> superset, D deps, DataOutputPlus out, int version) throws IOException
{
if (superset.domain() == Key) serializeSubset(deps.keyDeps.keys(), superset, out);
else KeySerializers.routingKeys.serialize(deps.keyDeps.keys(), out, version);
serializeWithoutKeys(deps, out, version);
}
@Override
public D deserialize(DataInputPlus in, int version) throws IOException
{
RoutingKeys keys = KeySerializers.routingKeys.deserialize(in, version);
return deserializeWithoutKeys(keys, in, version);
}
@Override
public D deserialize(Unseekables<?> superset, DataInputPlus in, int version) throws IOException
{
RoutingKeys keys;
if (superset.domain() == Key) keys = ((AbstractUnseekableKeys)deserializeSubset(superset, in)).toParticipants();
else keys = KeySerializers.routingKeys.deserialize(in, version);
return deserializeWithoutKeys(keys, in, version);
}
@Override
public long serializedSize(D deps, int version)
{
long size = KeySerializers.routingKeys.serializedSize(deps.keyDeps.keys(), version);
size += serializedSizeWithoutKeys(deps, version);
return size;
}
@Override
public long serializedSize(Unseekables<?> keys, D deps, int version)
{
long size;
if (keys.domain() == Key) size = serializedSubsetSize(deps.keyDeps.keys(), keys);
else size = KeySerializers.routingKeys.serializedSize(deps.keyDeps.keys(), version);
size += serializedSizeWithoutKeys(deps, version);
return size;
}
private void serializeWithoutKeys(D deps, DataOutputPlus out, int version) throws IOException
{
serializeKeyDepsWithoutKeys(deps.keyDeps, out, version);
{
RangeDeps rangeDeps = deps.rangeDeps;
int rangeCount = rangeDeps.rangeCount();
out.writeUnsignedVInt32(rangeCount);
for (int i = 0; i < rangeCount; i++)
TokenRange.serializer.serialize((TokenRange) rangeDeps.range(i), out, version);
int txnIdCount = rangeDeps.txnIdCount();
out.writeUnsignedVInt32(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
CommandSerializers.txnId.serialize(rangeDeps.txnId(i), out, version);
int rangesToTxnIdsCount = rangesToTxnIdsCount(rangeDeps);
out.writeUnsignedVInt32(rangesToTxnIdsCount);
for (int i = 0; i < rangesToTxnIdsCount; i++)
out.writeUnsignedVInt32(rangesToTxnIds(rangeDeps, i));
}
{
RoutingKeys keys = deps.directKeyDeps.keys();
boolean isSubset = isSubset(keys, deps.keyDeps.keys());
out.writeBoolean(isSubset);
if (isSubset) serializeSubset(keys, deps.keyDeps.keys(), out);
else KeySerializers.routingKeys.serialize(keys, out, version);
serializeKeyDepsWithoutKeys(deps.directKeyDeps, out, version);
}
}
private void serializeKeyDepsWithoutKeys(KeyDeps keyDeps, DataOutputPlus out, int version) throws IOException
{
int txnIdCount = keyDeps.txnIdCount();
out.writeUnsignedVInt32(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
CommandSerializers.txnId.serialize(keyDeps.txnId(i), out, version);
int keysToTxnIdsCount = keysToTxnIdsCount(keyDeps);
out.writeUnsignedVInt32(keysToTxnIdsCount);
for (int i = 0; i < keysToTxnIdsCount; i++)
out.writeUnsignedVInt32(keysToTxnIds(keyDeps, i));
}
private D deserializeWithoutKeys(RoutingKeys keys, DataInputPlus in, int version) throws IOException
{
KeyDeps keyDeps = deserializeKeyDeps(keys, in, version);
RangeDeps rangeDeps;
{
int rangeCount = Ints.checkedCast(in.readUnsignedVInt32());
Range[] ranges = new Range[rangeCount];
for (int i = 0; i < rangeCount; i++)
ranges[i] = TokenRange.serializer.deserialize(in, version);
int txnIdCount = in.readUnsignedVInt32();
TxnId[] txnIds = new TxnId[txnIdCount];
for (int i = 0; i < txnIdCount; i++)
txnIds[i] = CommandSerializers.txnId.deserialize(in, version);
int rangesToTxnIdsCount = in.readUnsignedVInt32();
int[] rangesToTxnIds = new int[rangesToTxnIdsCount];
for (int i = 0; i < rangesToTxnIdsCount; i++)
rangesToTxnIds[i] = in.readUnsignedVInt32();
rangeDeps = RangeDeps.SerializerSupport.create(ranges, txnIds, rangesToTxnIds);
}
KeyDeps directKeyDeps;
{
boolean isSubset = in.readBoolean();
RoutingKeys directKeys = isSubset ? (RoutingKeys)deserializeSubset(keys, in) : KeySerializers.routingKeys.deserialize(in, version);
directKeyDeps = deserializeKeyDeps(directKeys, in, version);
}
return deserialize(keyDeps, rangeDeps, directKeyDeps, in, version);
}
private static KeyDeps deserializeKeyDeps(RoutingKeys keys, DataInputPlus in, int version) throws IOException
{
int txnIdCount = in.readUnsignedVInt32();
TxnId[] txnIds = new TxnId[txnIdCount];
for (int i = 0; i < txnIdCount; i++)
txnIds[i] = CommandSerializers.txnId.deserialize(in, version);
int keysToTxnIdsCount = in.readUnsignedVInt32();
int[] keysToTxnIds = new int[keysToTxnIdsCount];
for (int i = 0; i < keysToTxnIdsCount; i++)
keysToTxnIds[i] = in.readUnsignedVInt32();
return KeyDeps.SerializerSupport.create(keys, txnIds, keysToTxnIds);
}
private long serializedSizeWithoutKeys(D deps, int version)
{
long size = serializedSizeOfKeyDepsWithoutKeys(deps.keyDeps, version);
RangeDeps rangeDeps = deps.rangeDeps;
{
int rangeCount = rangeDeps.rangeCount();
size += sizeofUnsignedVInt(rangeCount);
for (int i = 0 ; i < rangeCount ; ++i)
size += TokenRange.serializer.serializedSize((TokenRange) rangeDeps.range(i), version);
int txnIdCount = rangeDeps.txnIdCount();
size += sizeofUnsignedVInt(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
size += CommandSerializers.txnId.serializedSize(rangeDeps.txnId(i), version);
int rangesToTxnIdsCount = rangesToTxnIdsCount(rangeDeps);
size += sizeofUnsignedVInt(rangesToTxnIdsCount);
for (int i = 0; i < rangesToTxnIdsCount; i++)
size += sizeofUnsignedVInt(rangesToTxnIds(rangeDeps, i));
}
{
boolean isSubset = isSubset(deps.directKeyDeps.keys(), deps.keyDeps.keys());
size += 1;
size += isSubset ? serializedSubsetSize(deps.directKeyDeps.keys(), deps.keyDeps.keys()) : KeySerializers.routingKeys.serializedSize(deps.directKeyDeps.keys(), version);
size += serializedSizeOfKeyDepsWithoutKeys(deps.directKeyDeps, version);
}
return size;
}
private static long serializedSizeOfKeyDepsWithoutKeys(KeyDeps keyDeps, int version)
{
int txnIdCount = keyDeps.txnIdCount();
long size = sizeofUnsignedVInt(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
size += CommandSerializers.txnId.serializedSize(keyDeps.txnId(i), version);
int keysToTxnIdsCount = keysToTxnIdsCount(keyDeps);
size += sizeofUnsignedVInt(keysToTxnIdsCount);
for (int i = 0; i < keysToTxnIdsCount; i++)
size += sizeofUnsignedVInt(keysToTxnIds(keyDeps, i));
return size;
}
private static boolean isSubset(RoutingKeys test, RoutingKeys superset)
{
return test.foldl(superset, (k, p, v, i) -> v + 1, 0, 0, 0) == test.size();
}
}

334
src/java/org/apache/cassandra/service/accord/serializers/DepsSerializers.java Normal file
View File

@ -0,0 +1,334 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import java.io.IOException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
import accord.primitives.AbstractUnseekableKeys;
import accord.primitives.Deps;
import accord.primitives.KeyDeps;
import accord.primitives.PartialDeps;
import accord.primitives.Participants;
import accord.primitives.Range;
import accord.primitives.RangeDeps;
import accord.primitives.RoutingKeys;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.TokenRange;
import org.apache.cassandra.utils.NullableSerializer;
import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIds;
import static accord.primitives.KeyDeps.SerializerSupport.keysToTxnIdsCount;
import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIds;
import static accord.primitives.RangeDeps.SerializerSupport.rangesToTxnIdsCount;
import static accord.primitives.Routable.Domain.Key;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
public class DepsSerializers
{
public static final IVersionedSerializer<Range> tokenRange;
public static final DepsSerializer<Deps> deps;
public static final IVersionedSerializer<Deps> nullableDeps;
public static final DepsSerializer<PartialDeps> partialDeps;
public static final IVersionedSerializer<PartialDeps> nullablePartialDeps;
static
{
// We use a separate class for initialization to make it easier for BurnTest to plug its own serializers.
Impl serializers = new Impl((IVersionedSerializer<Range>) (IVersionedSerializer<?>) TokenRange.serializer);
tokenRange = serializers.tokenRange;
deps = serializers.deps;
nullableDeps = serializers.nullableDeps;
partialDeps = serializers.partialDeps;
nullablePartialDeps = serializers.nullablePartialDeps;
}
public static abstract class DepsSerializer<D extends Deps> extends IVersionedWithKeysSerializer.AbstractWithKeysSerializer implements IVersionedWithKeysSerializer<Unseekables<?>, D>
{
protected IVersionedSerializer<Range> tokenRange;
public DepsSerializer(IVersionedSerializer<Range> tokenRange)
{
this.tokenRange = tokenRange;
}
abstract D deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException;
@Override
public void serialize(D deps, DataOutputPlus out, int version) throws IOException
{
KeySerializers.routingKeys.serialize(deps.keyDeps.keys(), out, version);
serializeWithoutKeys(deps, out, version);
}
@Override
public void serialize(Unseekables<?> superset, D deps, DataOutputPlus out, int version) throws IOException
{
if (superset.domain() == Key) serializeSubset(deps.keyDeps.keys(), superset, out);
else KeySerializers.routingKeys.serialize(deps.keyDeps.keys(), out, version);
serializeWithoutKeys(deps, out, version);
}
@Override
public D deserialize(DataInputPlus in, int version) throws IOException
{
RoutingKeys keys = KeySerializers.routingKeys.deserialize(in, version);
return deserializeWithoutKeys(keys, in, version);
}
@Override
public D deserialize(Unseekables<?> superset, DataInputPlus in, int version) throws IOException
{
RoutingKeys keys;
if (superset.domain() == Key) keys = ((AbstractUnseekableKeys) deserializeSubset(superset, in)).toParticipants();
else keys = KeySerializers.routingKeys.deserialize(in, version);
return deserializeWithoutKeys(keys, in, version);
}
@Override
public long serializedSize(D deps, int version)
{
long size = KeySerializers.routingKeys.serializedSize(deps.keyDeps.keys(), version);
size += serializedSizeWithoutKeys(deps, version);
return size;
}
@Override
public long serializedSize(Unseekables<?> keys, D deps, int version)
{
long size;
if (keys.domain() == Key) size = serializedSubsetSize(deps.keyDeps.keys(), keys);
else size = KeySerializers.routingKeys.serializedSize(deps.keyDeps.keys(), version);
size += serializedSizeWithoutKeys(deps, version);
return size;
}
private void serializeWithoutKeys(D deps, DataOutputPlus out, int version) throws IOException
{
serializeKeyDepsWithoutKeys(deps.keyDeps, out, version);
{
RangeDeps rangeDeps = deps.rangeDeps;
int rangeCount = rangeDeps.rangeCount();
out.writeUnsignedVInt32(rangeCount);
for (int i = 0; i < rangeCount; i++)
tokenRange.serialize(rangeDeps.range(i), out, version);
int txnIdCount = rangeDeps.txnIdCount();
out.writeUnsignedVInt32(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
CommandSerializers.txnId.serialize(rangeDeps.txnId(i), out, version);
int rangesToTxnIdsCount = rangesToTxnIdsCount(rangeDeps);
out.writeUnsignedVInt32(rangesToTxnIdsCount);
for (int i = 0; i < rangesToTxnIdsCount; i++)
out.writeUnsignedVInt32(rangesToTxnIds(rangeDeps, i));
}
{
RoutingKeys keys = deps.directKeyDeps.keys();
boolean isSubset = isSubset(keys, deps.keyDeps.keys());
out.writeBoolean(isSubset);
if (isSubset) serializeSubset(keys, deps.keyDeps.keys(), out);
else KeySerializers.routingKeys.serialize(keys, out, version);
serializeKeyDepsWithoutKeys(deps.directKeyDeps, out, version);
}
}
private void serializeKeyDepsWithoutKeys(KeyDeps keyDeps, DataOutputPlus out, int version) throws IOException
{
int txnIdCount = keyDeps.txnIdCount();
out.writeUnsignedVInt32(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
CommandSerializers.txnId.serialize(keyDeps.txnId(i), out, version);
int keysToTxnIdsCount = keysToTxnIdsCount(keyDeps);
out.writeUnsignedVInt32(keysToTxnIdsCount);
for (int i = 0; i < keysToTxnIdsCount; i++)
out.writeUnsignedVInt32(keysToTxnIds(keyDeps, i));
}
private D deserializeWithoutKeys(RoutingKeys keys, DataInputPlus in, int version) throws IOException
{
KeyDeps keyDeps = deserializeKeyDeps(keys, in, version);
RangeDeps rangeDeps;
{
int rangeCount = Ints.checkedCast(in.readUnsignedVInt32());
Range[] ranges = new Range[rangeCount];
for (int i = 0; i < rangeCount; i++)
ranges[i] = tokenRange.deserialize(in, version);
int txnIdCount = in.readUnsignedVInt32();
TxnId[] txnIds = new TxnId[txnIdCount];
for (int i = 0; i < txnIdCount; i++)
txnIds[i] = CommandSerializers.txnId.deserialize(in, version);
int rangesToTxnIdsCount = in.readUnsignedVInt32();
int[] rangesToTxnIds = new int[rangesToTxnIdsCount];
for (int i = 0; i < rangesToTxnIdsCount; i++)
rangesToTxnIds[i] = in.readUnsignedVInt32();
rangeDeps = RangeDeps.SerializerSupport.create(ranges, txnIds, rangesToTxnIds);
}
KeyDeps directKeyDeps;
{
boolean isSubset = in.readBoolean();
RoutingKeys directKeys = isSubset ? (RoutingKeys) deserializeSubset(keys, in) : KeySerializers.routingKeys.deserialize(in, version);
directKeyDeps = deserializeKeyDeps(directKeys, in, version);
}
return deserialize(keyDeps, rangeDeps, directKeyDeps, in, version);
}
private long serializedSizeWithoutKeys(D deps, int version)
{
long size = serializedSizeOfKeyDepsWithoutKeys(deps.keyDeps, version);
RangeDeps rangeDeps = deps.rangeDeps;
{
int rangeCount = rangeDeps.rangeCount();
size += sizeofUnsignedVInt(rangeCount);
for (int i = 0; i < rangeCount; ++i)
size += tokenRange.serializedSize(rangeDeps.range(i), version);
int txnIdCount = rangeDeps.txnIdCount();
size += sizeofUnsignedVInt(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
size += CommandSerializers.txnId.serializedSize(rangeDeps.txnId(i), version);
int rangesToTxnIdsCount = rangesToTxnIdsCount(rangeDeps);
size += sizeofUnsignedVInt(rangesToTxnIdsCount);
for (int i = 0; i < rangesToTxnIdsCount; i++)
size += sizeofUnsignedVInt(rangesToTxnIds(rangeDeps, i));
}
{
boolean isSubset = isSubset(deps.directKeyDeps.keys(), deps.keyDeps.keys());
size += 1;
size += isSubset ? serializedSubsetSize(deps.directKeyDeps.keys(), deps.keyDeps.keys()) : KeySerializers.routingKeys.serializedSize(deps.directKeyDeps.keys(), version);
size += serializedSizeOfKeyDepsWithoutKeys(deps.directKeyDeps, version);
}
return size;
}
}
@VisibleForTesting
public static class Impl
{
final IVersionedSerializer<Range> tokenRange;
final DepsSerializer<Deps> deps;
final IVersionedSerializer<Deps> nullableDeps;
final DepsSerializer<PartialDeps> partialDeps;
final IVersionedSerializer<PartialDeps> nullablePartialDeps;
public Impl(IVersionedSerializer<Range> tokenRange)
{
this.tokenRange = tokenRange;
this.deps = new DepsSerializer<>(tokenRange)
{
@Override
Deps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version)
{
return new Deps(keyDeps, rangeDeps, directKeyDeps);
}
};
this.nullableDeps = NullableSerializer.wrap(deps);
this.partialDeps = new DepsSerializer<>(tokenRange)
{
@Override
PartialDeps deserialize(KeyDeps keyDeps, RangeDeps rangeDeps, KeyDeps directKeyDeps, DataInputPlus in, int version) throws IOException
{
Participants<?> covering = KeySerializers.participants.deserialize(in, version);
return new PartialDeps(covering, keyDeps, rangeDeps, directKeyDeps);
}
@Override
public void serialize(PartialDeps partialDeps, DataOutputPlus out, int version) throws IOException
{
super.serialize(partialDeps, out, version);
KeySerializers.participants.serialize(partialDeps.covering, out, version);
}
@Override
public void serialize(Unseekables<?> superset, PartialDeps partialDeps, DataOutputPlus out, int version) throws IOException
{
super.serialize(superset, partialDeps, out, version);
KeySerializers.participants.serialize(partialDeps.covering, out, version);
}
@Override
public long serializedSize(PartialDeps partialDeps, int version)
{
return super.serializedSize(partialDeps, version)
+ KeySerializers.participants.serializedSize(partialDeps.covering, version);
}
@Override
public long serializedSize(Unseekables<?> keys, PartialDeps partialDeps, int version)
{
return super.serializedSize(keys, partialDeps, version)
+ KeySerializers.participants.serializedSize(partialDeps.covering, version);
}
};
this.nullablePartialDeps = NullableSerializer.wrap(partialDeps);
}
}
private static KeyDeps deserializeKeyDeps(RoutingKeys keys, DataInputPlus in, int version) throws IOException
{
int txnIdCount = in.readUnsignedVInt32();
TxnId[] txnIds = new TxnId[txnIdCount];
for (int i = 0; i < txnIdCount; i++)
txnIds[i] = CommandSerializers.txnId.deserialize(in, version);
int keysToTxnIdsCount = in.readUnsignedVInt32();
int[] keysToTxnIds = new int[keysToTxnIdsCount];
for (int i = 0; i < keysToTxnIdsCount; i++)
keysToTxnIds[i] = in.readUnsignedVInt32();
return KeyDeps.SerializerSupport.create(keys, txnIds, keysToTxnIds);
}
private static long serializedSizeOfKeyDepsWithoutKeys(KeyDeps keyDeps, int version)
{
int txnIdCount = keyDeps.txnIdCount();
long size = sizeofUnsignedVInt(txnIdCount);
for (int i = 0; i < txnIdCount; i++)
size += CommandSerializers.txnId.serializedSize(keyDeps.txnId(i), version);
int keysToTxnIdsCount = keysToTxnIdsCount(keyDeps);
size += sizeofUnsignedVInt(keysToTxnIdsCount);
for (int i = 0; i < keysToTxnIdsCount; i++)
size += sizeofUnsignedVInt(keysToTxnIds(keyDeps, i));
return size;
}
private static boolean isSubset(RoutingKeys test, RoutingKeys superset)
{
return test.foldl(superset, (k, p, v, i) -> v + 1, 0, 0, 0) == test.size();
}
}

10
src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java
View File

@ -40,7 +40,7 @@ import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSi
public class FetchSerializers
{
public static final IVersionedSerializer<FetchRequest> request = new IVersionedSerializer<FetchRequest>()
public static final IVersionedSerializer<FetchRequest> request = new IVersionedSerializer<>()
{
@Override
public void serialize(FetchRequest request, DataOutputPlus out, int version) throws IOException
@ -48,7 +48,7 @@ public class FetchSerializers
out.writeUnsignedVInt(request.executeAtEpoch);
CommandSerializers.txnId.serialize(request.txnId, out, version);
KeySerializers.ranges.serialize((Ranges) request.readScope, out, version);
DepsSerializer.partialDeps.serialize(request.partialDeps, out, version);
DepsSerializers.partialDeps.serialize(request.partialDeps, out, version);
StreamingTxn.serializer.serialize(request.read, out, version);
}
@ -58,7 +58,7 @@ public class FetchSerializers
return new FetchRequest(in.readUnsignedVInt(),
CommandSerializers.txnId.deserialize(in, version),
KeySerializers.ranges.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version),
DepsSerializers.partialDeps.deserialize(in, version),
StreamingTxn.serializer.deserialize(in, version));
}
@ -68,12 +68,12 @@ public class FetchSerializers
return TypeSizes.sizeofUnsignedVInt(request.executeAtEpoch)
+ CommandSerializers.txnId.serializedSize(request.txnId, version)
+ KeySerializers.ranges.serializedSize((Ranges) request.readScope, version)
+ DepsSerializer.partialDeps.serializedSize(request.partialDeps, version)
+ DepsSerializers.partialDeps.serializedSize(request.partialDeps, version)
+ StreamingTxn.serializer.serializedSize(request.read, version);
}
};
public static final IVersionedSerializer<ReadReply> reply = new IVersionedSerializer<ReadReply>()
public static final IVersionedSerializer<ReadReply> reply = new IVersionedSerializer<>()
{
final CommitOrReadNack[] nacks = CommitOrReadNack.values();
final IVersionedSerializer<Data> streamDataSerializer = new CastingSerializer<>(StreamData.class, StreamData.serializer);

6
src/java/org/apache/cassandra/service/accord/serializers/GetEphmrlReadDepsSerializers.java
View File

@ -59,14 +59,14 @@ public class GetEphmrlReadDepsSerializers
@Override
public void serialize(GetEphemeralReadDepsOk reply, DataOutputPlus out, int version) throws IOException
{
DepsSerializer.deps.serialize(reply.deps, out, version);
DepsSerializers.deps.serialize(reply.deps, out, version);
out.writeUnsignedVInt(reply.latestEpoch);
}
@Override
public GetEphemeralReadDepsOk deserialize(DataInputPlus in, int version) throws IOException
{
Deps deps = DepsSerializer.deps.deserialize(in, version);
Deps deps = DepsSerializers.deps.deserialize(in, version);
long latestEpoch = in.readUnsignedVInt();
return new GetEphemeralReadDepsOk(deps, latestEpoch);
}
@ -74,7 +74,7 @@ public class GetEphmrlReadDepsSerializers
@Override
public long serializedSize(GetEphemeralReadDepsOk reply, int version)
{
return DepsSerializer.deps.serializedSize(reply.deps, version)
return DepsSerializers.deps.serializedSize(reply.deps, version)
+ TypeSizes.sizeofUnsignedVInt(reply.latestEpoch);
}
};

417
src/java/org/apache/cassandra/service/accord/serializers/KeySerializers.java
View File

@ -23,8 +23,11 @@ import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.IntFunction;
import com.google.common.annotations.VisibleForTesting;
import accord.api.Key;
import accord.api.RoutingKey;
import accord.primitives.AbstractKeys;
@ -57,161 +60,267 @@ import org.apache.cassandra.utils.NullableSerializer;
public class KeySerializers
{
private KeySerializers() {}
public static final AccordKeySerializer<Key> key;
public static final IVersionedSerializer<RoutingKey> routingKey;
public static final AccordKeySerializer<Key> key = (AccordKeySerializer<Key>) (AccordKeySerializer<?>) PartitionKey.serializer;
public static final IVersionedSerializer<RoutingKey> routingKey = (AccordKeySerializer<RoutingKey>) (AccordKeySerializer<?>) AccordRoutingKey.serializer;
public static final IVersionedSerializer<RoutingKey> nullableRoutingKey = NullableSerializer.wrap(routingKey);
public static final IVersionedSerializer<RoutingKey> nullableRoutingKey;
public static final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
public static final IVersionedSerializer<Keys> keys;
public static final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
public static final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
public static final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
public static final IVersionedSerializer<Range> range;
public static final AbstractRangesSerializer<Ranges> ranges;
public static final AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute;
public static final AbstractRangesSerializer<FullRangeRoute> fullRangeRoute;
public static final AbstractRoutablesSerializer<Route<?>> route;
public static final IVersionedSerializer<Route<?>> nullableRoute;
public static final IVersionedSerializer<PartialRoute<?>> partialRoute;
public static final IVersionedSerializer<FullRoute<?>> fullRoute;
public static final IVersionedSerializer<Seekables<?, ?>> seekables;
public static final IVersionedSerializer<FullRoute<?>> nullableFullRoute;
public static final IVersionedSerializer<Unseekables<?>> unseekables;
public static final IVersionedSerializer<Participants<?>> participants;
public static final IVersionedSerializer<Participants<?>> nullableParticipants;
static
{
@Override RoutingKeys deserialize(DataInputPlus in, int version, RoutingKey[] keys)
{
return RoutingKeys.SerializationSupport.create(keys);
}
};
Impl impl = new Impl();
key = impl.key;
routingKey = impl.routingKey;
public static final IVersionedSerializer<Keys> keys = new AbstractKeysSerializer<Key, Keys>(key, Key[]::new)
nullableRoutingKey = impl.nullableRoutingKey;
routingKeys = impl.routingKeys;
keys = impl.keys;
partialKeyRoute = impl.partialKeyRoute;
fullKeyRoute = impl.fullKeyRoute;
range = impl.range;
ranges = impl.ranges;
partialRangeRoute = impl.partialRangeRoute;
fullRangeRoute = impl.fullRangeRoute;
route = impl.route;
nullableRoute = impl.nullableRoute;
partialRoute = impl.partialRoute;
fullRoute = impl.fullRoute;
seekables = impl.seekables;
nullableFullRoute = impl.nullableFullRoute;
unseekables = impl.unseekables;
participants = impl.participants;
nullableParticipants = impl.nullableParticipants;
}
public static class Impl
{
@Override Keys deserialize(DataInputPlus in, int version, Key[] keys)
{
return Keys.SerializationSupport.create(keys);
}
};
final AccordKeySerializer<Key> key;
final IVersionedSerializer<RoutingKey> routingKey;
public static final AbstractRangesSerializer<Ranges> ranges = new AbstractRangesSerializer<Ranges>()
{
@Override
public Ranges deserialize(DataInputPlus in, int version, Range[] ranges)
{
return Ranges.ofSortedAndDeoverlapped(ranges);
}
};
final IVersionedSerializer<RoutingKey> nullableRoutingKey;
final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final IVersionedSerializer<Keys> keys;
public static final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute = new AbstractKeysSerializer<RoutingKey, PartialKeyRoute>(routingKey, RoutingKey[]::new)
{
@Override PartialKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final IVersionedSerializer<Range> range;
final AbstractRangesSerializer<Ranges> ranges;
final AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute;
final AbstractRangesSerializer<FullRangeRoute> fullRangeRoute;
final AbstractRoutablesSerializer<Route<?>> route;
final IVersionedSerializer<Route<?>> nullableRoute;
final IVersionedSerializer<PartialRoute<?>> partialRoute;
final IVersionedSerializer<FullRoute<?>> fullRoute;
final IVersionedSerializer<Seekables<?, ?>> seekables;
final IVersionedSerializer<FullRoute<?>> nullableFullRoute;
final IVersionedSerializer<Unseekables<?>> unseekables;
final IVersionedSerializer<Participants<?>> participants;
final IVersionedSerializer<Participants<?>> nullableParticipants;
private Impl()
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return PartialKeyRoute.SerializationSupport.create(homeKey, keys);
this((AccordKeySerializer<Key>) (AccordKeySerializer<?>) PartitionKey.serializer,
(AccordKeySerializer<RoutingKey>) (AccordKeySerializer<?>) AccordRoutingKey.serializer,
(IVersionedSerializer<Range>) (IVersionedSerializer<?>) TokenRange.serializer);
}
@Override
public void serialize(PartialKeyRoute route, DataOutputPlus out, int version) throws IOException
@VisibleForTesting
public Impl(AccordKeySerializer<Key> key,
IVersionedSerializer<RoutingKey> routingKey,
IVersionedSerializer<Range> range)
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
this.key = key;
this.routingKey = routingKey;
this.range = range;
this.nullableRoutingKey = NullableSerializer.wrap(routingKey);
this.routingKeys = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override RoutingKeys deserialize(DataInputPlus in, int version, RoutingKey[] keys)
{
return RoutingKeys.SerializationSupport.create(keys);
}
};
this.keys = new AbstractKeysSerializer<>(key, Key[]::new)
{
@Override Keys deserialize(DataInputPlus in, int version, Key[] keys)
{
return Keys.SerializationSupport.create(keys);
}
};
this.partialKeyRoute = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override PartialKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return PartialKeyRoute.SerializationSupport.create(homeKey, keys);
}
@Override
public void serialize(PartialKeyRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(PartialKeyRoute keys, int version)
{
return super.serializedSize(keys, version)
+ routingKey.serializedSize(keys.homeKey, version);
}
};
this.fullKeyRoute = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override FullKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return FullKeyRoute.SerializationSupport.create(homeKey, keys);
}
@Override
public void serialize(FullKeyRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(FullKeyRoute route, int version)
{
return super.serializedSize(route, version)
+ routingKey.serializedSize(route.homeKey, version);
}
};
this.ranges = new AbstractRangesSerializer<Ranges>(range)
{
@Override
public Ranges deserialize(DataInputPlus in, int version, Range[] ranges)
{
return Ranges.ofSortedAndDeoverlapped(ranges);
}
};
this.partialRangeRoute = new AbstractRangesSerializer<>(range)
{
@Override PartialRangeRoute deserialize(DataInputPlus in, int version, Range[] rs) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return PartialRangeRoute.SerializationSupport.create(homeKey, rs);
}
@Override
public void serialize(PartialRangeRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(PartialRangeRoute rs, int version)
{
return super.serializedSize(rs, version)
+ routingKey.serializedSize(rs.homeKey, version);
}
};
this.fullRangeRoute = new AbstractRangesSerializer<>(range)
{
@Override FullRangeRoute deserialize(DataInputPlus in, int version, Range[] Ranges) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return FullRangeRoute.SerializationSupport.create(homeKey, Ranges);
}
@Override
public void serialize(FullRangeRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(FullRangeRoute ranges, int version)
{
return super.serializedSize(ranges, version)
+ routingKey.serializedSize(ranges.homeKey(), version);
}
};
Function<EnumSet<UnseekablesKind>, AbstractRoutablesSerializer<?>> factory = (a) -> new AbstractRoutablesSerializer<>(a, routingKeys, partialKeyRoute, fullKeyRoute, ranges, partialRangeRoute, fullRangeRoute);
this.route = (AbstractRoutablesSerializer<Route<?>>) factory.apply(EnumSet.of(UnseekablesKind.PartialKeyRoute, UnseekablesKind.FullKeyRoute, UnseekablesKind.PartialRangeRoute, UnseekablesKind.FullRangeRoute));
this.nullableRoute = NullableSerializer.wrap(route);
this.partialRoute = (IVersionedSerializer<PartialRoute<?>>) factory.apply(EnumSet.of(UnseekablesKind.PartialKeyRoute, UnseekablesKind.PartialRangeRoute));
this.fullRoute = (IVersionedSerializer<FullRoute<?>>) factory.apply(EnumSet.of(UnseekablesKind.FullKeyRoute, UnseekablesKind.FullRangeRoute));
this.nullableFullRoute = NullableSerializer.wrap(fullRoute);
this.unseekables = (IVersionedSerializer<Unseekables<?>>) factory.apply(EnumSet.allOf(UnseekablesKind.class));
this.participants = (IVersionedSerializer<Participants<?>>) factory.apply(EnumSet.allOf(UnseekablesKind.class));
this.nullableParticipants = NullableSerializer.wrap(participants);
this.seekables = new AbstractSeekablesSerializer(keys, ranges);
}
@Override
public long serializedSize(PartialKeyRoute keys, int version)
{
return super.serializedSize(keys, version)
+ routingKey.serializedSize(keys.homeKey, version);
}
};
public static final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute = new AbstractKeysSerializer<>(routingKey, RoutingKey[]::new)
{
@Override FullKeyRoute deserialize(DataInputPlus in, int version, RoutingKey[] keys) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return FullKeyRoute.SerializationSupport.create(homeKey, keys);
}
@Override
public void serialize(FullKeyRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(FullKeyRoute route, int version)
{
return super.serializedSize(route, version)
+ routingKey.serializedSize(route.homeKey, version);
}
};
public static final AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute = new AbstractRangesSerializer<>()
{
@Override PartialRangeRoute deserialize(DataInputPlus in, int version, Range[] rs) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return PartialRangeRoute.SerializationSupport.create(homeKey, rs);
}
@Override
public void serialize(PartialRangeRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(PartialRangeRoute rs, int version)
{
return super.serializedSize(rs, version)
+ routingKey.serializedSize(rs.homeKey, version);
}
};
public static final AbstractRangesSerializer<FullRangeRoute> fullRangeRoute = new AbstractRangesSerializer<>()
{
@Override FullRangeRoute deserialize(DataInputPlus in, int version, Range[] Ranges) throws IOException
{
RoutingKey homeKey = routingKey.deserialize(in, version);
return FullRangeRoute.SerializationSupport.create(homeKey, Ranges);
}
@Override
public void serialize(FullRangeRoute route, DataOutputPlus out, int version) throws IOException
{
super.serialize(route, out, version);
routingKey.serialize(route.homeKey, out, version);
}
@Override
public long serializedSize(FullRangeRoute ranges, int version)
{
return super.serializedSize(ranges, version)
+ routingKey.serializedSize(ranges.homeKey(), version);
}
};
public static final AbstractRoutablesSerializer<Route<?>> route = new AbstractRoutablesSerializer<>(
EnumSet.of(UnseekablesKind.PartialKeyRoute, UnseekablesKind.FullKeyRoute, UnseekablesKind.PartialRangeRoute, UnseekablesKind.FullRangeRoute)
);
public static final IVersionedSerializer<Route<?>> nullableRoute = NullableSerializer.wrap(route);
public static final IVersionedSerializer<PartialRoute<?>> partialRoute = new AbstractRoutablesSerializer<>(
EnumSet.of(UnseekablesKind.PartialKeyRoute, UnseekablesKind.PartialRangeRoute)
);
public static final IVersionedSerializer<FullRoute<?>> fullRoute = new AbstractRoutablesSerializer<>(
EnumSet.of(UnseekablesKind.FullKeyRoute, UnseekablesKind.FullRangeRoute)
);
public static final IVersionedSerializer<FullRoute<?>> nullableFullRoute = NullableSerializer.wrap(fullRoute);
public static final IVersionedSerializer<Unseekables<?>> unseekables = new AbstractRoutablesSerializer<>(
EnumSet.allOf(UnseekablesKind.class)
);
public static final IVersionedSerializer<Participants<?>> participants = new AbstractRoutablesSerializer<>(
EnumSet.allOf(UnseekablesKind.class)
);
public static final IVersionedSerializer<Participants<?>> nullableParticipants = NullableSerializer.wrap(participants);
}
static class AbstractRoutablesSerializer<RS extends Unseekables<?>> implements IVersionedSerializer<RS>
{
final EnumSet<UnseekablesKind> permitted;
protected AbstractRoutablesSerializer(EnumSet<UnseekablesKind> permitted)
final AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys;
final AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute;
final AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute;
final AbstractRangesSerializer<Ranges> ranges;
final AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute;
final AbstractRangesSerializer<FullRangeRoute> fullRangeRoute;
protected AbstractRoutablesSerializer(EnumSet<UnseekablesKind> permitted,
AbstractKeysSerializer<RoutingKey, RoutingKeys> routingKeys,
AbstractKeysSerializer<?, PartialKeyRoute> partialKeyRoute,
AbstractKeysSerializer<?, FullKeyRoute> fullKeyRoute,
AbstractRangesSerializer<Ranges> ranges,
AbstractRangesSerializer<PartialRangeRoute> partialRangeRoute,
AbstractRangesSerializer<FullRangeRoute> fullRangeRoute)
{
this.permitted = permitted;
this.routingKeys = routingKeys;
this.partialKeyRoute = partialKeyRoute;
this.fullKeyRoute = fullKeyRoute;
this.ranges = ranges;
this.partialRangeRoute = partialRangeRoute;
this.fullRangeRoute = fullRangeRoute;
}
@Override
@ -309,8 +418,17 @@ public class KeySerializers
}
}
public static final IVersionedSerializer<Seekables<?, ?>> seekables = new IVersionedSerializer<Seekables<?, ?>>()
public static class AbstractSeekablesSerializer implements IVersionedSerializer<Seekables<?, ?>>
{
final IVersionedSerializer<Keys> keys;
final AbstractRangesSerializer<Ranges> ranges;
public AbstractSeekablesSerializer(IVersionedSerializer<Keys> keys, AbstractRangesSerializer<Ranges> ranges)
{
this.keys = keys;
this.ranges = ranges;
}
@Override
public void serialize(Seekables<?, ?> t, DataOutputPlus out, int version) throws IOException
{
@ -352,12 +470,9 @@ public class KeySerializers
return 1 + ranges.serializedSize((Ranges)t, version);
}
}
};
}
public static final IVersionedSerializer<Seekables<?, ?>> nullableSeekables = NullableSerializer.wrap(seekables);
public static final IVersionedSerializer<Unseekables<?>> nullableUnseekables = NullableSerializer.wrap(unseekables);
public static abstract class AbstractKeysSerializer<K extends RoutableKey, KS extends AbstractKeys<K>> implements IVersionedSerializer<KS>
public abstract static class AbstractKeysSerializer<K extends RoutableKey, KS extends AbstractKeys<K>> implements IVersionedSerializer<KS>
{
final IVersionedSerializer<K> keySerializer;
final IntFunction<K[]> allocate;
@ -404,21 +519,28 @@ public class KeySerializers
}
}
public static abstract class AbstractRangesSerializer<RS extends AbstractRanges> implements IVersionedSerializer<RS>
public abstract static class AbstractRangesSerializer<RS extends AbstractRanges> implements IVersionedSerializer<RS>
{
private final IVersionedSerializer<Range> rangeSerializer;
public AbstractRangesSerializer(IVersionedSerializer<Range> rangeSerializer)
{
this.rangeSerializer = rangeSerializer;
}
@Override
public void serialize(RS ranges, DataOutputPlus out, int version) throws IOException
{
out.writeUnsignedVInt32(ranges.size());
for (int i=0, mi=ranges.size(); i<mi; i++)
TokenRange.serializer.serialize((TokenRange) ranges.get(i), out, version);
rangeSerializer.serialize(ranges.get(i), out, version);
}
public void skip(DataInputPlus in, int version) throws IOException
{
int count = in.readUnsignedVInt32();
for (int i = 0; i < count ; i++)
TokenRange.serializer.deserialize(in, version);
rangeSerializer.deserialize(in, version);
}
@Override
@ -426,7 +548,7 @@ public class KeySerializers
{
Range[] ranges = new Range[in.readUnsignedVInt32()];
for (int i=0; i<ranges.length; i++)
ranges[i] = TokenRange.serializer.deserialize(in, version);
ranges[i] = rangeSerializer.deserialize(in, version);
return deserialize(in, version, ranges);
}
@ -442,6 +564,7 @@ public class KeySerializers
}
}
// TODO: port these to burn test serializers / journal
public static Map<ByteBuffer, ByteBuffer> rangesToBlobMap(Ranges ranges)
{
TreeMap<ByteBuffer, ByteBuffer> result = new TreeMap<>();

6
src/java/org/apache/cassandra/service/accord/serializers/PreacceptSerializers.java
View File

@ -83,7 +83,7 @@ public class PreacceptSerializers
PreAcceptOk preAcceptOk = (PreAcceptOk) reply;
CommandSerializers.txnId.serialize(preAcceptOk.txnId, out, version);
CommandSerializers.timestamp.serialize(preAcceptOk.witnessedAt, out, version);
DepsSerializer.deps.serialize(preAcceptOk.deps, out, version);
DepsSerializers.deps.serialize(preAcceptOk.deps, out, version);
}
@Override
@ -94,7 +94,7 @@ public class PreacceptSerializers
return new PreAcceptOk(CommandSerializers.txnId.deserialize(in, version),
CommandSerializers.timestamp.deserialize(in, version),
DepsSerializer.deps.deserialize(in, version));
DepsSerializers.deps.deserialize(in, version));
}
@Override
@ -107,7 +107,7 @@ public class PreacceptSerializers
PreAcceptOk preAcceptOk = (PreAcceptOk) reply;
size += CommandSerializers.txnId.serializedSize(preAcceptOk.txnId, version);
size += CommandSerializers.timestamp.serializedSize(preAcceptOk.witnessedAt, version);
size += DepsSerializer.deps.serializedSize(preAcceptOk.deps, version);
size += DepsSerializers.deps.serializedSize(preAcceptOk.deps, version);
return size;
}

14
src/java/org/apache/cassandra/service/accord/serializers/ReadDataSerializers.java
View File

@ -84,7 +84,7 @@ public class ReadDataSerializers
CommandSerializers.timestamp.serialize(msg.executeAt, out, version);
KeySerializers.fullRoute.serialize(msg.route, out, version);
CommandSerializers.partialTxn.serialize(msg.txn, out, version);
DepsSerializer.partialDeps.serialize(msg.deps, out, version);
DepsSerializers.partialDeps.serialize(msg.deps, out, version);
CommandSerializers.writes.serialize(msg.writes, out, version);
}
@ -98,9 +98,9 @@ public class ReadDataSerializers
CommandSerializers.timestamp.deserialize(in, version),
KeySerializers.fullRoute.deserialize(in, version),
CommandSerializers.partialTxn.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version),
DepsSerializers.partialDeps.deserialize(in, version),
CommandSerializers.writes.deserialize(in, version),
CommandSerializers.APPLIED);
ResultSerializers.APPLIED);
}
@Override
@ -112,7 +112,7 @@ public class ReadDataSerializers
+ CommandSerializers.timestamp.serializedSize(msg.executeAt, version)
+ KeySerializers.fullRoute.serializedSize(msg.route, version)
+ CommandSerializers.partialTxn.serializedSize(msg.txn, version)
+ DepsSerializer.partialDeps.serializedSize(msg.deps, version)
+ DepsSerializers.partialDeps.serializedSize(msg.deps, version)
+ CommandSerializers.writes.serializedSize(msg.writes, version);
}
}
@ -154,7 +154,7 @@ public class ReadDataSerializers
KeySerializers.participants.serialize(read.readScope, out, version);
out.writeUnsignedVInt(read.executeAtEpoch);
CommandSerializers.partialTxn.serialize(read.partialTxn(), out, version);
DepsSerializer.partialDeps.serialize(read.partialDeps(), out, version);
DepsSerializers.partialDeps.serialize(read.partialDeps(), out, version);
KeySerializers.fullRoute.serialize(read.route(), out, version);
}
@ -165,7 +165,7 @@ public class ReadDataSerializers
Participants<?> readScope = KeySerializers.participants.deserialize(in, version);
long executeAtEpoch = in.readUnsignedVInt();
PartialTxn partialTxn = CommandSerializers.partialTxn.deserialize(in, version);
PartialDeps partialDeps = DepsSerializer.partialDeps.deserialize(in, version);
PartialDeps partialDeps = DepsSerializers.partialDeps.deserialize(in, version);
FullRoute<?> route = KeySerializers.fullRoute.deserialize(in, version);
return ReadEphemeralTxnData.SerializerSupport.create(txnId, readScope, executeAtEpoch, partialTxn, partialDeps, route);
}
@ -177,7 +177,7 @@ public class ReadDataSerializers
+ KeySerializers.participants.serializedSize(read.readScope, version)
+ TypeSizes.sizeofUnsignedVInt(read.executeAtEpoch)
+ CommandSerializers.partialTxn.serializedSize(read.partialTxn(), version)
+ DepsSerializer.partialDeps.serializedSize(read.partialDeps(), version)
+ DepsSerializers.partialDeps.serializedSize(read.partialDeps(), version)
+ KeySerializers.fullRoute.serializedSize(read.route(), version);
}
};

26
src/java/org/apache/cassandra/service/accord/serializers/RecoverySerializers.java
View File

@ -96,8 +96,8 @@ public class RecoverySerializers
CommandSerializers.ballot.serialize(recoverOk.accepted, out, version);
CommandSerializers.nullableTimestamp.serialize(recoverOk.executeAt, out, version);
latestDeps.serialize(recoverOk.deps, out, version);
DepsSerializer.deps.serialize(recoverOk.earlierCommittedWitness, out, version);
DepsSerializer.deps.serialize(recoverOk.earlierAcceptedNoWitness, out, version);
DepsSerializers.deps.serialize(recoverOk.earlierCommittedWitness, out, version);
DepsSerializers.deps.serialize(recoverOk.earlierAcceptedNoWitness, out, version);
out.writeBoolean(recoverOk.acceptsFastPath);
out.writeBoolean(recoverOk.rejectsFastPath);
CommandSerializers.nullableWrites.serialize(recoverOk.writes, out, version);
@ -135,15 +135,15 @@ public class RecoverySerializers
Result result = null;
if (status == Status.PreApplied || status == Status.Applied || status == Status.Truncated)
result = CommandSerializers.APPLIED;
result = ResultSerializers.APPLIED;
return deserializeOk(id,
status,
CommandSerializers.ballot.deserialize(in, version),
CommandSerializers.nullableTimestamp.deserialize(in, version),
latestDeps.deserialize(in, version),
DepsSerializer.deps.deserialize(in, version),
DepsSerializer.deps.deserialize(in, version),
DepsSerializers.deps.deserialize(in, version),
DepsSerializers.deps.deserialize(in, version),
in.readBoolean(),
in.readBoolean(),
CommandSerializers.nullableWrites.deserialize(in, version),
@ -164,8 +164,8 @@ public class RecoverySerializers
size += CommandSerializers.ballot.serializedSize(recoverOk.accepted, version);
size += CommandSerializers.nullableTimestamp.serializedSize(recoverOk.executeAt, version);
size += latestDeps.serializedSize(recoverOk.deps, version);
size += DepsSerializer.deps.serializedSize(recoverOk.earlierCommittedWitness, version);
size += DepsSerializer.deps.serializedSize(recoverOk.earlierAcceptedNoWitness, version);
size += DepsSerializers.deps.serializedSize(recoverOk.earlierCommittedWitness, version);
size += DepsSerializers.deps.serializedSize(recoverOk.earlierAcceptedNoWitness, version);
size += TypeSizes.sizeof(recoverOk.acceptsFastPath);
size += TypeSizes.sizeof(recoverOk.rejectsFastPath);
size += CommandSerializers.nullableWrites.serializedSize(recoverOk.writes, version);
@ -199,8 +199,8 @@ public class RecoverySerializers
{
CommandSerializers.nullableKnownDeps.serialize(e.known, out, version);
CommandSerializers.ballot.serialize(e.ballot, out, version);
DepsSerializer.nullableDeps.serialize(e.coordinatedDeps, out, version);
DepsSerializer.nullableDeps.serialize(e.localDeps, out, version);
DepsSerializers.nullableDeps.serialize(e.coordinatedDeps, out, version);
DepsSerializers.nullableDeps.serialize(e.localDeps, out, version);
}
}
KeySerializers.routingKey.serialize(t.startAt(t.size()), out, version);
@ -220,8 +220,8 @@ public class RecoverySerializers
continue;
Ballot ballot = CommandSerializers.ballot.deserialize(in, version);
Deps coordinatedDeps = DepsSerializer.nullableDeps.deserialize(in, version);
Deps localDeps = DepsSerializer.nullableDeps.deserialize(in, version);
Deps coordinatedDeps = DepsSerializers.nullableDeps.deserialize(in, version);
Deps localDeps = DepsSerializers.nullableDeps.deserialize(in, version);
values[i] = new LatestDeps.LatestEntry(knownDeps, ballot, coordinatedDeps, localDeps);
}
starts[size] = KeySerializers.routingKey.deserialize(in, version);
@ -247,8 +247,8 @@ public class RecoverySerializers
{
size += CommandSerializers.nullableKnownDeps.serializedSize(e.known, version);
size += CommandSerializers.ballot.serializedSize(e.ballot, version);
size += DepsSerializer.nullableDeps.serializedSize(e.coordinatedDeps, version);
size += DepsSerializer.nullableDeps.serializedSize(e.localDeps, version);
size += DepsSerializers.nullableDeps.serializedSize(e.coordinatedDeps, version);
size += DepsSerializers.nullableDeps.serializedSize(e.localDeps, version);
}
}
size += KeySerializers.routingKey.serializedSize(t.startAt(t.size()), version);

52
src/java/org/apache/cassandra/service/accord/serializers/ResultSerializers.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord.serializers;
import accord.api.Result;
import accord.primitives.ProgressToken;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
public class ResultSerializers
{
// TODO (expected): this is meant to encode e.g. whether the transaction's condition met or not
public static final Result APPLIED = new Result()
{
@Override
public ProgressToken asProgressToken()
{
return ProgressToken.APPLIED;
}
};
public static final IVersionedSerializer<Result> result = new IVersionedSerializer<>()
{
public void serialize(Result t, DataOutputPlus out, int version) { }
public Result deserialize(DataInputPlus in, int version)
{
return APPLIED;
}
public long serializedSize(Result t, int version)
{
return 0;
}
};
}

12
src/java/org/apache/cassandra/service/accord/serializers/SetDurableSerializers.java
View File

@ -31,7 +31,7 @@ import org.apache.cassandra.io.util.DataOutputPlus;
public class SetDurableSerializers
{
public static final IVersionedSerializer<SetShardDurable> shardDurable = new IVersionedSerializer<SetShardDurable>()
public static final IVersionedSerializer<SetShardDurable> shardDurable = new IVersionedSerializer<>()
{
@Override
public void serialize(SetShardDurable msg, DataOutputPlus out, int version) throws IOException
@ -52,7 +52,7 @@ public class SetDurableSerializers
}
};
public static final IVersionedSerializer<SetGloballyDurable> globallyDurable = new IVersionedSerializer<SetGloballyDurable>()
public static final IVersionedSerializer<SetGloballyDurable> globallyDurable = new IVersionedSerializer<>()
{
@Override
public void serialize(SetGloballyDurable msg, DataOutputPlus out, int version) throws IOException
@ -73,13 +73,13 @@ public class SetDurableSerializers
}
};
public static final IVersionedSerializer<SyncPoint> syncPoint = new IVersionedSerializer<SyncPoint>()
public static final IVersionedSerializer<SyncPoint> syncPoint = new IVersionedSerializer<>()
{
@Override
public void serialize(SyncPoint sp, DataOutputPlus out, int version) throws IOException
{
CommandSerializers.txnId.serialize(sp.syncId, out, version);
DepsSerializer.deps.serialize(sp.waitFor, out, version);
DepsSerializers.deps.serialize(sp.waitFor, out, version);
KeySerializers.fullRoute.serialize(sp.route, out, version);
}
@ -87,7 +87,7 @@ public class SetDurableSerializers
public SyncPoint deserialize(DataInputPlus in, int version) throws IOException
{
TxnId syncId = CommandSerializers.txnId.deserialize(in, version);
Deps waitFor = DepsSerializer.deps.deserialize(in, version);
Deps waitFor = DepsSerializers.deps.deserialize(in, version);
FullRoute<?> route = KeySerializers.fullRoute.deserialize(in, version);
return SyncPoint.SerializationSupport.construct(syncId, waitFor, route);
}
@ -96,7 +96,7 @@ public class SetDurableSerializers
public long serializedSize(SyncPoint sp, int version)
{
return CommandSerializers.txnId.serializedSize(sp.syncId, version)
+ DepsSerializer.deps.serializedSize(sp.waitFor, version)
+ DepsSerializers.deps.serializedSize(sp.waitFor, version)
+ KeySerializers.fullRoute.serializedSize(sp.route, version);
}
};

60
src/java/org/apache/cassandra/tools/FieldUtil.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
public class FieldUtil
{
public static void setInstanceUnsafe(Class<?> klass, Object v, String fieldName)
{
try
{
Field field = klass.getDeclaredField(fieldName);
field.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(null, v);
}
catch (NoSuchFieldException | IllegalAccessException t)
{
throw new RuntimeException(t);
}
}
public static void transferFields(Object sourceInstance, Class<?> klass)
{
for (Field sourceField : sourceInstance.getClass().getDeclaredFields())
{
sourceField.setAccessible(true);
try
{
setInstanceUnsafe(klass, sourceField.get(sourceInstance), sourceField.getName());
}
catch (IllegalAccessException e)
{
throw new RuntimeException("Failed to transfer field: " + sourceField.getName(), e);
}
}
}
}

181
test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java Normal file
View File

@ -0,0 +1,181 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.nio.file.Files;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.burn.BurnTestBase;
import accord.burn.SimulationException;
import accord.impl.TopologyFactory;
import accord.impl.basic.Cluster;
import accord.impl.basic.RandomDelayQueue;
import accord.local.Node;
import accord.utils.DefaultRandom;
import accord.utils.RandomSource;
import org.apache.cassandra.ServerTestUtils;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.journal.TestParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.schema.Tables;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.DepsSerializers;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.tools.FieldUtil;
import static accord.impl.PrefixedIntHashKey.ranges;
public class AccordJournalBurnTest extends BurnTestBase
{
private static final Logger logger = LoggerFactory.getLogger(AccordJournalBurnTest.class);
public static void setUp() throws Throwable
{
StorageService.instance.registerMBeans();
StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance);
ServerTestUtils.prepareServerNoRegister();
Keyspace.setInitialized();
FieldUtil.transferFields(new KeySerializers.Impl(BurnTestKeySerializers.key,
BurnTestKeySerializers.routingKey,
BurnTestKeySerializers.range),
KeySerializers.class);
FieldUtil.transferFields(new CommandSerializers.QuerySerializers(BurnTestKeySerializers.read,
BurnTestKeySerializers.query,
BurnTestKeySerializers.update,
BurnTestKeySerializers.write),
CommandSerializers.class);
FieldUtil.transferFields(new DepsSerializers.Impl(BurnTestKeySerializers.range),
DepsSerializers.class);
FieldUtil.setInstanceUnsafe(ResultSerializers.class,
BurnTestKeySerializers.result,
"result");
}
private AtomicInteger counter = new AtomicInteger();
@Before
public void beforeTest() throws Throwable
{
}
@Test
public void testOne()
{
long seed = System.nanoTime();
int operations = 1000;
logger.info("Seed: {}", seed);
Cluster.trace.trace("Seed: {}", seed);
RandomSource random = new DefaultRandom(seed);
try
{
List<Node.Id> clients = generateIds(true, 1 + random.nextInt(4));
int rf;
float chance = random.nextFloat();
if (chance < 0.2f) { rf = random.nextInt(2, 9); }
else if (chance < 0.4f) { rf = 3; }
else if (chance < 0.7f) { rf = 5; }
else if (chance < 0.8f) { rf = 7; }
else { rf = 9; }
List<Node.Id> nodes = generateIds(false, random.nextInt(rf, rf * 3));
{
ServerTestUtils.daemonInitialization();
TableMetadata[] metadatas = new TableMetadata[5 + nodes.size()];
metadatas[0] = AccordKeyspace.Commands;
metadatas[1] = AccordKeyspace.TimestampsForKeys;
metadatas[2] = AccordKeyspace.CommandsForKeys;
metadatas[3] = AccordKeyspace.Topologies;
metadatas[4] = AccordKeyspace.EpochMetadata;
for (int i = 0; i < nodes.size(); i++)
metadatas[5 + i] = AccordKeyspace.journalMetadata("journal_" + nodes.get(i));
AccordKeyspace.tables = Tables.of(metadatas);
setUp();
}
Keyspace ks = Schema.instance.getKeyspaceInstance("system_accord");
burn(random, new TopologyFactory(rf, ranges(0, HASH_RANGE_START, HASH_RANGE_END, random.nextInt(Math.max(nodes.size() + 1, rf), nodes.size() * 3))),
clients,
nodes,
5 + random.nextInt(15),
5 + random.nextInt(15),
operations,
10 + random.nextInt(30),
new RandomDelayQueue.Factory(random).get(),
(node) -> {
try
{
File directory = new File(Files.createTempDirectory(Integer.toString(counter.incrementAndGet())));
directory.deleteRecursiveOnExit();
ColumnFamilyStore cfs = ks.getColumnFamilyStore("journal_" + node);
cfs.disableAutoCompaction();
AccordJournal journal = new AccordJournal(new TestParams()
{
@Override
public int segmentSize()
{
return 32 * 1024 * 1024;
}
@Override
public boolean enableCompaction()
{
return false;
}
}, new AccordAgent(), directory, cfs);
journal.start(null);
journal.unsafeSetStarted();
return journal;
}
catch (Throwable t)
{
throw new RuntimeException(t);
}
}
);
}
catch (Throwable t)
{
logger.error("Exception running burn test for seed {}:", seed, t);
throw SimulationException.wrap(seed, t);
}
}
}

10
test/distributed/org/apache/cassandra/service/accord/AccordJournalCompactionTest.java
View File

@ -52,9 +52,9 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordAgent;
import org.apache.cassandra.utils.AccordGenerators;
import static accord.api.Journal.FieldUpdates;
import static accord.local.CommandStores.RangesForEpoch;
import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.DurableBeforeAccumulator;
import static org.apache.cassandra.service.accord.AccordJournalValueSerializers.RedundantBeforeAccumulator;
public class AccordJournalCompactionTest
@ -86,7 +86,7 @@ public class AccordJournalCompactionTest
ColumnFamilyStore cfs = Keyspace.open(SchemaConstants.ACCORD_KEYSPACE_NAME).getColumnFamilyStore(AccordKeyspace.JOURNAL);
cfs.disableAutoCompaction();
RedundantBeforeAccumulator redundantBeforeAccumulator = new RedundantBeforeAccumulator();
RedundantBefore redundantBeforeAccumulator = RedundantBefore.EMPTY;
DurableBeforeAccumulator durableBeforeAccumulator = new DurableBeforeAccumulator();
NavigableMap<Timestamp, Ranges> safeToReadAtAccumulator = ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY);
NavigableMap<TxnId, Ranges> bootstrapBeganAtAccumulator = ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY);
@ -126,7 +126,7 @@ public class AccordJournalCompactionTest
for (int i = 0; i <= count; i++)
{
timestamp = timestamp.next();
AccordSafeCommandStore.FieldUpdates updates = new AccordSafeCommandStore.FieldUpdates();
FieldUpdates updates = new FieldUpdates();
DurableBefore addDurableBefore = durableBeforeGen.next(rs);
// TODO: improve redundant before generator and re-enable
// updates.addRedundantBefore = redundantBeforeGen.next(rs);
@ -135,9 +135,9 @@ public class AccordJournalCompactionTest
updates.newRangesForEpoch = rangesForEpochGen.next(rs);
journal.durableBeforePersister().persist(addDurableBefore, null);
journal.persistStoreState(1, updates, null);
journal.saveStoreState(1, updates, null);
redundantBeforeAccumulator.update(updates.newRedundantBefore);
redundantBeforeAccumulator = updates.newRedundantBefore;
durableBeforeAccumulator.update(addDurableBefore);
if (updates.newBootstrapBeganAt != null)
bootstrapBeganAtAccumulator = updates.newBootstrapBeganAt;

331
test/distributed/org/apache/cassandra/service/accord/BurnTestKeySerializers.java Normal file
View File

@ -0,0 +1,331 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.io.IOException;
import java.util.Map;
import java.util.function.Function;
import accord.api.Key;
import accord.api.Query;
import accord.api.Read;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.api.Update;
import accord.api.Write;
import accord.impl.PrefixedIntHashKey;
import accord.impl.list.ListQuery;
import accord.impl.list.ListRead;
import accord.impl.list.ListResult;
import accord.impl.list.ListUpdate;
import accord.impl.list.ListWrite;
import accord.local.Node;
import accord.primitives.Keys;
import accord.primitives.Range;
import accord.primitives.Seekables;
import accord.primitives.TxnId;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.accord.api.AccordRoutableKey;
import org.apache.cassandra.service.accord.api.AccordRoutableKey.AccordKeySerializer;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.KeySerializers;
import org.apache.cassandra.service.accord.serializers.TopologySerializers;
import org.apache.cassandra.utils.CastingSerializer;
@SuppressWarnings("unchecked")
public class BurnTestKeySerializers
{
private BurnTestKeySerializers() {}
public static final AccordRoutableKey.AccordKeySerializer<Key> key =
(AccordRoutableKey.AccordKeySerializer<Key>)
(AccordRoutableKey.AccordKeySerializer<?>)
new AccordRoutableKey.AccordKeySerializer<PrefixedIntHashKey>()
{
public void serialize(PrefixedIntHashKey t, DataOutputPlus out, int version) throws IOException
{
assert t instanceof PrefixedIntHashKey.Key;
out.writeInt(t.prefix);
out.writeInt(t.key);
out.writeInt(t.hash);
}
public PrefixedIntHashKey deserialize(DataInputPlus in, int version) throws IOException
{
int prefix = in.readInt();
int key = in.readInt();
int hash = in.readInt();
return PrefixedIntHashKey.key(prefix, key, hash);
}
public long serializedSize(PrefixedIntHashKey t, int version)
{
return 3 * Integer.BYTES;
}
public void skip(DataInputPlus in, int version) throws IOException
{
in.skipBytesFully(3 * Integer.BYTES);
}
};
public static final IVersionedSerializer<RoutingKey> routingKey =
(AccordKeySerializer<RoutingKey>)
(AccordKeySerializer<?>)
new AccordRoutableKey.AccordKeySerializer<PrefixedIntHashKey.Hash>()
{
public void serialize(PrefixedIntHashKey.Hash t, DataOutputPlus out, int version) throws IOException
{
out.writeInt(t.prefix);
out.writeInt(t.hash);
}
public PrefixedIntHashKey.Hash deserialize(DataInputPlus in, int version) throws IOException
{
int prefix = in.readInt();
int hash = in.readInt();
return new PrefixedIntHashKey.Hash(prefix, hash);
}
public long serializedSize(PrefixedIntHashKey.Hash t, int version)
{
return 2 * Integer.BYTES;
}
public void skip(DataInputPlus in, int version) throws IOException
{
in.skipBytesFully(2 * Integer.BYTES);
}
};
public static final IVersionedSerializer<Range> range =
(IVersionedSerializer<Range>)
(IVersionedSerializer<?>)
new IVersionedSerializer<PrefixedIntHashKey.Range>()
{
@Override
public void serialize(PrefixedIntHashKey.Range t, DataOutputPlus out, int version) throws IOException
{
routingKey.serialize(t.start(), out, version);
routingKey.serialize(t.end(), out, version);
}
@Override
public PrefixedIntHashKey.Range deserialize(DataInputPlus in, int version) throws IOException
{
RoutingKey start = routingKey.deserialize(in, version);
RoutingKey end = routingKey.deserialize(in, version);
return new PrefixedIntHashKey.Range((PrefixedIntHashKey.PrefixedIntRoutingKey) start, (PrefixedIntHashKey.PrefixedIntRoutingKey) end);
}
@Override
public long serializedSize(PrefixedIntHashKey.Range t, int version)
{
throw new RuntimeException("not implemented");
}
};
public static final IVersionedSerializer<Read> read = new CastingSerializer<>(ListRead.class, new IVersionedSerializer<>()
{
public void serialize(ListRead t, DataOutputPlus out, int version) throws IOException
{
out.writeBoolean(t.isEphemeralRead);
KeySerializers.seekables.serialize(t.userReadKeys, out, version);
KeySerializers.seekables.serialize(t.keys, out, version);
}
public ListRead deserialize(DataInputPlus in, int version) throws IOException
{
boolean isEphemeralRead = in.readBoolean();
Seekables<?, ?> userReadKeys = KeySerializers.seekables.deserialize(in, version);
Seekables<?, ?> keys = KeySerializers.seekables.deserialize(in, version);
return new ListRead(Function.identity(), isEphemeralRead, userReadKeys, keys);
}
public long serializedSize(ListRead t, int version)
{
throw new RuntimeException("not implemented");
}
});
public static final IVersionedSerializer<Query> query = new CastingSerializer<>(ListQuery.class, new IVersionedSerializer<>()
{
public void serialize(ListQuery t, DataOutputPlus out, int version) throws IOException
{
if (t == null)
{
out.writeByte(0);
return;
}
out.writeByte(1);
TopologySerializers.NodeIdSerializer.serialize(t.client, out);
out.writeLong(t.requestId);
out.writeBoolean(t.isEphemeralRead);
}
public ListQuery deserialize(DataInputPlus in, int version) throws IOException
{
switch (in.readByte())
{
case 0:
return null;
case 1:
break;
default:
throw new AssertionError();
}
Node.Id client = TopologySerializers.NodeIdSerializer.deserialize(in);
long requestId = in.readLong();
boolean isEphemeralRead = in.readBoolean();
return new ListQuery(client, requestId, isEphemeralRead);
}
public long serializedSize(ListQuery t, int version)
{
throw new RuntimeException("not implemented");
}
});
public static final IVersionedSerializer<Update> update = new CastingSerializer<>(ListUpdate.class, new IVersionedSerializer<>()
{
public void serialize(ListUpdate t, DataOutputPlus out, int version) throws IOException
{
out.writeInt(t.size());
for (Map.Entry<Key, Integer> e : t.entrySet())
{
KeySerializers.key.serialize(e.getKey(), out, version);
out.writeInt(e.getValue());
}
}
public ListUpdate deserialize(DataInputPlus in, int version) throws IOException
{
int size = in.readInt();
ListUpdate listUpdate = new ListUpdate(Function.identity());
for (int i = 0; i < size; i++)
{
Key k = KeySerializers.key.deserialize(in, version);
int v = in.readInt();
listUpdate.put(k, v);
}
return listUpdate;
}
public long serializedSize(ListUpdate t, int version)
{
throw new RuntimeException("not implemented");
}
});
public static final IVersionedSerializer<Write> write = new CastingSerializer<>(ListWrite.class, new IVersionedSerializer<>()
{
public void serialize(ListWrite t, DataOutputPlus out, int version) throws IOException
{
out.writeInt(t.size());
for (Map.Entry<Key, int[]> e : t.entrySet())
{
KeySerializers.key.serialize(e.getKey(), out, version);
out.writeInt(e.getValue().length);
for (int v : e.getValue())
out.writeInt(v);
}
}
public ListWrite deserialize(DataInputPlus in, int version) throws IOException
{
int size = in.readInt();
ListWrite write = new ListWrite(Function.identity());
for (int i = 0; i < size; i++)
{
Key k = KeySerializers.key.deserialize(in, version);
int len = in.readInt();
int[] vals = new int[len];
for (int j = 0; j < len; j++)
vals[j] = in.readInt();
write.put(k, vals);
}
return write;
}
public long serializedSize(ListWrite t, int version)
{
throw new RuntimeException("not implemented");
}
});
public static final IVersionedSerializer<Result> result = new CastingSerializer<>(ListResult.class, new IVersionedSerializer<>()
{
public void serialize(ListResult t, DataOutputPlus out, int version) throws IOException
{
TopologySerializers.NodeIdSerializer.serialize(t.client, out);
out.writeLong(t.requestId);
CommandSerializers.txnId.serialize(t.txnId, out, version);
KeySerializers.seekables.serialize(t.readKeys, out, version);
KeySerializers.keys.serialize(t.responseKeys, out, version);
out.writeInt(t.read.length);
for (int[] ints : t.read)
{
out.writeInt(ints.length);
for (int i : ints)
out.writeInt(i);
}
out.writeInt(t.update == null ? 0 : 1);
if (t.update != null)
update.serialize(t.update, out, version);
out.writeInt(t.status.ordinal());
}
public ListResult deserialize(DataInputPlus in, int version) throws IOException
{
Node.Id client = TopologySerializers.NodeIdSerializer.deserialize(in);
long requestId = in.readLong();
TxnId txnId = CommandSerializers.txnId.deserialize(in);
Seekables<?, ?> readKeys = KeySerializers.seekables.deserialize(in, version);
Keys responseKeys = KeySerializers.keys.deserialize(in, version);
int[][] read = new int[in.readInt()][];
for (int i = 0; i < read.length; i++)
{
int[] v = new int[in.readInt()];
for (int j = 0; j < v.length; j++)
{
v[j] = in.readInt();
}
read[i] = v;
}
ListUpdate update = null;
if (in.readInt() != 0)
update = (ListUpdate) BurnTestKeySerializers.update.deserialize(in, version);
ListResult.Status status = ListResult.Status.values()[in.readInt()];
return new ListResult(status, client, requestId, txnId, readKeys, responseKeys, read, update);
}
public long serializedSize(ListResult t, int version)
{
throw new RuntimeException("not implemented");
}
});
}

2
test/unit/org/apache/cassandra/Util.java
View File

@ -1,4 +1,3 @@
package org.apache.cassandra;
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
@ -18,6 +17,7 @@ package org.apache.cassandra;
* under the License.
*
*/
package org.apache.cassandra;
import java.io.Closeable;
import java.io.DataInputStream;

6
test/unit/org/apache/cassandra/db/virtual/AccordDebugKeyspaceTest.java
View File

@ -31,7 +31,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.Utils;
import accord.messages.TxnRequest;
import accord.primitives.Routable;
import accord.primitives.SaveStatus;
@ -53,6 +52,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.concurrent.Condition;
import org.awaitility.Awaitility;
import static org.apache.cassandra.Util.spinUntilSuccess;
import static org.apache.cassandra.service.accord.AccordTestUtils.createTxn;
public class AccordDebugKeyspaceTest extends CQLTester
@ -92,8 +92,8 @@ public class AccordDebugKeyspaceTest extends CQLTester
Txn txn = createTxn(wrapInTxn(String.format("INSERT INTO %s.%s(k, c, v) VALUES (?, ?, ?)", KEYSPACE, tableName)), 0, 0, 0);
AsyncChains.getBlocking(accord.node().coordinate(id, txn));
Utils.spinUntilSuccess(() -> assertRows(execute(QUERY_TXN_BLOCKED_BY, id.toString()),
row(id.toString(), anyInt(), 0, ByteBufferUtil.EMPTY_BYTE_BUFFER, "Self", any(), null, anyOf(SaveStatus.ReadyToExecute.name(), SaveStatus.Applying.name(), SaveStatus.Applied.name()))));
spinUntilSuccess(() -> assertRows(execute(QUERY_TXN_BLOCKED_BY, id.toString()),
row(id.toString(), anyInt(), 0, ByteBufferUtil.EMPTY_BYTE_BUFFER, "Self", any(), null, anyOf(SaveStatus.ReadyToExecute.name(), SaveStatus.Applying.name(), SaveStatus.Applied.name()))));
}
@Test

10
test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
View File

@ -34,10 +34,9 @@ import accord.api.Result;
import accord.impl.TimestampsForKey;
import accord.impl.TimestampsForKeys;
import accord.local.Command;
import accord.local.CommonAttributes;
import accord.local.StoreParticipants;
import accord.local.cfk.CommandsForKey;
import accord.local.CommonAttributes;
import accord.primitives.SaveStatus;
import accord.primitives.Ballot;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
@ -46,6 +45,7 @@ import accord.primitives.Ranges;
import accord.primitives.Routable;
import accord.primitives.Route;
import accord.primitives.RoutingKeys;
import accord.primitives.SaveStatus;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
@ -64,7 +64,7 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.api.AccordRoutingKey.TokenKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.ResultSerializers;
import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.utils.Pair;
@ -140,7 +140,7 @@ public class AccordCommandStoreTest
Pair<Writes, Result> result = AccordTestUtils.processTxnResult(commandStore, txnId, txn, executeAt);
Command expected = Command.SerializerSupport.executed(attrs, SaveStatus.Applied, executeAt, promised, accepted,
waitingOn, result.left, CommandSerializers.APPLIED);
waitingOn, result.left, ResultSerializers.APPLIED);
AccordSafeCommand safeCommand = new AccordSafeCommand(loaded(txnId, null));
safeCommand.set(expected);
@ -159,8 +159,6 @@ public class AccordCommandStoreTest
{
AtomicLong clock = new AtomicLong(0);
AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
// SafeCommandStore safeStore =
Timestamp maxTimestamp = timestamp(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(1);
TokenKey key = ((PartitionKey) getOnlyElement(txn.keys())).toUnseekable();

8
test/unit/org/apache/cassandra/service/accord/AccordJournalOrderTest.java
View File

@ -26,6 +26,7 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import accord.api.Journal;
import accord.local.Command;
import accord.local.CommonAttributes;
import accord.primitives.Ballot;
@ -47,7 +48,6 @@ import org.apache.cassandra.service.consensus.TransactionalMode;
import org.apache.cassandra.utils.StorageCompatibilityMode;
import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
import static org.apache.cassandra.service.accord.SavedCommand.getFlags;
public class AccordJournalOrderTest
{
@ -81,9 +81,9 @@ public class AccordJournalOrderTest
JournalKey key = new JournalKey(txnId, JournalKey.Type.COMMAND_DIFF, randomSource.nextInt(5));
res.compute(key, (k, prev) -> prev == null ? 1 : prev + 1);
Command command = Command.NotDefined.notDefined(new CommonAttributes.Mutable(txnId), Ballot.ZERO);
accordJournal.appendCommand(key.commandStoreId,
new SavedCommand.Writer(command, getFlags(null, command)),
() -> {});
accordJournal.saveCommand(key.commandStoreId,
new Journal.CommandUpdate(null, command),
() -> {});
}
Runnable check = () -> {

9
test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
View File

@ -35,6 +35,7 @@ import com.google.common.collect.Sets;
import org.junit.Assert;
import accord.api.Data;
import accord.api.Journal;
import accord.api.ProgressLog.NoOpProgressLog;
import accord.api.RemoteListeners.NoOpRemoteListeners;
import accord.api.Result;
@ -504,15 +505,9 @@ public class AccordTestUtils
return range(token(left), token(right));
}
public static void appendCommandsBlocking(AccordCommandStore commandStore, Command after)
{
appendCommandsBlocking(commandStore, null, after);
}
public static void appendCommandsBlocking(AccordCommandStore commandStore, Command before, Command after)
{
SavedCommand.Writer diff = SavedCommand.diff(before, after);
if (diff == null) return;
Journal.CommandUpdate diff = new Journal.CommandUpdate(before, after);
Condition condition = Condition.newOneTimeCondition();
commandStore.appendCommands(Collections.singletonList(diff), condition::signal);
condition.awaitUninterruptibly(30, TimeUnit.SECONDS);

422
test/unit/org/apache/cassandra/service/accord/MockJournal.java
View File

@ -1,422 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSortedMap;
import accord.api.Result;
import accord.local.Command;
import accord.local.CommandStores;
import accord.local.CommonAttributes;
import accord.local.DurableBefore;
import accord.local.RedundantBefore;
import accord.local.StoreParticipants;
import accord.primitives.Known;
import accord.primitives.SaveStatus;
import accord.primitives.Status;
import accord.primitives.Ballot;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Ranges;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.Invariants;
import accord.utils.PersistentField.Persister;
import accord.utils.async.AsyncResult;
import accord.utils.async.AsyncResults;
import org.apache.cassandra.service.accord.AccordJournalValueSerializers.DurableBeforeAccumulator;
import org.apache.cassandra.service.accord.AccordJournalValueSerializers.IdentityAccumulator;
import org.apache.cassandra.service.accord.AccordJournalValueSerializers.RedundantBeforeAccumulator;
import org.apache.cassandra.service.accord.SavedCommand.Load;
import org.apache.cassandra.service.accord.SavedCommand.MinimalCommand;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
public class MockJournal implements IJournal
{
private final Map<JournalKey, List<LoadedDiff>> commands = new HashMap<>();
private static class FieldUpdates
{
final RedundantBeforeAccumulator redundantBeforeAccumulator = new RedundantBeforeAccumulator();
final IdentityAccumulator<NavigableMap<TxnId, Ranges>> bootstrapBeganAtAccumulator = new IdentityAccumulator<>(ImmutableSortedMap.of(TxnId.NONE, Ranges.EMPTY));
final IdentityAccumulator<NavigableMap<Timestamp, Ranges>> safeToReadAccumulator = new IdentityAccumulator<>(ImmutableSortedMap.of(Timestamp.NONE, Ranges.EMPTY));
final IdentityAccumulator<CommandStores.RangesForEpoch.Snapshot> rangesForEpochAccumulator = new IdentityAccumulator<>(null);
}
final DurableBeforeAccumulator durableBeforeAccumulator = new DurableBeforeAccumulator();
private final Map<Integer, FieldUpdates> fieldUpdates = new HashMap<>();
@Override
public Command loadCommand(int store, TxnId txnId, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
JournalKey key = new JournalKey(txnId, JournalKey.Type.COMMAND_DIFF, store);
List<LoadedDiff> saved = commands.get(key);
if (saved == null)
return null;
return reconstructFromDiff(new ArrayList<>(saved));
}
@Override
public MinimalCommand loadMinimal(int store, TxnId txnId, Load load, RedundantBefore redundantBefore, DurableBefore durableBefore)
{
JournalKey key = new JournalKey(txnId, JournalKey.Type.COMMAND_DIFF, store);
List<LoadedDiff> saved = commands.get(key);
if (saved == null)
return null;
Command command = reconstructFromDiff(new ArrayList<>(saved));
return new MinimalCommand(command.txnId(), command.saveStatus(), command.participants(), command.durability(), command.executeAt(), command.writes());
}
@Override
public RedundantBefore loadRedundantBefore(int store)
{
return fieldUpdates(store).redundantBeforeAccumulator.get();
}
@Override
public Persister<DurableBefore, DurableBefore> durableBeforePersister()
{
return new Persister<>()
{
@Override
public AsyncResult<?> persist(DurableBefore addDurableBefore, DurableBefore newDurableBefore)
{
durableBeforeAccumulator.update(addDurableBefore);
return AsyncResults.success(null);
}
@Override
public DurableBefore load()
{
return durableBeforeAccumulator.get();
}
};
}
@Override
public NavigableMap<TxnId, Ranges> loadBootstrapBeganAt(int store)
{
return fieldUpdates(store).bootstrapBeganAtAccumulator.get();
}
@Override
public NavigableMap<Timestamp, Ranges> loadSafeToRead(int store)
{
return fieldUpdates(store).safeToReadAccumulator.get();
}
@Override
public CommandStores.RangesForEpoch.Snapshot loadRangesForEpoch(int store)
{
return fieldUpdates(store).rangesForEpochAccumulator.get();
}
@Override
public void appendCommand(int store, SavedCommand.Writer diff, Runnable onFlush)
{
if (diff != null)
{
commands.computeIfAbsent(new JournalKey(diff.after().txnId(), JournalKey.Type.COMMAND_DIFF, store),
(ignore_) -> new ArrayList<>())
.add(diff(null, diff.after()));
}
if (onFlush != null)
onFlush.run();
}
private FieldUpdates fieldUpdates(int store)
{
return fieldUpdates.computeIfAbsent(store, (o) -> new FieldUpdates());
}
@Override
public void persistStoreState(int store, AccordSafeCommandStore.FieldUpdates fieldUpdates, Runnable onFlush)
{
FieldUpdates updates = fieldUpdates(store);
if (fieldUpdates.addRedundantBefore != null)
updates.redundantBeforeAccumulator.update(fieldUpdates.addRedundantBefore);
if (fieldUpdates.newBootstrapBeganAt != null)
updates.bootstrapBeganAtAccumulator.update(fieldUpdates.newBootstrapBeganAt);
if (fieldUpdates.newSafeToRead != null)
updates.safeToReadAccumulator.update(fieldUpdates.newSafeToRead);
if (fieldUpdates.newRangesForEpoch != null)
updates.rangesForEpochAccumulator.update(fieldUpdates.newRangesForEpoch);
if (onFlush != null)
onFlush.run();
}
/**
* Emulating journal behaviour
*/
public static LoadedDiff diff(Command before, Command after)
{
if (before == after)
return null;
// TODO: we do not need to save `waitingOn` _every_ time.
Command.WaitingOn waitingOn = getWaitingOn(after);
return new LoadedDiff(after.txnId(),
ifNotEqual(before, after, Command::executeAt, true),
ifNotEqual(before, after, Command::saveStatus, false),
ifNotEqual(before, after, Command::durability, false),
ifNotEqual(before, after, Command::acceptedOrCommitted, false),
ifNotEqual(before, after, Command::promised, false),
ifNotEqual(before, after, Command::participants, false),
ifNotEqual(before, after, Command::partialTxn, false),
ifNotEqual(before, after, Command::partialDeps, false),
new NewValue<>((k, deps) -> waitingOn),
ifNotEqual(before, after, Command::writes, false));
}
static Command reconstructFromDiff(List<LoadedDiff> diffs)
{
return reconstructFromDiff(diffs, CommandSerializers.APPLIED);
}
/**
* @param result is exposed because we are _not_ persisting result, since during loading or replay
* we do not expect we will have to send a result to the client, and data results
* can potentially contain a large number of entries, so it's best if they are not
* written into the log.
*/
@VisibleForTesting
static Command reconstructFromDiff(List<LoadedDiff> diffs, Result result)
{
TxnId txnId = null;
Timestamp executeAt = null;
SaveStatus saveStatus = null;
Status.Durability durability = null;
Ballot acceptedOrCommitted = Ballot.ZERO;
Ballot promised = null;
StoreParticipants participants = null;
PartialTxn partialTxn = null;
PartialDeps partialDeps = null;
SavedCommand.WaitingOnProvider waitingOnProvider = null;
Writes writes = null;
for (LoadedDiff diff : diffs)
{
if (diff.txnId != null)
txnId = diff.txnId;
if (diff.executeAt != null)
executeAt = diff.executeAt.get();
if (diff.saveStatus != null)
saveStatus = diff.saveStatus.get();
if (diff.durability != null)
durability = diff.durability.get();
if (diff.acceptedOrCommitted != null)
acceptedOrCommitted = diff.acceptedOrCommitted.get();
if (diff.promised != null)
promised = diff.promised.get();
if (diff.participants != null)
participants = diff.participants.get();
if (diff.partialTxn != null)
partialTxn = diff.partialTxn.get();
if (diff.partialDeps != null)
partialDeps = diff.partialDeps.get();
if (diff.waitingOn != null)
waitingOnProvider = diff.waitingOn.get();
if (diff.writes != null)
writes = diff.writes.get();
}
CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(txnId);
if (partialTxn != null)
attrs.partialTxn(partialTxn);
if (durability != null)
attrs.durability(durability);
if (participants != null)
attrs.setParticipants(participants);
if (partialDeps != null &&
(saveStatus.known.deps != Known.KnownDeps.NoDeps &&
saveStatus.known.deps != Known.KnownDeps.DepsErased &&
saveStatus.known.deps != Known.KnownDeps.DepsUnknown))
attrs.partialDeps(partialDeps);
Command.WaitingOn waitingOn = null;
if (waitingOnProvider != null)
waitingOn = waitingOnProvider.provide(txnId, partialDeps);
Invariants.checkState(saveStatus != null,
"Save status is null after applying %s", diffs);
switch (saveStatus.status)
{
case NotDefined:
return saveStatus == SaveStatus.Uninitialised ? Command.NotDefined.uninitialised(attrs.txnId())
: Command.NotDefined.notDefined(attrs, promised);
case PreAccepted:
return Command.PreAccepted.preAccepted(attrs, executeAt, promised);
case AcceptedInvalidate:
case Accepted:
case PreCommitted:
return Command.Accepted.accepted(attrs, saveStatus, executeAt, promised, acceptedOrCommitted);
case Committed:
case Stable:
return Command.Committed.committed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn);
case PreApplied:
case Applied:
return Command.Executed.executed(attrs, saveStatus, executeAt, promised, acceptedOrCommitted, waitingOn, writes, result);
case Truncated:
case Invalidated:
default:
throw new IllegalStateException();
}
}
// TODO (required): this convert function was added only because AsyncOperationTest was failing without it;
// maybe after switching to loading from the log we can just pass l and r directly or remove != null checks.
private static <OBJ, VAL> NewValue<VAL> ifNotEqual(OBJ lo, OBJ ro, Function<OBJ, VAL> convert, boolean allowClassMismatch)
{
VAL l = null;
VAL r = null;
if (lo != null) l = convert.apply(lo);
if (ro != null) r = convert.apply(ro);
if (l == r)
return null; // null here means there was no change
if (l == null || r == null)
return NewValue.of(r);
assert allowClassMismatch || l.getClass() == r.getClass() : String.format("%s != %s", l.getClass(), r.getClass());
if (l.equals(r))
return null;
return NewValue.of(r);
}
public static class NewValue<T>
{
final T value;
private NewValue(T value)
{
this.value = value;
}
public boolean isNull()
{
return value == null;
}
public T get()
{
return value;
}
public static <T> NewValue<T> of(T value)
{
return new NewValue<>(value);
}
public String toString()
{
return "" + value;
}
}
static Command.WaitingOn getWaitingOn(Command command)
{
if (command instanceof Command.Committed)
return command.asCommitted().waitingOn();
return null;
}
public static class LoadedDiff extends SavedCommand
{
public final TxnId txnId;
public final NewValue<Timestamp> executeAt;
public final NewValue<SaveStatus> saveStatus;
public final NewValue<Status.Durability> durability;
public final NewValue<Ballot> acceptedOrCommitted;
public final NewValue<Ballot> promised;
public final NewValue<StoreParticipants> participants;
public final NewValue<PartialTxn> partialTxn;
public final NewValue<PartialDeps> partialDeps;
public final NewValue<Writes> writes;
public final NewValue<WaitingOnProvider> waitingOn;
public LoadedDiff(TxnId txnId,
NewValue<Timestamp> executeAt,
NewValue<SaveStatus> saveStatus,
NewValue<Status.Durability> durability,
NewValue<Ballot> acceptedOrCommitted,
NewValue<Ballot> promised,
NewValue<StoreParticipants> participants,
NewValue<PartialTxn> partialTxn,
NewValue<PartialDeps> partialDeps,
NewValue<SavedCommand.WaitingOnProvider> waitingOn,
NewValue<Writes> writes)
{
this.txnId = txnId;
this.executeAt = executeAt;
this.saveStatus = saveStatus;
this.durability = durability;
this.acceptedOrCommitted = acceptedOrCommitted;
this.promised = promised;
this.participants = participants;
this.partialTxn = partialTxn;
this.partialDeps = partialDeps;
this.writes = writes;
this.waitingOn = waitingOn;
}
public String toString()
{
return "LoadedDiff{" +
"waitingOn=" + waitingOn +
'}';
}
}
}

32
test/unit/org/apache/cassandra/service/accord/SimulatedAccordCommandStore.java
View File

@ -46,10 +46,10 @@ import accord.local.CommandStores;
import accord.local.DurableBefore;
import accord.local.Node;
import accord.local.NodeCommandStoreService;
import accord.local.TimeService;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
import accord.local.SafeCommandStore;
import accord.local.TimeService;
import accord.messages.BeginRecovery;
import accord.messages.PreAccept;
import accord.messages.Reply;
@ -68,6 +68,7 @@ import accord.primitives.Unseekables;
import accord.topology.Topologies;
import accord.topology.Topology;
import accord.utils.Gens;
import accord.utils.PersistentField;
import accord.utils.RandomSource;
import accord.utils.async.AsyncChains;
import accord.utils.async.AsyncResult;
@ -104,7 +105,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
public final Node.Id nodeId;
public final Topology topology;
public final Topologies topologies;
public final MockJournal journal;
public final IJournal journal;
public final ScheduledExecutorPlus unorderedScheduled;
public final List<String> evictions = new ArrayList<>();
public Predicate<Throwable> ignoreExceptions = ignore -> false;
@ -184,7 +185,7 @@ public class SimulatedAccordCommandStore implements AutoCloseable
}
};
this.journal = new MockJournal();
this.journal = new InMemoryJournal(nodeId);
TestAgent.RethrowAgent agent = new TestAgent.RethrowAgent()
{
@Override
@ -204,12 +205,12 @@ public class SimulatedAccordCommandStore implements AutoCloseable
storeService,
agent,
null,
ignore -> new ProgressLog.NoOpProgressLog(),
cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
{
@Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
@Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
}),
ignore -> new ProgressLog.NoOpProgressLog(),
cs -> new DefaultLocalListeners(new RemoteListeners.NoOpRemoteListeners(), new DefaultLocalListeners.NotifySink()
{
@Override public void notify(SafeCommandStore safeStore, SafeCommand safeCommand, TxnId listener) {}
@Override public boolean notify(SafeCommandStore safeStore, SafeCommand safeCommand, LocalListeners.ComplexListener listener) { return false; }
}),
updateHolder,
journal,
new AccordExecutorSimple(0, CommandStore.class.getSimpleName() + '[' + 0 + ']', new AccordCacheMetrics("test"), agent));
@ -245,6 +246,19 @@ public class SimulatedAccordCommandStore implements AutoCloseable
});
}
private final class InMemoryJournal extends accord.impl.basic.InMemoryJournal implements IJournal
{
public InMemoryJournal(Node.Id id)
{
super(id);
}
public PersistentField.Persister<DurableBefore, DurableBefore> durableBeforePersister()
{
throw new IllegalArgumentException("Not implemented");
}
}
private <K, V> void updateLoadFunction(AccordCache.Type<K, V, ?> i, FunctionWrapper wrapper)
{
i.unsafeSetLoadFunction(wrapper.wrap(i.unsafeGetLoadFunction()));

4
test/unit/org/apache/cassandra/service/accord/serializers/DepsSerializerTest.java → test/unit/org/apache/cassandra/service/accord/serializers/DepsSerializersTest.java
View File

@ -38,7 +38,7 @@ import org.mockito.Mockito;
import static accord.utils.Property.qt;
import static org.apache.cassandra.net.MessagingService.Version.VERSION_51;
public class DepsSerializerTest
public class DepsSerializersTest
{
static
{
@ -59,7 +59,7 @@ public class DepsSerializerTest
Mockito.when(Schema.instance.getExistingTablePartitioner(Mockito.any())).thenReturn(partitioner);
Deps deps = AccordGenerators.depsGen(partitioner).next(rs);
for (MessagingService.Version version : SUPPORTED_VERSIONS)
IVersionedSerializers.testSerde(buffer, DepsSerializer.deps, deps, version.value);
IVersionedSerializers.testSerde(buffer, DepsSerializers.deps, deps, version.value);
});
}
}