natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce command deps 

Patch by Blake Eggleston; Reviewed by Benedict Elliott Smith for CASSANDRA-18784
This commit is contained in:
Blake Eggleston 2023-10-27 12:18:49 -07:00 committed by David Capwell
parent 050688228f
commit d07a36106d
30 changed files with 2078 additions and 603 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/accord

@ -1 +1 @@
Subproject commit 746dabe0b43bf719badbd605e68a76037d01256d
Subproject commit d9ef555302f8774ed03325ba22d38ee0b80130a8

93
src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
View File

@ -80,7 +80,8 @@ import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandRows;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsColumns;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyRows;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyAccessor;
import org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyRows;
import org.apache.cassandra.service.accord.AccordService;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.service.accord.api.PartitionKey;
@ -97,11 +98,11 @@ import static org.apache.cassandra.config.Config.PaxosStatePurging.legacy;
import static org.apache.cassandra.config.DatabaseDescriptor.paxosStatePurging;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandRows.maybeDropTruncatedCommandColumns;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandRows.truncatedApply;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyColumns.last_executed_micros;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyColumns.last_executed_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyColumns.last_write_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyColumns.max_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyRows.truncateStaticRow;
import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_executed_micros;
import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_executed_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.last_write_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyColumns.max_timestamp;
import static org.apache.cassandra.service.accord.AccordKeyspace.TimestampsForKeyRows.truncateTimestampsForKeyRow;
import static org.apache.cassandra.service.accord.AccordKeyspace.deserializeDurabilityOrNull;
import static org.apache.cassandra.service.accord.AccordKeyspace.deserializeRouteOrNull;
import static org.apache.cassandra.service.accord.AccordKeyspace.deserializeSaveStatusOrNull;
@ -204,8 +205,11 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
? new PaxosPurger()
: isAccordCommands(controller.cfs)
? new AccordCommandsPurger(accordService)
: isAccordCommandsForKey(controller.cfs) ? new AccordCommandsForKeyPurger(accordService)
: new Purger(controller, nowInSec);
: isAccordDepsCommandsForKey(controller.cfs)
? new AccordCommandsForKeyPurger(AccordKeyspace.DepsCommandsForKeysAccessor, accordService)
: isAccordAllCommandsForKey(controller.cfs)
? new AccordCommandsForKeyPurger(AccordKeyspace.AllCommandsForKeysAccessor, accordService)
: new Purger(controller, nowInSec);
merged = Transformation.apply(merged, purger);
merged = DuplicateRowChecker.duringCompaction(merged, type);
compacted = Transformation.apply(merged, new AbortableUnfilteredPartitionTransformation(this));
@ -817,7 +821,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
if (executeAt == null || durability == null || saveStatus == null || route == null)
return row;
Commands.Cleanup cleanup = Commands.shouldCleanup(txnId, saveStatus.status, durability, executeAt, route, redundantBefore, durableBefore, false);
Commands.Cleanup cleanup = Commands.shouldCleanup(txnId, saveStatus.status,
durability, executeAt, route,
redundantBefore, durableBefore);
switch (cleanup)
{
default: throw new AssertionError(String.format("Unexpected cleanup task: %s", cleanup));
@ -840,6 +846,9 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
}
}
@Override
protected Row applyToStatic(Row row)
{
@ -848,26 +857,25 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
}
}
class AccordCommandsForKeyPurger extends AbstractPurger
class AccordTimestampsForKeyPurger extends AbstractPurger
{
final Int2ObjectHashMap<RedundantBefore> redundantBefores;
int storeId;
PartitionKey partitionKey;
AccordCommandsForKeyPurger(Supplier<IAccordService> accordService)
AccordTimestampsForKeyPurger(Supplier<IAccordService> accordService)
{
this.redundantBefores = accordService.get().getRedundantBeforesAndDurableBefore().left;
}
protected void beginPartition(UnfilteredRowIterator partition)
{
ByteBuffer[] partitionKeyComponents = CommandsForKeyRows.splitPartitionKey(partition.partitionKey());
storeId = CommandsForKeyRows.getStoreId(partitionKeyComponents);
partitionKey = CommandsForKeyRows.getKey(partitionKeyComponents);
ByteBuffer[] partitionKeyComponents = TimestampsForKeyRows.splitPartitionKey(partition.partitionKey());
storeId = TimestampsForKeyRows.getStoreId(partitionKeyComponents);
partitionKey = TimestampsForKeyRows.getKey(partitionKeyComponents);
}
@Override
protected Row applyToStatic(Row row)
protected Row applyToRow(Row row)
{
updateProgress();
@ -920,7 +928,35 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
maxTimestampCell == null)
return null;
return truncateStaticRow(nowInSec, row, lastExecuteMicrosCell, lastExecuteCell, lastWriteCell, maxTimestampCell);
return truncateTimestampsForKeyRow(nowInSec, row, lastExecuteMicrosCell, lastExecuteCell, lastWriteCell, maxTimestampCell);
}
@Override
protected Row applyToStatic(Row row)
{
checkState(row.isStatic() && row.isEmpty());
return row;
}
}
class AccordCommandsForKeyPurger extends AbstractPurger
{
final CommandsForKeyAccessor accessor;
final Int2ObjectHashMap<RedundantBefore> redundantBefores;
int storeId;
PartitionKey partitionKey;
AccordCommandsForKeyPurger(CommandsForKeyAccessor accessor, Supplier<IAccordService> accordService)
{
this.accessor = accessor;
this.redundantBefores = accordService.get().getRedundantBeforesAndDurableBefore().left;
}
protected void beginPartition(UnfilteredRowIterator partition)
{
ByteBuffer[] partitionKeyComponents = accessor.splitPartitionKey(partition.partitionKey());
storeId = accessor.getStoreId(partitionKeyComponents);
partitionKey = accessor.getKey(partitionKeyComponents);
}
@Override
@ -938,12 +974,19 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
return row;
TxnId redundantBeforeTxnId = redundantBeforeEntry.shardRedundantBefore();
Timestamp timestamp = CommandsForKeyRows.getTimestamp(row);
Timestamp timestamp = accessor.getTimestamp(row);
if (timestamp != null && timestamp.compareTo(redundantBeforeTxnId) < 0)
return null;
return row;
}
@Override
protected Row applyToStatic(Row row)
{
checkState(row.isStatic() && row.isEmpty());
return row;
}
}
private static class AbortableUnfilteredPartitionTransformation extends Transformation<UnfilteredRowIterator>
@ -991,8 +1034,18 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
return cfs.name.equals(AccordKeyspace.COMMANDS) && cfs.keyspace.getName().equals(SchemaConstants.ACCORD_KEYSPACE_NAME);
}
private static boolean isAccordCommandsForKey(ColumnFamilyStore cfs)
private static boolean isAccordCommandsForKey(ColumnFamilyStore cfs, String name)
{
return cfs.name.equals(AccordKeyspace.COMMANDS_FOR_KEY) && cfs.keyspace.getName().equals(SchemaConstants.ACCORD_KEYSPACE_NAME);
return cfs.name.equals(name) && cfs.keyspace.getName().equals(SchemaConstants.ACCORD_KEYSPACE_NAME);
}
private static boolean isAccordDepsCommandsForKey(ColumnFamilyStore cfs)
{
return isAccordCommandsForKey(cfs, AccordKeyspace.DEPS_COMMANDS_FOR_KEY);
}
private static boolean isAccordAllCommandsForKey(ColumnFamilyStore cfs)
{
return isAccordCommandsForKey(cfs, AccordKeyspace.ALL_COMMANDS_FOR_KEY);
}
}

54
src/java/org/apache/cassandra/service/accord/AccordCachingState.java
View File

@ -22,11 +22,13 @@ import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.ToLongFunction;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
import accord.local.Command.TransientListener;
import accord.local.Listeners;
import accord.utils.IntrusiveLinkedListNode;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncResults.RunnableResult;
import org.apache.cassandra.concurrent.ExecutorPlus;
@ -47,13 +49,24 @@ import static org.apache.cassandra.service.accord.AccordCachingState.Status.UNIN
*/
public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
{
static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCachingState<>(null, null));
static final long EMPTY_SIZE = ObjectSizes.measure(new AccordCachingState<>(null, 0, null));
public interface Factory<K, V>
{
AccordCachingState<K, V> create(K key, int index);
}
static <K, V> Factory<K, V> defaultFactory()
{
return AccordCachingState::new;
}
private final K key;
private State<K, V> state;
int references = 0;
int lastQueriedEstimatedSizeOnHeap = 0;
final byte index;
private boolean shouldUpdateSize;
/**
@ -61,16 +74,20 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
*/
private Listeners<TransientListener> transientListeners;
public AccordCachingState(K key)
AccordCachingState(K key, int index)
{
this.key = key;
Invariants.checkArgument(index >= 0 && index <= Byte.MAX_VALUE);
this.index = (byte) index;
//noinspection unchecked
this.state = (State<K, V>) Uninitialized.instance;
}
AccordCachingState(K key, State<K, V> state)
private AccordCachingState(K key, int index, State<K, V> state)
{
this.key = key;
Invariants.checkArgument(index >= 0 && index <= Byte.MAX_VALUE);
this.index = (byte) index;
this.state = state;
}
@ -104,6 +121,11 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
return status().isComplete();
}
public boolean canEvict()
{
return true;
}
int estimatedSizeOnHeap(ToLongFunction<V> estimator)
{
shouldUpdateSize = false;
@ -179,7 +201,12 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
return loading;
}
private State<K, V> state(State<K, V> next)
public void initialize(V value)
{
state(state.initialize(value));
}
protected State<K, V> state(State<K, V> next)
{
State<K, V> prev = state;
if (prev != next)
@ -187,6 +214,12 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
return state = next;
}
@VisibleForTesting
protected State<K, V> state()
{
return state;
}
public AsyncChain<V> loading()
{
// do *not* attempt to complete, to prevent races where the caller found a pending load, attempts
@ -209,7 +242,8 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
* Submits a save runnable to the specified executor. When the runnable
* has completed, the state save will have either completed or failed.
*/
void save(ExecutorPlus executor, BiFunction<?, ?, Runnable> saveFunction)
@VisibleForTesting
public void save(ExecutorPlus executor, BiFunction<?, ?, Runnable> saveFunction)
{
@SuppressWarnings("unchecked")
State<K, V> savingOrLoaded = state.save((BiFunction<V, V, Runnable>) saveFunction);
@ -300,6 +334,11 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
throw illegalState(this, "load(key, loadFunction)");
}
default Loaded<K, V> initialize(V value)
{
throw illegalState(this, "initialize(value)");
}
default RunnableResult<V> loading()
{
throw illegalState(this, "loading()");
@ -373,6 +412,11 @@ public class AccordCachingState<K, V> extends IntrusiveLinkedListNode
return new Loading<>(() -> loadFunction.apply(key));
}
public Loaded<K, V> initialize(V value)
{
return new Loaded<>(value);
}
@Override
public Evicted<K, V> evict()
{

147
src/java/org/apache/cassandra/service/accord/AccordCommandStore.java
View File

@ -27,7 +27,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
@ -43,8 +42,10 @@ import accord.api.Agent;
import accord.api.DataStore;
import accord.api.Key;
import accord.api.ProgressLog;
import accord.impl.CommandTimeseriesHolder;
import accord.impl.CommandsForKey;
import accord.impl.DomainCommands;
import accord.impl.DomainTimestamps;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.CommandStore;
import accord.local.DurableBefore;
@ -69,6 +70,7 @@ import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.utils.Invariants;
import accord.utils.ReducingRangeMap;
import accord.utils.TriFunction;
import accord.utils.async.AsyncChain;
import accord.utils.async.AsyncChains;
import accord.utils.async.Observable;
@ -112,9 +114,13 @@ public class AccordCommandStore extends CommandStore implements CacheSize
private final AccordJournal journal;
private final ExecutorService executor;
private final ExecutionOrder executionOrder;
private final AccordCommandsForKeys keyCoordinator;
private final AccordStateCache stateCache;
private final AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache;
private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache;
private final AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache;
private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> depsCommandsForKeyCache;
private final AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> allCommandsForKeyCache;
private final AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updatesForKeyCache;
private AsyncOperation<?> currentOperation = null;
private AccordSafeCommandStore current = null;
private long lastSystemTimestampMicros = Long.MIN_VALUE;
@ -147,26 +153,56 @@ public class AccordCommandStore extends CommandStore implements CacheSize
super(id, time, agent, dataStore, progressLogFactory, epochUpdateHolder);
this.journal = journal;
loggingId = String.format("[%s]", id);
keyCoordinator = new AccordCommandsForKeys(this);
executor = executorFactory().sequential(CommandStore.class.getSimpleName() + '[' + id + ']');
executionOrder = new ExecutionOrder();
threadId = getThreadId(executor);
stateCache = new AccordStateCache(loadExecutor, saveExecutor, 8 << 20, cacheMetrics);
commandCache =
stateCache.instance(TxnId.class,
TxnId.class,
AccordSafeCommand.class,
AccordSafeCommand::new,
this::loadCommand,
this::saveCommand,
this::validateCommand,
AccordObjectSizes::command);
commandsForKeyCache =
timestampsForKeyCache =
stateCache.instance(RoutableKey.class,
PartitionKey.class,
AccordSafeTimestampsForKey.class,
AccordSafeTimestampsForKey::new,
this::loadTimestampsForKey,
this::saveTimestampsForKey,
this::validateTimestampsForKey,
AccordObjectSizes::timestampsForKey);
depsCommandsForKeyCache =
stateCache.instance(RoutableKey.class,
AccordSafeCommandsForKey.class,
AccordSafeCommandsForKey::new,
this::loadCommandsForKey,
this::loadDepsCommandsForKey,
this::saveCommandsForKey,
this::validateCommandsForKey,
AccordObjectSizes::commandsForKey);
this::validateDepsCommandsForKey,
AccordObjectSizes::commandsForKey,
keyCoordinator::createDepsCommandsNode);
allCommandsForKeyCache =
stateCache.instance(RoutableKey.class,
AccordSafeCommandsForKey.class,
AccordSafeCommandsForKey::new,
this::loadAllCommandsForKey,
this::saveCommandsForKey,
this::validateAllCommandsForKey,
AccordObjectSizes::commandsForKey,
keyCoordinator::createDepsCommandsNode);
updatesForKeyCache =
stateCache.instance(RoutableKey.class,
AccordSafeCommandsForKeyUpdate.class,
AccordSafeCommandsForKeyUpdate::new,
this::loadCommandsForKeyUpdate,
this::saveCommandsForKeyUpdate,
(key, evicting) -> true,
CommandsForKeyUpdate::estimatedSizeOnHeap,
keyCoordinator::createUpdatesNode);
//>>>>>>> 701eeff2b4 (deps pruning integration)
AccordKeyspace.loadCommandStoreMetadata(id, ((rejectBefore, durableBefore, redundantBefore, bootstrapBeganAt, safeToRead) -> {
executor.submit(() -> {
if (rejectBefore != null)
@ -181,6 +217,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
super.setSafeToRead(safeToRead);
});
}));
executor.execute(() -> CommandStore.register(this));
executor.execute(this::loadRangesToCommands);
}
@ -299,9 +336,24 @@ public class AccordCommandStore extends CommandStore implements CacheSize
return commandCache;
}
public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> commandsForKeyCache()
public AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsForKeyCache()
{
return commandsForKeyCache;
return timestampsForKeyCache;
}
public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> depsCommandsForKeyCache()
{
return depsCommandsForKeyCache;
}
public AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> allCommandsForKeyCache()
{
return allCommandsForKeyCache;
}
public AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updatesForKeyCache()
{
return updatesForKeyCache;
}
Command loadCommand(TxnId txnId)
@ -322,22 +374,63 @@ public class AccordCommandStore extends CommandStore implements CacheSize
return (evicting == null && reloaded == null) || (evicting != null && reloaded != null && reloaded.isEqualOrFuller(evicting));
}
CommandsForKey loadCommandsForKey(RoutableKey key)
boolean validateTimestampsForKey(RoutableKey key, TimestampsForKey evicting)
{
return AccordKeyspace.loadCommandsForKey(this, (PartitionKey) key);
TimestampsForKey reloaded = AccordKeyspace.unsafeLoadTimestampsForKey(this, (PartitionKey) key);
return Objects.equals(evicting, reloaded);
}
TimestampsForKey loadTimestampsForKey(RoutableKey key)
{
return AccordKeyspace.loadTimestampsForKey(this, (PartitionKey) key);
}
CommandsForKey loadDepsCommandsForKey(RoutableKey key)
{
return AccordKeyspace.loadDepsCommandsForKey(this, (PartitionKey) key);
}
CommandsForKey loadAllCommandsForKey(RoutableKey key)
{
return AccordKeyspace.loadAllCommandsForKey(this, (PartitionKey) key);
}
CommandsForKeyUpdate loadCommandsForKeyUpdate(RoutableKey key)
{
throw new IllegalStateException();
}
boolean validateDepsCommandsForKey(RoutableKey key, CommandsForKey evicting)
{
CommandsForKey reloaded = AccordKeyspace.loadDepsCommandsForKey(this, (PartitionKey) key);
return Objects.equals(evicting, reloaded);
}
boolean validateAllCommandsForKey(RoutableKey key, CommandsForKey evicting)
{
CommandsForKey reloaded = AccordKeyspace.loadAllCommandsForKey(this, (PartitionKey) key);
return Objects.equals(evicting, reloaded);
}
@Nullable
private Runnable saveCommandsForKey(CommandsForKey before, CommandsForKey after)
{
Mutation mutation = AccordKeyspace.getCommandsForKeyMutation(id, before, after, nextSystemTimestampMicros());
throw new IllegalStateException();
}
@Nullable
private Runnable saveTimestampsForKey(TimestampsForKey before, TimestampsForKey after)
{
Mutation mutation = AccordKeyspace.getTimestampsForKeyMutation(id, before, after, nextSystemTimestampMicros());
return null != mutation ? mutation::apply : null;
}
boolean validateCommandsForKey(RoutableKey key, CommandsForKey evicting)
@Nullable
private Runnable saveCommandsForKeyUpdate(CommandsForKeyUpdate before, CommandsForKeyUpdate after)
{
CommandsForKey reloaded = AccordKeyspace.unsafeLoadCommandsForKey(this, (PartitionKey) key);
return Objects.equals(evicting, reloaded);
Mutation mutation = AccordKeyspace.getCommandsForKeyMutation(id, after, nextSystemTimestampMicros());
return null != mutation ? mutation::apply : null;
}
@VisibleForTesting
@ -431,12 +524,16 @@ public class AccordCommandStore extends CommandStore implements CacheSize
public AccordSafeCommandStore beginOperation(PreLoadContext preLoadContext,
Map<TxnId, AccordSafeCommand> commands,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeys)
NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKeys,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys)
{
Invariants.checkState(current == null);
commands.values().forEach(AccordSafeState::preExecute);
commandsForKeys.values().forEach(AccordSafeState::preExecute);
current = new AccordSafeCommandStore(preLoadContext, commands, commandsForKeys, this);
depsCommandsForKeys.values().forEach(AccordSafeState::preExecute);
timestampsForKeys.values().forEach(AccordSafeState::preExecute);
current = new AccordSafeCommandStore(preLoadContext, commands, timestampsForKeys, depsCommandsForKeys, allCommandsForKeys, updatesForKeys, this);
return current;
}
@ -452,7 +549,7 @@ public class AccordCommandStore extends CommandStore implements CacheSize
current = null;
}
<O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandTimeseriesHolder, O, O> map, O accumulate, Predicate<? super O> terminate)
<O> O mapReduceForRange(Routables<?> keysOrRanges, Ranges slice, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
{
keysOrRanges = keysOrRanges.slice(slice, Routables.Slice.Minimal);
switch (keysOrRanges.domain())
@ -460,9 +557,9 @@ public class AccordCommandStore extends CommandStore implements CacheSize
case Key:
{
AbstractKeys<Key> keys = (AbstractKeys<Key>) keysOrRanges;
for (CommandTimeseriesHolder summary : commandsForRanges.search(keys))
for (CommandsForRanges.DomainInfo summary : commandsForRanges.search(keys))
{
accumulate = map.apply(summary, accumulate);
accumulate = map.apply(summary, summary, accumulate);
if (terminate.test(accumulate))
return accumulate;
}
@ -473,10 +570,10 @@ public class AccordCommandStore extends CommandStore implements CacheSize
AbstractRanges ranges = (AbstractRanges) keysOrRanges;
for (Range range : ranges)
{
CommandTimeseriesHolder summary = commandsForRanges.search(range);
CommandsForRanges.DomainInfo summary = commandsForRanges.search(range);
if (summary == null)
continue;
accumulate = map.apply(summary, accumulate);
accumulate = map.apply(summary, summary, accumulate);
if (terminate.test(accumulate))
return accumulate;
}

252
src/java/org/apache/cassandra/service/accord/AccordCommandsForKeys.java Normal file
View File

@ -0,0 +1,252 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.nio.ByteBuffer;
import java.util.function.Function;
import accord.api.Key;
import accord.impl.CommandsForKey;
import accord.impl.CommandsForKeyGroupUpdater;
import accord.impl.CommandsForKeyUpdater;
import accord.primitives.RoutableKey;
import accord.utils.Invariants;
import accord.utils.async.AsyncChain;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
/**
* Ties together the separate commands for key and commands for key update classes to make loading,
* saving, and evicting coherent
*/
public class AccordCommandsForKeys
{
private final AccordCommandStore commandStore;
public AccordCommandsForKeys(AccordCommandStore commandStore)
{
this.commandStore = commandStore;
}
AccordCachingState<RoutableKey, CommandsForKey> createDepsCommandsNode(RoutableKey key, int index)
{
return new DepsCommandsCachingState(key, index);
}
AccordCachingState<RoutableKey, CommandsForKey> createAllCommandsNode(RoutableKey key, int index)
{
return new AllCommandsCachingState(key, index);
}
AccordCachingState<RoutableKey, CommandsForKeyUpdate> createUpdatesNode(RoutableKey key, int index)
{
return new UpdateCachingState(key, index);
}
protected static boolean hasEvictableStatus(AccordCachingState<?,?> state)
{
if (state == null)
return true;
switch (state.status())
{
case LOADING:
case SAVING:
return false;
}
return true;
}
boolean canEvictKey(RoutableKey key)
{
return hasEvictableStatus(commandStore.depsCommandsForKeyCache().getUnsafe(key))
&& hasEvictableStatus(commandStore.allCommandsForKeyCache().getUnsafe(key))
&& hasEvictableStatus(commandStore.updatesForKeyCache().getUnsafe(key));
}
public abstract class CommandsCachingState extends AccordCachingState<RoutableKey, CommandsForKey>
{
protected CommandsCachingState(RoutableKey key, int index)
{
super(key, index);
}
private CommandsForKey initializeIfNull(CommandsForKey commands)
{
if (commands != null)
return commands;
return new CommandsForKey((Key) key(), CommandsForKeySerializer.loader);
}
private State<RoutableKey, CommandsForKey> maybeApplyUpdates(State<RoutableKey, CommandsForKey> state)
{
if (!(state instanceof Loaded))
return state;
Loaded<RoutableKey, CommandsForKey> loaded = (Loaded<RoutableKey, CommandsForKey>) state;
CommandsForKey commands = loaded.get();
UpdateCachingState updates = (UpdateCachingState) commandStore.updatesForKeyCache().getUnsafe(key());
if (updates == null)
return loaded;
CommandsForKeyUpdate update = updates.getUpdateIfAvailable();
if (update == null)
return loaded;
CommandsForKey updated = apply(initializeIfNull(commands), update);
if (updated == commands)
return loaded;
return new Loaded<>(updated);
}
protected abstract CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update);
private void maybeApplyUpdates(CommandsForKeyUpdate update)
{
if (status() != Status.LOADED)
return;
CommandsForKey commands = get();
CommandsForKey updated = apply(initializeIfNull(commands), update);
if (commands != updated)
super.state(new Loaded<>(updated));
}
protected State<RoutableKey, CommandsForKey> state(State<RoutableKey, CommandsForKey> next)
{
Status nextStatus = next.status();
Invariants.checkState(nextStatus != Status.MODIFIED && nextStatus != Status.SAVING,
"CommandsForKey cannot have state %s", nextStatus);
return super.state(maybeApplyUpdates(next));
}
@Override
public boolean canEvict()
{
return canEvictKey(key());
}
}
public class DepsCommandsCachingState extends CommandsCachingState
{
public DepsCommandsCachingState(RoutableKey key, int index)
{
super(key, index);
}
protected CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update)
{
return update.applyToDeps(current);
}
}
public class AllCommandsCachingState extends CommandsCachingState
{
public AllCommandsCachingState(RoutableKey key, int index)
{
super(key, index);
}
protected CommandsForKey apply(CommandsForKey current, CommandsForKeyUpdate update)
{
return update.applyToAll(current);
}
}
public class UpdateCachingState extends AccordCachingState<RoutableKey, CommandsForKeyUpdate> implements CommandsForKeyGroupUpdater.Immutable.Factory<ByteBuffer, CommandsForKeyUpdate>
{
public UpdateCachingState(RoutableKey key, int index)
{
super(key, index);
}
public AsyncChain<CommandsForKeyUpdate> load(ExecutorPlus executor, Function<RoutableKey, CommandsForKeyUpdate> loadFunction)
{
if (status() == Status.UNINITIALIZED)
{
CommandsForKeyUpdate initialized = CommandsForKeyUpdate.empty(key());
state(state().initialize(initialized));
return null;
}
return super.load(executor, loadFunction);
}
// update in memory cfk data with the update results
protected void maybeUpdateCommands(CommandsForKeyUpdate update)
{
CommandsCachingState commands = (CommandsCachingState) commandStore.depsCommandsForKeyCache().getUnsafe(key());
if (commands == null)
return;
commands.maybeApplyUpdates(update);
}
public CommandsForKeyUpdate create(CommandsForKeyUpdater.Immutable<ByteBuffer> deps, CommandsForKeyUpdater.Immutable<ByteBuffer> all, CommandsForKeyUpdater.Immutable<ByteBuffer> common)
{
return new CommandsForKeyUpdate((PartitionKey) key(), deps, all, common);
}
protected State<RoutableKey, CommandsForKeyUpdate> maybeProcessModification(State<RoutableKey, CommandsForKeyUpdate> next)
{
if (!(next instanceof Modified))
return next;
Modified<RoutableKey, CommandsForKeyUpdate> modified = (Modified<RoutableKey, CommandsForKeyUpdate>) next;
CommandsForKeyUpdate current = modified.current;
maybeUpdateCommands(current);
// combine in memory updates
current = CommandsForKeyGroupUpdater.Immutable.merge(modified.original, current, this);
return new Modified<>(null, current);
}
protected State<RoutableKey, CommandsForKeyUpdate> state(State<RoutableKey, CommandsForKeyUpdate> next)
{
Status nextStatus = next.status();
Invariants.checkState(nextStatus != Status.LOADING,
"CommandsForKeyUpdate cannot have state %s", nextStatus);
return super.state(maybeProcessModification(next));
}
CommandsForKeyUpdate getUpdateIfAvailable()
{
switch (status())
{
case LOADED:
case MODIFIED:
return get();
}
return null;
}
@Override
public boolean canEvict()
{
return canEvictKey(key());
}
}
}

579
src/java/org/apache/cassandra/service/accord/AccordKeyspace.java
View File

@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@ -35,10 +34,12 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Iterables;
@ -47,8 +48,11 @@ import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.Key;
import accord.impl.CommandTimeseries;
import accord.impl.CommandsForKey;
import accord.impl.CommandsForKeyUpdater;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.Command.WaitingOn;
import accord.local.CommandStore;
@ -150,8 +154,9 @@ import org.apache.cassandra.service.accord.serializers.TopologySerializers;
import org.apache.cassandra.service.accord.serializers.WaitingOnSerializer;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.MonotonicClock;
import org.apache.cassandra.utils.btree.BTree;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.bytecomparable.ByteComparable;
import static accord.utils.Invariants.checkArgument;
@ -171,7 +176,9 @@ public class AccordKeyspace
private static final Logger logger = LoggerFactory.getLogger(AccordKeyspace.class);
public static final String COMMANDS = "commands";
public static final String COMMANDS_FOR_KEY = "commands_for_key";
public static final String TIMESTAMPS_FOR_KEY = "timestamps_for_key";
public static final String DEPS_COMMANDS_FOR_KEY = "deps_commands_for_key";
public static final String ALL_COMMANDS_FOR_KEY = "all_commands_for_key";
public static final String TOPOLOGIES = "topologies";
public static final String EPOCH_METADATA = "epoch_metadata";
public static final String COMMAND_STORE_METADATA = "command_store_metadata";
@ -183,28 +190,6 @@ public class AccordKeyspace
private static final ClusteringIndexFilter FULL_PARTITION = new ClusteringIndexSliceFilter(Slices.ALL, false);
public enum SeriesKind
{
BY_ID(CommandsForKey::byId),
BY_EXECUTE_AT(CommandsForKey::byExecuteAt);
private final Function<CommandsForKey, CommandTimeseries<?>> getSeries;
SeriesKind(Function<CommandsForKey, CommandTimeseries<?>> getSeries)
{
this.getSeries = getSeries;
}
ImmutableSortedMap<Timestamp, ByteBuffer> getValues(CommandsForKey cfk)
{
if (cfk == null)
return ImmutableSortedMap.of();
CommandTimeseries<?> series = getSeries.apply(cfk);
return (ImmutableSortedMap<Timestamp, ByteBuffer>) series.commands;
}
}
private enum TokenType
{
Murmur3((byte) 1),
@ -421,81 +406,50 @@ public class AccordKeyspace
}
}
private static final TableMetadata CommandsForKeys =
parse(COMMANDS_FOR_KEY,
"accord commands per key",
private static final TableMetadata TimestampsForKeys =
parse(TIMESTAMPS_FOR_KEY,
"accord timestamps per key",
"CREATE TABLE %s ("
+ "store_id int, "
+ "key_token blob, " // can't use "token" as this is restricted word in CQL
+ format("key %s, ", KEY_TUPLE)
+ format("max_timestamp %s static, ", TIMESTAMP_TUPLE)
+ format("last_executed_timestamp %s static, ", TIMESTAMP_TUPLE)
+ "last_executed_micros bigint static, "
+ format("last_write_timestamp %s static, ", TIMESTAMP_TUPLE)
+ "series int, "
+ format("timestamp %s, ", TIMESTAMP_TUPLE)
+ "data blob, "
+ "PRIMARY KEY((store_id, key_token, key), series, timestamp)"
+ format("max_timestamp %s, ", TIMESTAMP_TUPLE)
+ format("last_executed_timestamp %s, ", TIMESTAMP_TUPLE)
+ "last_executed_micros bigint, "
+ format("last_write_timestamp %s, ", TIMESTAMP_TUPLE)
+ "PRIMARY KEY((store_id, key_token, key))"
+ ')')
.partitioner(new LocalPartitioner(CompositeType.getInstance(Int32Type.instance, BytesType.instance, KEY_TYPE)))
.build();
public static class CommandsForKeyColumns
public static class TimestampsForKeyColumns
{
static final ClusteringComparator keyComparator = CommandsForKeys.partitionKeyAsClusteringComparator();
static final CompositeType partitionKeyType = (CompositeType) CommandsForKeys.partitionKeyType;
static final ColumnFilter allColumns = ColumnFilter.all(CommandsForKeys);
static final ColumnMetadata store_id = getColumn(CommandsForKeys, "store_id");
static final ColumnMetadata key_token = getColumn(CommandsForKeys, "key_token");
static final ColumnMetadata key = getColumn(CommandsForKeys, "key");
static final ColumnMetadata timestamp = getColumn(CommandsForKeys, "timestamp");
public static final ColumnMetadata max_timestamp = getColumn(CommandsForKeys, "max_timestamp");
public static final ColumnMetadata last_executed_timestamp = getColumn(CommandsForKeys, "last_executed_timestamp");
public static final ColumnMetadata last_executed_micros = getColumn(CommandsForKeys, "last_executed_micros");
public static final ColumnMetadata last_write_timestamp = getColumn(CommandsForKeys, "last_write_timestamp");
static final ClusteringComparator keyComparator = TimestampsForKeys.partitionKeyAsClusteringComparator();
static final CompositeType partitionKeyType = (CompositeType) TimestampsForKeys.partitionKeyType;
static final ColumnFilter allColumns = ColumnFilter.all(TimestampsForKeys);
static final ColumnMetadata store_id = getColumn(TimestampsForKeys, "store_id");
static final ColumnMetadata key_token = getColumn(TimestampsForKeys, "key_token");
static final ColumnMetadata key = getColumn(TimestampsForKeys, "key");
public static final ColumnMetadata max_timestamp = getColumn(TimestampsForKeys, "max_timestamp");
public static final ColumnMetadata last_executed_timestamp = getColumn(TimestampsForKeys, "last_executed_timestamp");
public static final ColumnMetadata last_executed_micros = getColumn(TimestampsForKeys, "last_executed_micros");
public static final ColumnMetadata last_write_timestamp = getColumn(TimestampsForKeys, "last_write_timestamp");
static final ColumnMetadata data = getColumn(CommandsForKeys, "data");
static final Columns columns = Columns.from(Lists.newArrayList(max_timestamp, last_executed_timestamp, last_executed_micros, last_write_timestamp));
// Ordered by columnn name because it will be used to construct btree leaf arrays
static final ColumnMetadata[] static_columns_metadata = new ColumnMetadata[] { last_executed_micros, last_executed_timestamp, last_write_timestamp, max_timestamp };
static final Columns statics = Columns.from(Lists.newArrayList(max_timestamp, last_executed_timestamp, last_executed_micros, last_write_timestamp));
static final Columns regulars = Columns.from(Lists.newArrayList(data));
private static final RegularAndStaticColumns all = new RegularAndStaticColumns(statics, regulars);
private static final RegularAndStaticColumns justStatic = new RegularAndStaticColumns(statics, Columns.NONE);
private static final RegularAndStaticColumns justRegular = new RegularAndStaticColumns(Columns.NONE, regulars);
static boolean hasStaticChanges(CommandsForKey original, CommandsForKey current)
static ByteBuffer makePartitionKey(int storeId, Key key)
{
return valueModified(CommandsForKey::max, original, current)
|| valueModified(CommandsForKey::lastExecutedTimestamp, original, current)
|| valueModified(CommandsForKey::lastWriteTimestamp, original, current)
|| valueModified(CommandsForKey::rawLastExecutedHlc, original, current);
PartitionKey pk = (PartitionKey) key;
return keyComparator.make(storeId, serializeToken(pk.token()), serializeKey(pk)).serializeAsPartitionKey();
}
private static boolean hasRegularChanges(CommandsForKey original, CommandsForKey current)
static ByteBuffer makePartitionKey(int storeId, TimestampsForKey timestamps)
{
return valueModified(CommandsForKey::byId, original, current)
|| valueModified(CommandsForKey::byExecuteAt, original, current);
}
static RegularAndStaticColumns columnsFor(CommandsForKey original, CommandsForKey current)
{
boolean hasStaticChanges = hasStaticChanges(original, current);
boolean hasRegularChanges = hasRegularChanges(original, current);
if (hasStaticChanges && hasRegularChanges)
return all;
else if (hasStaticChanges)
return justStatic;
else if (hasRegularChanges)
return justRegular;
else
throw new IllegalArgumentException("No Static or Regular columns changed for CFK " + current.key());
return makePartitionKey(storeId, timestamps.key());
}
}
public static class CommandsForKeyRows extends CommandsForKeyColumns
public static class TimestampsForKeyRows extends TimestampsForKeyColumns
{
public static ByteBuffer[] splitPartitionKey(DecoratedKey key)
{
@ -507,6 +461,11 @@ public class AccordKeyspace
return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
}
public static PartitionKey getKey(ByteBuffer[] partitionKeyComponents)
{
return deserializeKey(partitionKeyComponents[key.position()]);
}
@Nullable
public static Timestamp getMaxTimestamp(Row row)
{
@ -533,17 +492,6 @@ public class AccordKeyspace
return cell.accessor().getLong(cell.value(), 0);
}
public static PartitionKey getKey(ByteBuffer[] partitionKeyComponents)
{
return deserializeKey(partitionKeyComponents[key.position()]);
}
@Nullable
public static Timestamp getTimestamp(Row row)
{
return deserializeTimestampOrNull(row.clustering().bufferAt(CommandsForKeyColumns.timestamp.position()), Timestamp::fromBits);
}
@Nullable
public static Timestamp getLastWriteTimestamp(Row row)
{
@ -553,12 +501,12 @@ public class AccordKeyspace
return deserializeTimestampOrNull(cell.value(), cell.accessor(), Timestamp::fromBits);
}
public static Row truncateStaticRow(long nowInSec, Row row, Cell lastExecuteMicrosCell, Cell lastExecuteCell, Cell lastWriteCell, Cell maxTimestampCell)
public static Row truncateTimestampsForKeyRow(long nowInSec, Row row, Cell lastExecuteMicrosCell, Cell lastExecuteCell, Cell lastWriteCell, Cell maxTimestampCell)
{
checkArgument(lastExecuteMicrosCell == null || lastExecuteMicrosCell.column() == CommandsForKeyColumns.last_executed_micros);
checkArgument(lastExecuteCell == null || lastExecuteCell.column() == CommandsForKeyColumns.last_executed_timestamp);
checkArgument(lastWriteCell == null || lastWriteCell.column() == CommandsForKeyColumns.last_write_timestamp);
checkArgument(maxTimestampCell == null || maxTimestampCell.column() == CommandsForKeyColumns.max_timestamp);
checkArgument(lastExecuteMicrosCell == null || lastExecuteMicrosCell.column() == last_executed_micros);
checkArgument(lastExecuteCell == null || lastExecuteCell.column() == last_executed_timestamp);
checkArgument(lastWriteCell == null || lastWriteCell.column() == last_write_timestamp);
checkArgument(maxTimestampCell == null || maxTimestampCell.column() == max_timestamp);
long timestamp = row.primaryKeyLivenessInfo().timestamp();
@ -572,8 +520,7 @@ public class AccordKeyspace
if (maxTimestampCell != null)
colCount++;
ColumnMetadata[] fields = CommandsForKeyColumns.static_columns_metadata;
checkState(fields.length >= colCount, "CommandsForKeyColumns.static_columns_metadata should include all the columns");
checkState(columns.size() >= colCount, "CommandsForKeyColumns.static_columns_metadata should include all the columns");
Object[] newLeaf = BTree.unsafeAllocateNonEmptyLeaf(colCount);
int colIndex = 0;
@ -591,6 +538,79 @@ public class AccordKeyspace
}
}
private static final TableMetadata DepsCommandsForKeys = commandsForKeysTable(DEPS_COMMANDS_FOR_KEY);
private static final TableMetadata AllCommandsForKeys = commandsForKeysTable(ALL_COMMANDS_FOR_KEY);
private static TableMetadata commandsForKeysTable(String tableName)
{
return parse(tableName,
"accord commands per key",
"CREATE TABLE %s ("
+ "store_id int, "
+ "key_token blob, " // can't use "token" as this is restricted word in CQL
+ format("key %s, ", KEY_TUPLE)
+ format("timestamp %s, ", TIMESTAMP_TUPLE)
+ "data blob, "
+ "PRIMARY KEY((store_id, key_token, key), timestamp)"
+ ')')
.partitioner(new LocalPartitioner(CompositeType.getInstance(Int32Type.instance, BytesType.instance, KEY_TYPE)))
.build();
}
public static class CommandsForKeyAccessor
{
final TableMetadata table;
final ClusteringComparator keyComparator;
final CompositeType partitionKeyType;
final ColumnFilter allColumns;
final ColumnMetadata store_id;
final ColumnMetadata key_token;
final ColumnMetadata key;
final ColumnMetadata timestamp;
final ColumnMetadata data;
final RegularAndStaticColumns columns;
public CommandsForKeyAccessor(TableMetadata table)
{
this.table = table;
this.keyComparator = table.partitionKeyAsClusteringComparator();
this.partitionKeyType = (CompositeType) table.partitionKeyType;
this.allColumns = ColumnFilter.all(table);
this.store_id = getColumn(table, "store_id");
this.key_token = getColumn(table, "key_token");
this.key = getColumn(table, "key");
this.timestamp = getColumn(table, "timestamp");
this.data = getColumn(table, "data");
this.columns = new RegularAndStaticColumns(Columns.NONE, Columns.from(Lists.newArrayList(data)));
}
public ByteBuffer[] splitPartitionKey(DecoratedKey key)
{
return partitionKeyType.split(key.getKey());
}
public int getStoreId(ByteBuffer[] partitionKeyComponents)
{
return Int32Type.instance.compose(partitionKeyComponents[store_id.position()]);
}
public PartitionKey getKey(ByteBuffer[] partitionKeyComponents)
{
return deserializeKey(partitionKeyComponents[key.position()]);
}
@Nullable
public Timestamp getTimestamp(Row row)
{
return deserializeTimestampOrNull(row.clustering().bufferAt(timestamp.position()), Timestamp::fromBits);
}
}
public static final CommandsForKeyAccessor DepsCommandsForKeysAccessor = new CommandsForKeyAccessor(DepsCommandsForKeys);
public static final CommandsForKeyAccessor AllCommandsForKeysAccessor = new CommandsForKeyAccessor(AllCommandsForKeys);
private static final TableMetadata Topologies =
parse(TOPOLOGIES,
"accord topologies",
@ -648,7 +668,7 @@ public class AccordKeyspace
private static Tables tables()
{
return Tables.of(Commands, CommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata);
return Tables.of(Commands, TimestampsForKeys, DepsCommandsForKeys, AllCommandsForKeys, Topologies, EpochMetadata, CommandStoreMetadata);
}
private static <T> ByteBuffer serialize(T obj, LocalVersionedSerializer<T> serializer) throws IOException
@ -871,6 +891,14 @@ public class AccordKeyspace
return Timestamp.fromBits(accessor.getLong(split.get(0), 0), accessor.getLong(split.get(1), 0), new Node.Id(accessor.getInt(split.get(2), 0)));
}
public static <V, T extends Timestamp> T deserializeTimestampOrDefault(V value, ValueAccessor<V> accessor, TimestampFactory<T> factory, T defaultVal)
{
if (value == null || accessor.isEmpty(value))
return defaultVal;
List<V> split = TIMESTAMP_TYPE.unpack(value, accessor);
return factory.create(accessor.getLong(split.get(0), 0), accessor.getLong(split.get(1), 0), new Node.Id(accessor.getInt(split.get(2), 0)));
}
public static <V, T extends Timestamp> T deserializeTimestampOrNull(V value, ValueAccessor<V> accessor, TimestampFactory<T> factory)
{
if (value == null || accessor.isEmpty(value))
@ -884,6 +912,11 @@ public class AccordKeyspace
return deserializeTimestampOrNull(row.getBlob(name), factory);
}
private static <T extends Timestamp> T deserializeTimestampOrDefault(UntypedResultSet.Row row, String name, TimestampFactory<T> factory, T defaultVal)
{
return deserializeTimestampOrDefault(row.getBlob(name), ByteBufferAccessor.instance, factory, defaultVal);
}
private static ByteBuffer bytesOrNull(Row row, ColumnMetadata column)
{
Cell<?> cell = row.getCell(column);
@ -1093,14 +1126,14 @@ public class AccordKeyspace
this.start = start;
this.end = end;
String selection = selection(CommandsForKeys, requiredColumns, COLUMNS_FOR_ITERATION);
String selection = selection(TimestampsForKeys, requiredColumns, COLUMNS_FOR_ITERATION);
this.cqlFirst = format("SELECT DISTINCT %s\n" +
"FROM %s\n" +
"WHERE store_id = ?\n" +
(startInclusive ? " AND key_token >= ?\n" : " AND key_token > ?\n") +
(endInclusive ? " AND key_token <= ?\n" : " AND key_token < ?\n") +
"ALLOW FILTERING",
selection, CommandsForKeys);
selection, TimestampsForKeys);
this.cqlContinue = format("SELECT DISTINCT %s\n" +
"FROM %s\n" +
"WHERE store_id = ?\n" +
@ -1108,7 +1141,7 @@ public class AccordKeyspace
" AND key > ?\n" +
(endInclusive ? " AND key_token <= ?\n" : " AND key_token < ?\n") +
"ALLOW FILTERING",
selection, CommandsForKeys);
selection, TimestampsForKeys);
}
@Override
@ -1273,111 +1306,30 @@ public class AccordKeyspace
return deserializeKey(row.getBytes("key"));
}
private static void addSeriesMutations(ImmutableSortedMap<Timestamp, ByteBuffer> prev,
ImmutableSortedMap<Timestamp, ByteBuffer> value,
SeriesKind kind,
PartitionUpdate.Builder partitionBuilder,
Row.Builder rowBuilder,
LivenessInfo livenessInfo,
int nowInSeconds)
{
if (prev == value)
return;
long timestampMicros = livenessInfo.timestamp();
Set<Timestamp> deletions = Sets.difference(prev.keySet(), value.keySet());
Row.Deletion deletion = !deletions.isEmpty() ?
Row.Deletion.regular(DeletionTime.build(timestampMicros, nowInSeconds)) :
null;
ByteBuffer ordinalBytes = bytes(kind.ordinal());
value.forEach((timestamp, bytes) -> {
if (bytes.equals(prev.get(timestamp)))
return;
rowBuilder.newRow(Clustering.make(ordinalBytes, serializeTimestamp(timestamp)));
rowBuilder.addCell(live(CommandsForKeyColumns.data, timestampMicros, bytes));
rowBuilder.addPrimaryKeyLivenessInfo(livenessInfo);
partitionBuilder.add(rowBuilder.build());
});
deletions.forEach(timestamp -> {
rowBuilder.newRow(Clustering.make(ordinalBytes, serializeTimestamp(timestamp)));
rowBuilder.addRowDeletion(deletion);
partitionBuilder.add(rowBuilder.build());
});
}
private static void addSeriesMutations(CommandsForKey original,
CommandsForKey cfk,
SeriesKind kind,
PartitionUpdate.Builder partitionBuilder,
Row.Builder rowBuilder,
LivenessInfo livenessInfo,
int nowInSeconds)
{
addSeriesMutations(kind.getValues(original), kind.getValues(cfk), kind, partitionBuilder, rowBuilder, livenessInfo, nowInSeconds);
}
private static DecoratedKey makeKey(int storeId, PartitionKey key)
{
Token token = key.token();
ByteBuffer pk = CommandsForKeyColumns.keyComparator.make(storeId,
serializeToken(token),
serializeKey(key)).serializeAsPartitionKey();
return CommandsForKeys.partitioner.decorateKey(pk);
}
private static DecoratedKey makeKey(int storeId, CommandsForKey cfk)
{
return makeKey(storeId, (PartitionKey) cfk.key());
}
public static Mutation getCommandsForKeyMutation(AccordCommandStore commandStore, AccordSafeCommandsForKey liveCfk, long timestampMicros)
{
return getCommandsForKeyMutation(commandStore.id(), liveCfk.original(), liveCfk.current(), timestampMicros);
}
public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKey original, CommandsForKey cfk, long timestampMicros)
public static Mutation getTimestampsForKeyMutation(int storeId, TimestampsForKey original, TimestampsForKey current, long timestampMicros)
{
try
{
Invariants.checkArgument(original != cfk);
Invariants.checkArgument(original != current);
// TODO: convert to byte arrays
ValueAccessor<ByteBuffer> accessor = ByteBufferAccessor.instance;
Row.Builder builder = BTreeRow.unsortedBuilder();
builder.newRow(Clustering.EMPTY);
int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
LivenessInfo livenessInfo = LivenessInfo.create(timestampMicros, nowInSeconds);
builder.addPrimaryKeyLivenessInfo(livenessInfo);
addCellIfModified(TimestampsForKeyColumns.max_timestamp, TimestampsForKey::max, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
addCellIfModified(TimestampsForKeyColumns.last_executed_timestamp, TimestampsForKey::lastExecutedTimestamp, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
addCellIfModified(TimestampsForKeyColumns.last_executed_micros, TimestampsForKey::rawLastExecutedHlc, accessor::valueOf, builder, timestampMicros, nowInSeconds, original, current);
addCellIfModified(TimestampsForKeyColumns.last_write_timestamp, TimestampsForKey::lastWriteTimestamp, AccordKeyspace::serializeTimestamp, builder, timestampMicros, nowInSeconds, original, current);
boolean hasStaticChanges = CommandsForKeyColumns.hasStaticChanges(original, cfk);
int expectedRows = (hasStaticChanges ? 1 : 0)
+ estimateMapChanges(c -> c.byId().commands, original, cfk)
+ estimateMapChanges(c -> c.byExecuteAt().commands, original, cfk);
PartitionUpdate.Builder partitionBuilder = new PartitionUpdate.Builder(CommandsForKeys,
makeKey(storeId, cfk),
CommandsForKeyColumns.columnsFor(original, cfk),
expectedRows);
Row.Builder rowBuilder = BTreeRow.unsortedBuilder();
if (hasStaticChanges)
{
rowBuilder.newRow(Clustering.STATIC_CLUSTERING);
addCellIfModified(CommandsForKeyColumns.max_timestamp, CommandsForKey::max, AccordKeyspace::serializeTimestamp, rowBuilder, timestampMicros, nowInSeconds, original, cfk);
addCellIfModified(CommandsForKeyColumns.last_executed_timestamp, CommandsForKey::lastExecutedTimestamp, AccordKeyspace::serializeTimestamp, rowBuilder, timestampMicros, nowInSeconds, original, cfk);
addCellIfModified(CommandsForKeyColumns.last_executed_micros, CommandsForKey::rawLastExecutedHlc, accessor::valueOf, rowBuilder, timestampMicros, nowInSeconds, original, cfk);
addCellIfModified(CommandsForKeyColumns.last_write_timestamp, CommandsForKey::lastWriteTimestamp, AccordKeyspace::serializeTimestamp, rowBuilder, timestampMicros, nowInSeconds, original, cfk);
rowBuilder.addPrimaryKeyLivenessInfo(livenessInfo);
Row row = rowBuilder.build();
if (!row.isEmpty())
partitionBuilder.add(row);
}
addSeriesMutations(original, cfk, SeriesKind.BY_ID, partitionBuilder, rowBuilder, livenessInfo, nowInSeconds);
addSeriesMutations(original, cfk, SeriesKind.BY_EXECUTE_AT, partitionBuilder, rowBuilder, livenessInfo, nowInSeconds);
PartitionUpdate update = partitionBuilder.build();
if (update.isEmpty())
Row row = builder.build();
if (row.isEmpty())
return null;
ByteBuffer key = TimestampsForKeyColumns.makePartitionKey(storeId, current.key());
PartitionUpdate update = PartitionUpdate.singleRowUpdate(TimestampsForKeys, key, row);
return new Mutation(update);
}
catch (IOException e)
@ -1386,6 +1338,158 @@ public class AccordKeyspace
}
}
public static Mutation getTimestampsForKeyMutation(AccordCommandStore commandStore, AccordSafeTimestampsForKey liveTimestamps, long timestampMicros)
{
return getTimestampsForKeyMutation(commandStore.id(), liveTimestamps.original(), liveTimestamps.current(), timestampMicros);
}
public static UntypedResultSet loadTimestampsForKeyRow(CommandStore commandStore, PartitionKey key)
{
String cql = "SELECT * FROM %s.%s " +
"WHERE store_id = ? " +
"AND key_token = ? " +
"AND key=(?, ?)";
return executeInternal(format(cql, ACCORD_KEYSPACE_NAME, TIMESTAMPS_FOR_KEY),
commandStore.id(),
serializeToken(key.token()),
key.tableId().asUUID(), key.partitionKey().getKey());
}
public static TimestampsForKey loadTimestampsForKey(AccordCommandStore commandStore, PartitionKey key)
{
commandStore.checkNotInStoreThread();
return unsafeLoadTimestampsForKey(commandStore, key);
}
public static TimestampsForKey unsafeLoadTimestampsForKey(AccordCommandStore commandStore, PartitionKey key)
{
UntypedResultSet rows = loadTimestampsForKeyRow(commandStore, key);
if (rows.isEmpty())
{
return null;
}
UntypedResultSet.Row row = rows.one();
checkState(deserializeKey(row).equals(key));
Timestamp max = deserializeTimestampOrDefault(row, "max_timestamp", Timestamp::fromBits, Timestamp.NONE);
Timestamp lastExecutedTimestamp = deserializeTimestampOrDefault(row, "last_executed_timestamp", Timestamp::fromBits, Timestamp.NONE);
long lastExecutedMicros = row.has("last_executed_micros") ? row.getLong("last_executed_micros") : 0;
Timestamp lastWriteTimestamp = deserializeTimestampOrDefault(row, "last_write_timestamp", Timestamp::fromBits, Timestamp.NONE);
return TimestampsForKey.SerializerSupport.create(key, max, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp);
}
private static <T extends Timestamp> void addSeriesMutations(CommandsForKeyAccessor accessor,
CommandTimeseries.Update<T, ByteBuffer> update,
PartitionUpdate.Builder partitionBuilder,
Row.Builder rowBuilder,
LivenessInfo livenessInfo,
long timestampMicros,
Row.Deletion deletion,
Predicate<T> predicate)
{
if (update.isEmpty())
return;
update.forEachWrite((timestamp, bytes) -> {
if (!predicate.test(timestamp))
return;
rowBuilder.newRow(Clustering.make(serializeTimestamp(timestamp)));
rowBuilder.addCell(live(accessor.data, timestampMicros, bytes));
rowBuilder.addPrimaryKeyLivenessInfo(livenessInfo);
partitionBuilder.add(rowBuilder.build());
});
update.forEachDelete(timestamp -> {
if (!predicate.test(timestamp))
return;
rowBuilder.newRow(Clustering.make(serializeTimestamp(timestamp)));
rowBuilder.addRowDeletion(deletion);
partitionBuilder.add(rowBuilder.build());
});
}
private static <T extends Timestamp> void addSeriesMutations(CommandsForKeyAccessor accessor,
CommandTimeseries.Update<T, ByteBuffer> common,
CommandTimeseries.Update<T, ByteBuffer> update,
PartitionUpdate.Builder partitionBuilder,
Row.Builder rowBuilder,
LivenessInfo livenessInfo,
int nowInSeconds)
{
long timestampMicros = livenessInfo.timestamp();
Row.Deletion deletion = common.numDeletes() + update.numDeletes() > 0 ?
Row.Deletion.regular(DeletionTime.build(timestampMicros, nowInSeconds)) :
null;
addSeriesMutations(accessor, common, partitionBuilder, rowBuilder, livenessInfo, timestampMicros, deletion, ts -> !update.contains(ts));
addSeriesMutations(accessor, update, partitionBuilder, rowBuilder, livenessInfo, timestampMicros, deletion, ts -> true);
}
private static DecoratedKey makeKey(CommandsForKeyAccessor accessor, int storeId, PartitionKey key)
{
Token token = key.token();
ByteBuffer pk = accessor.keyComparator.make(storeId,
serializeToken(token),
serializeKey(key)).serializeAsPartitionKey();
return accessor.table.partitioner.decorateKey(pk);
}
private static PartitionUpdate getCommandsForKeyPartitionUpdate(CommandsForKeyAccessor accessor, int storeId, PartitionKey key, CommandsForKeyUpdater<ByteBuffer> common, CommandsForKeyUpdater<ByteBuffer> update, long timestampMicros)
{
int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
LivenessInfo livenessInfo = LivenessInfo.create(timestampMicros, nowInSeconds);
int expectedRows = common.totalChanges() + update.totalChanges();
PartitionUpdate.Builder partitionBuilder = new PartitionUpdate.Builder(accessor.table,
makeKey(accessor, storeId, key),
accessor.columns,
expectedRows);
Row.Builder rowBuilder = BTreeRow.unsortedBuilder();
addSeriesMutations(accessor, common.commands(), update.commands(), partitionBuilder, rowBuilder, livenessInfo, nowInSeconds);
PartitionUpdate partitionUpdate = partitionBuilder.build();
if (partitionUpdate.isEmpty())
return null;
return partitionUpdate;
}
public static Mutation getCommandsForKeyMutation(int storeId, CommandsForKeyUpdate update, long timestampMicros)
{
PartitionUpdate depsUpdate = getCommandsForKeyPartitionUpdate(DepsCommandsForKeysAccessor,
storeId,
update.key(),
update.common(),
update.deps(),
timestampMicros);
PartitionUpdate allUpdate = getCommandsForKeyPartitionUpdate(AllCommandsForKeysAccessor,
storeId,
update.key(),
update.common(),
update.all(),
timestampMicros);
if (depsUpdate == null && allUpdate == null)
return null;
if (depsUpdate == null)
return new Mutation(allUpdate);
else if (allUpdate == null)
return new Mutation(depsUpdate);
return new Mutation(ACCORD_KEYSPACE_NAME, depsUpdate.partitionKey(),
ImmutableMap.of(depsUpdate.metadata().id, depsUpdate, allUpdate.metadata().id, allUpdate),
MonotonicClock.Global.approxTime.now(), false);
}
private static <T> ByteBuffer cellValue(Cell<T> cell)
{
return cell.accessor().toBuffer(cell.value());
@ -1403,32 +1507,35 @@ public class AccordKeyspace
return clustering.accessor().toBuffer(clustering.get(idx));
}
public static SinglePartitionReadCommand getCommandsForKeyRead(int storeId, PartitionKey key, long nowInSeconds)
private static SinglePartitionReadCommand getCommandsForKeyRead(CommandsForKeyAccessor accessor, int storeId, PartitionKey key, long nowInSeconds)
{
return SinglePartitionReadCommand.create(CommandsForKeys, nowInSeconds,
CommandsForKeyColumns.allColumns,
return SinglePartitionReadCommand.create(accessor.table, nowInSeconds,
accessor.allColumns,
RowFilter.none(),
DataLimits.NONE,
makeKey(storeId, key),
makeKey(accessor, storeId, key),
FULL_PARTITION);
}
public static CommandsForKey loadCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
public static SinglePartitionReadCommand getDepsCommandsForKeyRead(int storeId, PartitionKey key, int nowInSeconds)
{
commandStore.checkNotInStoreThread();
return unsafeLoadCommandsForKey(commandStore, key);
return getCommandsForKeyRead(DepsCommandsForKeysAccessor, storeId, key, nowInSeconds);
}
static CommandsForKey unsafeLoadCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
public static SinglePartitionReadCommand getAllCommandsForKeyRead(int storeId, PartitionKey key, int nowInSeconds)
{
return getCommandsForKeyRead(AllCommandsForKeysAccessor, storeId, key, nowInSeconds);
}
static CommandsForKey unsafeLoadCommandsForKey(CommandsForKeyAccessor accessor, AccordCommandStore commandStore, PartitionKey key)
{
long timestampMicros = TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis());
int nowInSeconds = (int) TimeUnit.MICROSECONDS.toSeconds(timestampMicros);
SinglePartitionReadCommand command = getCommandsForKeyRead(commandStore.id(), key, nowInSeconds);
SinglePartitionReadCommand command = getCommandsForKeyRead(accessor, commandStore.id(), key, nowInSeconds);
EnumMap<SeriesKind, ImmutableSortedMap.Builder<Timestamp, ByteBuffer>> seriesMaps = new EnumMap<>(SeriesKind.class);
for (SeriesKind kind : SeriesKind.values())
seriesMaps.put(kind, new ImmutableSortedMap.Builder<>(Comparator.naturalOrder()));
ImmutableSortedMap.Builder<Timestamp, ByteBuffer> commands = new ImmutableSortedMap.Builder<>(Comparator.naturalOrder());
try (ReadExecutionController controller = command.executionController();
FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
@ -1438,42 +1545,22 @@ public class AccordKeyspace
return null;
}
Timestamp max = Timestamp.NONE;
Timestamp lastExecutedTimestamp = Timestamp.NONE;
long lastExecutedMicros = 0;
Timestamp lastWriteTimestamp = Timestamp.NONE;
try (RowIterator partition = partitions.next())
{
// empty static row will be interpreted as all null cells which will cause everything to be initialized
Row staticRow = partition.staticRow();
max = deserializeTimestampOrDefault(staticRow, CommandsForKeyColumns.max_timestamp, Timestamp::fromBits, max);
lastExecutedTimestamp = deserializeTimestampOrDefault(staticRow, CommandsForKeyColumns.last_executed_timestamp, Timestamp::fromBits, lastExecutedTimestamp);
ByteBuffer microsBytes = bytesOrNull(staticRow, CommandsForKeyColumns.last_executed_micros);
if (microsBytes != null)
lastExecutedMicros = microsBytes.getLong(microsBytes.position());
lastWriteTimestamp = deserializeTimestampOrDefault(staticRow, CommandsForKeyColumns.last_write_timestamp, Timestamp::fromBits, lastWriteTimestamp);
while (partition.hasNext())
{
Row row = partition.next();
Clustering<?> clustering = row.clustering();
int ordinal = Int32Type.instance.compose(clusteringValue(clustering, 0));
Timestamp timestamp = deserializeTimestampOrNull(clusteringValue(clustering, 1), Timestamp::fromBits);
ByteBuffer data = cellValue(row, CommandsForKeyColumns.data);
Timestamp timestamp = deserializeTimestampOrNull(clusteringValue(clustering, 0), Timestamp::fromBits);
ByteBuffer data = cellValue(row, accessor.data);
if (data == null)
continue;
seriesMaps.get(SeriesKind.values()[ordinal]).put(timestamp, data);
commands.put(timestamp, data);
}
}
checkState(!partitions.hasNext());
return CommandsForKey.SerializerSupport.create(key, max, lastExecutedTimestamp, lastExecutedMicros, lastWriteTimestamp,
CommandsForKeySerializer.loader,
seriesMaps.get(SeriesKind.BY_ID).build(),
seriesMaps.get(SeriesKind.BY_EXECUTE_AT).build());
return CommandsForKey.SerializerSupport.create(key, CommandsForKeySerializer.loader, commands.build());
}
catch (Throwable t)
{
@ -1482,6 +1569,28 @@ public class AccordKeyspace
}
}
public static CommandsForKey unsafeLoadDepsCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
{
return unsafeLoadCommandsForKey(DepsCommandsForKeysAccessor, commandStore, key);
}
public static CommandsForKey unsafeLoadAllCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
{
return unsafeLoadCommandsForKey(AllCommandsForKeysAccessor, commandStore, key);
}
public static CommandsForKey loadDepsCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
{
commandStore.checkNotInStoreThread();
return unsafeLoadCommandsForKey(DepsCommandsForKeysAccessor, commandStore, key);
}
public static CommandsForKey loadAllCommandsForKey(AccordCommandStore commandStore, PartitionKey key)
{
commandStore.checkNotInStoreThread();
return unsafeLoadCommandsForKey(AllCommandsForKeysAccessor, commandStore, key);
}
public static class EpochDiskState
{
public static final EpochDiskState EMPTY = new EpochDiskState(0, 0);

2
src/java/org/apache/cassandra/service/accord/AccordMessageSink.java
View File

@ -164,7 +164,7 @@ public class AccordMessageSink implements MessageSink
for (MessageType type : Iterables.concat(AccordMessageType.values, MessageType.values))
{
// Any request can receive a generic failure response
if (type == MessageType.FAILURE_RSP)
if (type == MessageType.FAILURE_RSP || type.isLocal())
continue;
if (mapping.containsKey(type))

54
src/java/org/apache/cassandra/service/accord/AccordObjectSizes.java
View File

@ -28,11 +28,13 @@ import accord.api.Key;
import accord.api.Result;
import accord.api.RoutingKey;
import accord.impl.CommandsForKey;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.Command.WaitingOn;
import accord.local.CommonAttributes;
import accord.local.Node;
import accord.local.SaveStatus;
import accord.local.Status;
import accord.primitives.AbstractKeys;
import accord.primitives.AbstractRanges;
import accord.primitives.Ballot;
@ -271,16 +273,29 @@ public class AccordObjectSizes
{
private final static TokenKey EMPTY_KEY = new TokenKey("doesnotexist", null);
private final static TxnId EMPTY_TXNID = new TxnId(42, 42, Kind.Read, Domain.Key, new Node.Id(42));
private final static CommonAttributes.Mutable EMPTY_ATTRS = new CommonAttributes.Mutable(EMPTY_TXNID)
.partialDeps(PartialDeps.NONE)
.route(new FullKeyRoute(EMPTY_KEY, true, new RoutingKey[] {EMPTY_KEY} ));
final static long NOT_DEFINED = measure(Command.SerializerSupport.notDefined(EMPTY_ATTRS, Ballot.ZERO));
final static long PREACCEPTED = measure(Command.SerializerSupport.preaccepted(EMPTY_ATTRS, EMPTY_TXNID, null));;
final static long ACCEPTED = measure(Command.SerializerSupport.accepted(EMPTY_ATTRS, SaveStatus.Accepted, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO));
final static long COMMITTED = measure(Command.SerializerSupport.committed(EMPTY_ATTRS, SaveStatus.Committed, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.EMPTY));
final static long EXECUTED = measure(Command.SerializerSupport.executed(EMPTY_ATTRS, SaveStatus.Applied, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.EMPTY, null, null));
final static long TRUNCATED = measure(Command.SerializerSupport.truncatedApply(EMPTY_ATTRS, SaveStatus.TruncatedApply, EMPTY_TXNID, null, null));
private static CommonAttributes attrs(boolean hasDeps, boolean hasTxn)
{
CommonAttributes.Mutable attrs = new CommonAttributes.Mutable(EMPTY_TXNID).route(new FullKeyRoute(EMPTY_KEY, true, new RoutingKey[]{ EMPTY_KEY }));
attrs.durability(Status.Durability.NotDurable);
if (hasDeps)
attrs.partialDeps(PartialDeps.NONE);
if (hasTxn)
attrs.partialTxn(new PartialTxn.InMemory(null, null, null, null, null, null));
return attrs;
}
private static final Writes EMPTY_WRITES = new Writes(EMPTY_TXNID, EMPTY_TXNID, Keys.EMPTY, (key, safeStore, executeAt, store, txn) -> null);
private static final Result EMPTY_RESULT = new Result() {};
final static long NOT_DEFINED = measure(Command.SerializerSupport.notDefined(attrs(false, false), Ballot.ZERO));
final static long PREACCEPTED = measure(Command.SerializerSupport.preaccepted(attrs(false, true), EMPTY_TXNID, null));;
final static long ACCEPTED = measure(Command.SerializerSupport.accepted(attrs(true, false), SaveStatus.Accepted, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO));
final static long COMMITTED = measure(Command.SerializerSupport.committed(attrs(true, true), SaveStatus.Committed, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.EMPTY));
final static long EXECUTED = measure(Command.SerializerSupport.executed(attrs(true, true), SaveStatus.Applied, EMPTY_TXNID, Ballot.ZERO, Ballot.ZERO, WaitingOn.EMPTY, EMPTY_WRITES, EMPTY_RESULT));
final static long TRUNCATED = measure(Command.SerializerSupport.truncatedApply(attrs(false, false), SaveStatus.TruncatedApply, EMPTY_TXNID, null, null));
final static long INVALIDATED = measure(Command.SerializerSupport.invalidated(EMPTY_TXNID, null));
private static long emptySize(Command command)
@ -354,18 +369,23 @@ public class AccordObjectSizes
return size;
}
private static final long EMPTY_CFK_SIZE = measure(CommandsForKey.SerializerSupport.create(null, null, null, 0, null, null,
ImmutableSortedMap.of(),
ImmutableSortedMap.of()));
private static long EMPTY_TFK_SIZE = measure(TimestampsForKey.SerializerSupport.create(null, null, null, 0, null));
public static long timestampsForKey(TimestampsForKey timestamps)
{
long size = EMPTY_TFK_SIZE;
size += timestamp(timestamps.max());
size += timestamp(timestamps.lastExecutedTimestamp());
size += timestamp(timestamps.lastWriteTimestamp());
return size;
}
private static long EMPTY_CFK_SIZE = measure(CommandsForKey.SerializerSupport.create(null, null, ImmutableSortedMap.of()));
public static long commandsForKey(CommandsForKey cfk)
{
long size = EMPTY_CFK_SIZE;
size += key(cfk.key());
size += timestamp(cfk.max());
size += timestamp(cfk.lastExecutedTimestamp());
size += timestamp(cfk.lastWriteTimestamp());
size += cfkSeriesSize((ImmutableSortedMap<Timestamp, ByteBuffer>) cfk.byId().commands);
size += cfkSeriesSize((ImmutableSortedMap<Timestamp, ByteBuffer>) cfk.byExecuteAt().commands);
size += cfkSeriesSize((ImmutableSortedMap<Timestamp, ByteBuffer>) cfk.commands().commands);
return size;
}
}

190
src/java/org/apache/cassandra/service/accord/AccordSafeCommandStore.java
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.service.accord;
import java.util.Map;
import java.util.NavigableMap;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import javax.annotation.Nullable;
@ -33,11 +32,14 @@ import accord.api.ProgressLog;
import accord.impl.AbstractSafeCommandStore;
import accord.impl.CommandTimeseries;
import accord.impl.CommandTimeseries.CommandLoader;
import accord.impl.CommandTimeseriesHolder;
import accord.impl.CommandsForKey;
import accord.impl.SafeCommandsForKey;
import accord.impl.CommandsForKeys;
import accord.impl.DomainCommands;
import accord.impl.DomainTimestamps;
import accord.impl.SafeTimestampsForKey;
import accord.local.CommandStores.RangesForEpoch;
import accord.local.CommonAttributes;
import accord.local.KeyHistory;
import accord.local.NodeTimeService;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
@ -53,24 +55,34 @@ import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.Txn;
import accord.primitives.TxnId;
import accord.utils.TriFunction;
import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeCommandsForKey>
public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeCommand, AccordSafeTimestampsForKey, AccordSafeCommandsForKey, AccordSafeCommandsForKeyUpdate>
{
private final Map<TxnId, AccordSafeCommand> commands;
private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeys;
private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys;
private final NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys;
private final NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKeys;
private final NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys;
private final AccordCommandStore commandStore;
private final RangesForEpoch ranges;
CommandsForRanges.Updater rangeUpdates = null;
public AccordSafeCommandStore(PreLoadContext context,
Map<TxnId, AccordSafeCommand> commands,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKey,
NavigableMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKey,
NavigableMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
NavigableMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys,
AccordCommandStore commandStore)
{
super(context);
this.commands = commands;
this.commandsForKeys = commandsForKey;
this.timestampsForKeys = timestampsForKey;
this.depsCommandsForKeys = depsCommandsForKey;
this.allCommandsForKeys = allCommandsForKeys;
this.updatesForKeys = updatesForKeys;
this.commandStore = commandStore;
this.ranges = commandStore.updateRangesForEpoch();
}
@ -87,18 +99,6 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
commands.put(command.txnId(), command);
}
@Override
protected AccordSafeCommandsForKey getCommandsForKeyInternal(RoutableKey key)
{
return commandsForKeys.get(key);
}
@Override
protected void addCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
{
commandsForKeys.put(cfk.key(), cfk);
}
@Override
protected AccordSafeCommand getIfLoaded(TxnId txnId)
{
@ -107,14 +107,100 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
return command;
}
@Override
protected AccordSafeCommandsForKey getIfLoaded(RoutableKey key)
private NavigableMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeyMap(KeyHistory history)
{
AccordSafeCommandsForKey cfk = commandStore.commandsForKeyCache().acquireIfLoaded(key);
switch (history)
{
case DEPS:
return depsCommandsForKeys;
case ALL:
return allCommandsForKeys;
default:
throw new IllegalArgumentException();
}
}
@Override
protected AccordSafeCommandsForKey getDepsCommandsForKeyInternal(RoutableKey key)
{
return depsCommandsForKeys.get(key);
}
@Override
protected void addDepsCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
{
depsCommandsForKeys.put(cfk.key(), cfk);
}
@Override
protected AccordSafeCommandsForKey getDepsCommandsForKeyIfLoaded(RoutableKey key)
{
AccordSafeCommandsForKey cfk = commandStore.depsCommandsForKeyCache().acquireIfLoaded(key);
if (cfk != null) cfk.preExecute();
return cfk;
}
@Override
protected AccordSafeCommandsForKey getAllCommandsForKeyInternal(RoutableKey key)
{
return allCommandsForKeys.get(key);
}
@Override
protected void addAllCommandsForKeyInternal(AccordSafeCommandsForKey cfk)
{
allCommandsForKeys.put(cfk.key(), cfk);
}
@Override
protected AccordSafeCommandsForKey getAllCommandsForKeyIfLoaded(RoutableKey key)
{
AccordSafeCommandsForKey cfk = commandStore.allCommandsForKeyCache().acquireIfLoaded(key);
if (cfk != null) cfk.preExecute();
return cfk;
}
@Override
protected AccordSafeTimestampsForKey getTimestampsForKeyInternal(RoutableKey key)
{
return timestampsForKeys.get(key);
}
@Override
protected void addTimestampsForKeyInternal(AccordSafeTimestampsForKey cfk)
{
timestampsForKeys.put(cfk.key(), cfk);
}
@Override
protected AccordSafeTimestampsForKey getTimestampsForKeyIfLoaded(RoutableKey key)
{
AccordSafeTimestampsForKey cfk = commandStore.timestampsForKeyCache().acquireIfLoaded(key);
if (cfk != null) cfk.preExecute();
return cfk;
}
protected AccordSafeCommandsForKeyUpdate getCommandsForKeyUpdateInternal(RoutableKey key)
{
return updatesForKeys.get(key);
}
protected AccordSafeCommandsForKeyUpdate createCommandsForKeyUpdateInternal(RoutableKey key)
{
throw new IllegalStateException("CFK updates should be initialized for operation");
}
protected void addCommandsForKeyUpdateInternal(AccordSafeCommandsForKeyUpdate update)
{
updatesForKeys.put(update.key(), update);
}
protected void applyCommandForKeyUpdates()
{
// TODO (now): should this happen as part of invalidate? Less obvious it's happening, but eliminates possibility of post update changes
updatesForKeys.values().forEach(AccordSafeCommandsForKeyUpdate::setUpdates);
}
@Override
public AccordCommandStore commandStore()
{
@ -160,7 +246,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
@Override
public Timestamp maxConflict(Seekables<?, ?> keysOrRanges, Ranges slice)
{
Timestamp maxConflict = mapReduce(keysOrRanges, slice, (ts, accum) -> Timestamp.max(ts.max(), accum), Timestamp.NONE, Predicates.isNull());
Timestamp maxConflict = mapReduce(keysOrRanges, slice, KeyHistory.NONE, (ts, commands, accum) -> Timestamp.max(ts.max(), accum), Timestamp.NONE, Predicates.isNull());
return Timestamp.nonNullOrMax(maxConflict, commandStore.commandsForRanges().maxRedundant());
}
@ -171,7 +257,6 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
// We find a set of dependencies for a range then update CommandsFor to know about them
Ranges allRanges = ranges.all();
deps.keyDeps.keys().forEach(allRanges, key -> {
SafeCommandsForKey cfk = commandsForKey(key);
deps.keyDeps.forEach(key, txnId -> {
// TODO (desired, efficiency): this can be made more efficient by batching by epoch
if (ranges.coordinates(txnId).contains(key))
@ -179,7 +264,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
if (!ranges.allBefore(txnId.epoch()).contains(key))
return;
cfk.registerNotWitnessed(txnId);
CommandsForKeys.registerNotWitnessed(this, key, txnId);
});
});
CommandsForRanges commandsForRanges = commandStore.commandsForRanges();
@ -202,15 +287,15 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
{
}
private <O> O mapReduce(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandTimeseriesHolder, O, O> map, O accumulate, Predicate<? super O> terminate)
private <O> O mapReduce(Routables<?> keysOrRanges, Ranges slice, KeyHistory keyHistory, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
{
accumulate = commandStore.mapReduceForRange(keysOrRanges, slice, map, accumulate, terminate);
if (terminate.test(accumulate))
return accumulate;
return mapReduceForKey(keysOrRanges, slice, map, accumulate, terminate);
return mapReduceForKey(keysOrRanges, slice, keyHistory, map, accumulate, terminate);
}
private <O> O mapReduceForKey(Routables<?> keysOrRanges, Ranges slice, BiFunction<CommandTimeseriesHolder, O, O> map, O accumulate, Predicate<? super O> terminate)
private <O> O mapReduceForKey(Routables<?> keysOrRanges, Ranges slice, KeyHistory keyHistory, TriFunction<DomainTimestamps, DomainCommands, O, O> map, O accumulate, Predicate<? super O> terminate)
{
switch (keysOrRanges.domain())
{
@ -223,8 +308,9 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
for (Key key : keys)
{
if (!slice.contains(key)) continue;
SafeCommandsForKey forKey = commandsForKey(key);
accumulate = map.apply(forKey.current(), accumulate);
SafeTimestampsForKey timestamps = timestampsForKey(key);
CommandsForKey commands = !keyHistory.isNone() ? commandsForKey(key, keyHistory).current() : null;
accumulate = map.apply(timestamps.current(), commands, accumulate);
if (terminate.test((accumulate)))
return accumulate;
}
@ -237,12 +323,13 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
Routables<?> sliced = keysOrRanges.slice(slice, Routables.Slice.Minimal);
if (!context.keys().slice(slice, Routables.Slice.Minimal).containsAll(sliced))
throw new AssertionError("Range(s) detected not present in the PreLoadContext: expected " + context.keys() + " but given " + keysOrRanges);
for (RoutableKey key : commandsForKeys.keySet())
for (RoutableKey key : timestampsForKeys.keySet())
{
//TODO (duplicate code): this is a repeat of Key... only change is checking contains in range
if (!sliced.contains(key)) continue;
SafeCommandsForKey forKey = commandsForKey(key);
accumulate = map.apply(forKey.current(), accumulate);
SafeTimestampsForKey timestamps = timestampsForKey(key);
CommandsForKey commands = !keyHistory.isNone() ? commandsForKey(key, keyHistory).current() : null;
accumulate = map.apply(timestamps.current(), commands, accumulate);
if (terminate.test(accumulate))
return accumulate;
}
@ -253,19 +340,25 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
}
@Override
public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, Txn.Kind.Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate) {
accumulate = mapReduce(keysOrRanges, slice, (forKey, prev) -> {
CommandTimeseries<?> timeseries;
//<<<<<<< HEAD
// public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, Txn.Kind.Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate) {
// accumulate = mapReduce(keysOrRanges, slice, (forKey, prev) -> {
// CommandTimeseries<?> timeseries;
//=======
public <P1, T> T mapReduce(Seekables<?, ?> keysOrRanges, Ranges slice, KeyHistory keyHistory, Txn.Kind.Kinds testKind, TestTimestamp testTimestamp, Timestamp timestamp, TestDep testDep, @Nullable TxnId depId, @Nullable Status minStatus, @Nullable Status maxStatus, CommandFunction<P1, T, T> map, P1 p1, T accumulate, Predicate<? super T> terminate)
{
accumulate = mapReduce(keysOrRanges, slice, keyHistory, (timestamps, commands, prev) -> {
CommandTimeseries.TimestampType timestampType;
switch (testTimestamp)
{
default: throw new AssertionError();
case STARTED_AFTER:
case STARTED_BEFORE:
timeseries = forKey.byId();
timestampType = CommandTimeseries.TimestampType.TXN_ID;
break;
case EXECUTES_AFTER:
case MAY_EXECUTE_BEFORE:
timeseries = forKey.byExecuteAt();
timestampType = CommandTimeseries.TimestampType.EXECUTE_AT;
}
CommandTimeseries.TestTimestamp remapTestTimestamp;
switch (testTimestamp)
@ -279,7 +372,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
case MAY_EXECUTE_BEFORE:
remapTestTimestamp = CommandTimeseries.TestTimestamp.BEFORE;
}
return timeseries.mapReduce(testKind, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, prev, terminate);
return commands.commands().mapReduce(testKind, timestampType, remapTestTimestamp, timestamp, testDep, depId, minStatus, maxStatus, map, p1, prev, terminate);
}, accumulate, terminate);
return accumulate;
@ -303,8 +396,7 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
Key key = seekable.asKey();
if (ranges.contains(key))
{
AccordSafeCommandsForKey cfk = commandsForKey(key);
cfk.register(liveCommand.current());
CommandsForKeys.registerCommand(this, key, liveCommand.current());
attrs = attrs.mutable().addListener(new CommandsForKey.Listener(key));
}
}
@ -340,7 +432,10 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
protected void invalidateSafeState()
{
commands.values().forEach(AccordSafeCommand::invalidate);
commandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
timestampsForKeys.values().forEach(AccordSafeTimestampsForKey::invalidate);
depsCommandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
allCommandsForKeys.values().forEach(AccordSafeCommandsForKey::invalidate);
updatesForKeys.values().forEach(AccordSafeCommandsForKeyUpdate::invalidate);
}
@Override
@ -350,11 +445,18 @@ public class AccordSafeCommandStore extends AbstractSafeCommandStore<AccordSafeC
}
public void postExecute(Map<TxnId, AccordSafeCommand> commands,
Map<RoutableKey, AccordSafeCommandsForKey> commandsForKeys)
Map<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey,
Map<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys,
Map<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys,
Map<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys
)
{
postExecute();
commands.values().forEach(AccordSafeState::postExecute);
commandsForKeys.values().forEach(AccordSafeState::postExecute);
timestampsForKey.values().forEach(AccordSafeState::postExecute);
depsCommandsForKeys.values().forEach(AccordSafeState::postExecute);
allCommandsForKeys.values().forEach(AccordSafeState::postExecute);
updatesForKeys.values().forEach(AccordSafeState::postExecute);
if (rangeUpdates != null)
rangeUpdates.apply();
}

24
src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKey.java
View File

@ -19,7 +19,6 @@
package org.apache.cassandra.service.accord;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
@ -27,7 +26,6 @@ import accord.api.Key;
import accord.impl.CommandsForKey;
import accord.impl.SafeCommandsForKey;
import accord.primitives.RoutableKey;
import accord.primitives.Timestamp;
public class AccordSafeCommandsForKey extends SafeCommandsForKey implements AccordSafeState<RoutableKey, CommandsForKey>
{
@ -110,27 +108,7 @@ public class AccordSafeCommandsForKey extends SafeCommandsForKey implements Acco
public void postExecute()
{
checkNotInvalidated();
global.set(current);
}
public long lastExecutedMicros()
{
return current().lastExecutedHlc();
}
public long timestampMicrosFor(Timestamp executeAt, boolean isForWriteTxn)
{
return current().hlcFor(executeAt, isForWriteTxn);
}
public int nowInSecondsFor(Timestamp executeAt, boolean isForWriteTxn)
{
CommandsForKey current = current();
current.validateExecuteAtTime(executeAt, isForWriteTxn);
// we use the executeAt time instead of the monotonic database timestamp to prevent uneven
// ttl expiration in extreme cases, ie 1M+ writes/second to a key causing timestamps to overflow
// into the next second on some keys and not others.
return Math.toIntExact(TimeUnit.MICROSECONDS.toSeconds(current.lastExecutedTimestamp().hlc()));
// updates are applied directly by CommandsForKeyUpdate
}
@Override

122
src/java/org/apache/cassandra/service/accord/AccordSafeCommandsForKeyUpdate.java Normal file
View File

@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.nio.ByteBuffer;
import com.google.common.annotations.VisibleForTesting;
import accord.api.Key;
import accord.impl.SafeCommandsForKey;
import accord.primitives.RoutableKey;
import accord.utils.async.AsyncChain;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
public class AccordSafeCommandsForKeyUpdate extends SafeCommandsForKey.Update<CommandsForKeyUpdate, ByteBuffer> implements AccordSafeState<RoutableKey, CommandsForKeyUpdate>
{
private boolean invalidated;
private final AccordCachingState<RoutableKey, CommandsForKeyUpdate> global;
private CommandsForKeyUpdate original;
private CommandsForKeyUpdate current;
public AccordSafeCommandsForKeyUpdate(AccordCachingState<RoutableKey, CommandsForKeyUpdate> global)
{
super((Key) global.key(), CommandsForKeySerializer.loader);
this.global = global;
this.original = null;
this.current = null;
}
@Override
public void initialize()
{
set(CommandsForKeyUpdate.empty((PartitionKey) key()));
}
@Override
public AccordCachingState<RoutableKey, CommandsForKeyUpdate> global()
{
checkNotInvalidated();
return global;
}
@Override
public CommandsForKeyUpdate current()
{
checkNotInvalidated();
return current;
}
public AsyncChain<?> loading()
{
throw new IllegalStateException("Updates aren't loaded");
}
@Override
@VisibleForTesting
public void set(CommandsForKeyUpdate cfk)
{
checkNotInvalidated();
this.current = cfk;
}
public CommandsForKeyUpdate original()
{
checkNotInvalidated();
return original;
}
public CommandsForKeyUpdate setUpdates()
{
CommandsForKeyUpdate next = new CommandsForKeyUpdate((PartitionKey) key(),
deps().toImmutable(),
all().toImmutable(),
common().toImmutable());
set(next);
return next;
}
@Override
public void preExecute()
{
checkNotInvalidated();
original = global.get();
current = original;
}
@Override
public void postExecute()
{
checkNotInvalidated();
global.set(current);
}
@Override
public void invalidate()
{
invalidated = true;
}
@Override
public boolean invalidated()
{
return invalidated;
}
}

146
src/java/org/apache/cassandra/service/accord/AccordSafeTimestampsForKey.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import accord.api.Key;
import accord.impl.SafeTimestampsForKey;
import accord.impl.TimestampsForKey;
import accord.primitives.RoutableKey;
import accord.primitives.Timestamp;
public class AccordSafeTimestampsForKey extends SafeTimestampsForKey implements AccordSafeState<RoutableKey, TimestampsForKey>
{
private boolean invalidated;
private final AccordCachingState<RoutableKey, TimestampsForKey> global;
private TimestampsForKey original;
private TimestampsForKey current;
public AccordSafeTimestampsForKey(AccordCachingState<RoutableKey, TimestampsForKey> global)
{
super((Key) global.key());
this.global = global;
this.original = null;
this.current = null;
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AccordSafeTimestampsForKey that = (AccordSafeTimestampsForKey) o;
return Objects.equals(original, that.original) && Objects.equals(current, that.current);
}
@Override
public int hashCode()
{
throw new UnsupportedOperationException();
}
@Override
public String toString()
{
return "AccordSafeTimestampsForKey{" +
"invalidated=" + invalidated +
", global=" + global +
", original=" + original +
", current=" + current +
'}';
}
@Override
public AccordCachingState<RoutableKey, TimestampsForKey> global()
{
checkNotInvalidated();
return global;
}
@Override
public TimestampsForKey current()
{
checkNotInvalidated();
return current;
}
@Override
@VisibleForTesting
public void set(TimestampsForKey cfk)
{
checkNotInvalidated();
this.current = cfk;
}
public TimestampsForKey original()
{
checkNotInvalidated();
return original;
}
@Override
public void preExecute()
{
checkNotInvalidated();
original = global.get();
current = original;
}
@Override
public void postExecute()
{
checkNotInvalidated();
global.set(current);
}
@Override
public void invalidate()
{
invalidated = true;
}
@Override
public boolean invalidated()
{
return invalidated;
}
public long lastExecutedMicros()
{
return current().lastExecutedHlc();
}
public static long timestampMicrosFor(TimestampsForKey timestamps, Timestamp executeAt, boolean isForWriteTxn)
{
return timestamps.hlcFor(executeAt, isForWriteTxn);
}
public static int nowInSecondsFor(TimestampsForKey timestamps, Timestamp executeAt, boolean isForWriteTxn)
{
timestamps.validateExecuteAtTime(executeAt, isForWriteTxn);
// we use the executeAt time instead of the monotonic database timestamp to prevent uneven
// ttl expiration in extreme cases, ie 1M+ writes/second to a key causing timestamps to overflow
// into the next second on some keys and not others.
return Math.toIntExact(TimeUnit.MICROSECONDS.toSeconds(timestamps.lastExecutedTimestamp().hlc()));
}
}

163
src/java/org/apache/cassandra/service/accord/AccordStateCache.java
View File

@ -26,10 +26,12 @@ import java.util.function.ToLongFunction;
import java.util.stream.Stream;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.utils.IntrusiveLinkedList;
import accord.utils.Invariants;
import accord.utils.async.AsyncChains;
import org.apache.cassandra.cache.CacheSize;
import org.apache.cassandra.concurrent.ExecutorPlus;
@ -38,7 +40,6 @@ import org.apache.cassandra.metrics.CacheAccessMetrics;
import org.apache.cassandra.service.accord.AccordCachingState.Status;
import static accord.utils.Invariants.checkState;
import static java.lang.String.format;
import static org.apache.cassandra.service.accord.AccordCachingState.Status.EVICTED;
import static org.apache.cassandra.service.accord.AccordCachingState.Status.FAILED_TO_LOAD;
import static org.apache.cassandra.service.accord.AccordCachingState.Status.LOADED;
@ -65,8 +66,14 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
VALIDATE_LOAD_ON_EVICT = value;
}
private final Map<Object, AccordCachingState<?, ?>> cache = new HashMap<>();
private final HashMap<Class<?>, Instance<?, ?, ?>> instances = new HashMap<>();
static class Stats
{
private long queries;
private long hits;
private long misses;
}
private ImmutableList<Instance<?, ?, ?>> instances = ImmutableList.of();
private final ExecutorPlus loadExecutor, saveExecutor;
@ -136,6 +143,8 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
AccordCachingState<?, ?> node = iter.next();
checkState(node.references == 0);
if (!node.canEvict())
continue;
/*
* TODO (expected, efficiency):
* can this be reworked so we're not skipping unevictable nodes everytime we try to evict?
@ -186,9 +195,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
if (!node.hasListeners())
{
AccordCachingState<?, ?> self = cache.remove(node.key());
if (self != null)
instance.itemsCached--;
AccordCachingState<?, ?> self = instances.get(node.index).cache.remove(node.key());
checkState(self == node, "Leaked node detected; was attempting to remove %s but cache had %s", node, self);
}
else
@ -199,29 +206,45 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
private Instance<?, ?, ?> instanceForNode(AccordCachingState<?, ?> node)
{
return instances.get(node.key().getClass());
return instances.get(node.index);
}
public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
Class<K> keyClass,
Class<? extends K> realKeyClass,
Class<S> valClass,
Function<AccordCachingState<K, V>, S> safeRefFactory,
Function<K, V> loadFunction,
BiFunction<V, V, Runnable> saveFunction,
BiFunction<K, V, Boolean> validateFunction,
ToLongFunction<V> heapEstimator,
AccordCachingState.Factory<K, V> nodeFactory)
{
int index = instances.size();
Instance<K, V, S> instance =
new Instance<>(index, keyClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, nodeFactory);
instances = ImmutableList.<Instance<?, ?, ?>>builder().addAll(instances).add(instance).build();
return instance;
}
public <K, V, S extends AccordSafeState<K, V>> Instance<K, V, S> instance(
Class<K> keyClass,
Class<S> valClass,
Function<AccordCachingState<K, V>, S> safeRefFactory,
Function<K, V> loadFunction,
BiFunction<V, V, Runnable> saveFunction,
BiFunction<K, V, Boolean> validateFunction,
ToLongFunction<V> heapEstimator)
{
Instance<K, V, S> instance =
new Instance<>(keyClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator);
if (instances.put(realKeyClass, instance) != null)
throw new IllegalArgumentException(format("Cache instances for key type %s already exists", realKeyClass.getName()));
return instance;
return instance(keyClass, valClass, safeRefFactory, loadFunction, saveFunction, validateFunction, heapEstimator, AccordCachingState.defaultFactory());
}
public class Instance<K, V, S extends AccordSafeState<K, V>> implements CacheSize
{
private final int index;
private final Class<K> keyClass;
private final Function<AccordCachingState<K, V>, S> safeRefFactory;
private Function<K, V> loadFunction;
@ -229,19 +252,24 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
private final BiFunction<K, V, Boolean> validateFunction;
private final ToLongFunction<V> heapEstimator;
private long bytesCached;
private int itemsCached;
// private int itemsCached;
@VisibleForTesting
final CacheAccessMetrics instanceMetrics;
private final Stats stats = new Stats();
private final Map<Object, AccordCachingState<?, ?>> cache = new HashMap<>();
private final AccordCachingState.Factory<K, V> nodeFactory;
public Instance(
Class<K> keyClass,
int index, Class<K> keyClass,
Function<AccordCachingState<K, V>, S> safeRefFactory,
Function<K, V> loadFunction,
BiFunction<V, V, Runnable> saveFunction,
BiFunction<K, V, Boolean> validateFunction,
ToLongFunction<V> heapEstimator)
ToLongFunction<V> heapEstimator,
AccordCachingState.Factory<K, V> nodeFactory)
{
this.index = index;
this.keyClass = keyClass;
this.safeRefFactory = safeRefFactory;
this.loadFunction = loadFunction;
@ -249,6 +277,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
this.validateFunction = validateFunction;
this.heapEstimator = heapEstimator;
this.instanceMetrics = metrics.forInstance(keyClass);
this.nodeFactory = nodeFactory;
}
public Stream<AccordCachingState<K, V>> stream()
@ -258,6 +287,34 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
.map(e -> (AccordCachingState<K, V>) e.getValue());
}
public S acquireOrInitialize(K key, Function<K, V> valueFactory)
{
incrementCacheQueries();
@SuppressWarnings("unchecked")
AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
if (node == null)
{
node = nodeFactory.create(key, index);
node.initialize(valueFactory.apply(key));
cache.put(key, node);
}
AccordCachingState<K, V> acquired = acquireExisting(node, true);
Invariants.checkState(acquired != null, "%s could not be acquired", node);
return safeRefFactory.apply(acquired);
}
public S acquireIfExists(K key)
{
incrementCacheQueries();
@SuppressWarnings("unchecked")
AccordCachingState<K, V> node = (AccordCachingState<K, V>) cache.get(key);
if (node == null)
{
return null;
}
return safeRefFactory.apply(acquireExisting(node, false));
}
public S acquire(K key)
{
AccordCachingState<K, V> node = acquire(key, false);
@ -290,12 +347,11 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
incrementCacheMisses();
if (onlyIfLoaded)
return null;
AccordCachingState<K, V> node = new AccordCachingState<>(key);
AccordCachingState<K, V> node = nodeFactory.create(key, index);
node.load(loadExecutor, loadFunction);
node.references++;
if (cache.put(key, node) == null)
itemsCached++;
cache.put(key, node);
maybeUpdateSize(node, heapEstimator);
metrics.objectSize.update(node.lastQueriedEstimatedSizeOnHeap);
maybeEvictSomeNodes();
@ -429,6 +485,27 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
node.complete();
}
@VisibleForTesting
boolean keyIsReferenced(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null && node.references > 0;
}
@VisibleForTesting
boolean keyIsCached(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null && node.status() != EVICTED;
}
@VisibleForTesting
int references(Object key, Class<? extends AccordSafeState<?, ?>> valClass)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null ? node.references : 0;
}
private void incrementCacheQueries()
{
instanceMetrics.requests.mark();
@ -474,7 +551,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
@Override
public int size()
{
return itemsCached;
return cache.size();
}
@Override
@ -487,13 +564,12 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
@VisibleForTesting
void unsafeClear()
{
cache.clear();
bytesCached = 0;
metrics.reset();;
instances.values().forEach(i -> {
i.itemsCached = 0;
i.bytesCached = 0;
i.instanceMetrics.reset();
instances.forEach(instance -> {
instance.cache.clear();
instance.bytesCached = 0;
instance.instanceMetrics.reset();
});
//noinspection StatementWithEmptyBody
while (null != poll());
@ -524,10 +600,18 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
AsyncChains.awaitUninterruptibly(node.saving());
}
private int cacheSize()
{
int size = 0;
for (Instance<?, ?, ?> instance : instances)
size += instance.cache.size();
return size;
}
@VisibleForTesting
int numReferencedEntries()
{
return cache.size() - unreferenced;
return cacheSize() - unreferenced;
}
@VisibleForTesting
@ -539,7 +623,7 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
@Override
public int size()
{
return cache.size();
return cacheSize();
}
@Override
@ -547,25 +631,4 @@ public class AccordStateCache extends IntrusiveLinkedList<AccordCachingState<?,?
{
return bytesCached;
}
@VisibleForTesting
boolean keyIsReferenced(Object key)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null && node.references > 0;
}
@VisibleForTesting
boolean keyIsCached(Object key)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null && node.status() != EVICTED;
}
@VisibleForTesting
int references(Object key)
{
AccordCachingState<?, ?> node = cache.get(key);
return node != null ? node.references : 0;
}
}

101
src/java/org/apache/cassandra/service/accord/CommandsForKeyUpdate.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.accord;
import java.nio.ByteBuffer;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import accord.impl.CommandTimeseries;
import accord.impl.CommandsForKey;
import accord.impl.CommandsForKeyGroupUpdater;
import accord.impl.CommandsForKeyUpdater;
import accord.primitives.RoutableKey;
import accord.primitives.Timestamp;
import org.apache.cassandra.db.marshal.ByteBufferAccessor;
import org.apache.cassandra.service.accord.api.PartitionKey;
import static org.apache.cassandra.utils.ObjectSizes.measure;
public class CommandsForKeyUpdate extends CommandsForKeyGroupUpdater.Immutable<ByteBuffer> implements CommandsForKey.Update
{
private static final CommandsForKeyUpdate EMPTY = new CommandsForKeyUpdate(null, null, null, null);
static long EMPTY_SIZE = measure(EMPTY);
private static long EMPTY_TIMESERIES_UPDATE_SIZE = measure(new CommandTimeseries.ImmutableUpdate(ImmutableMap.of(), ImmutableSet.of()));
private static <T extends Timestamp> long immutableTimeseriesUpdate(CommandTimeseries.ImmutableUpdate<T, ByteBuffer> update)
{
long size = EMPTY_TIMESERIES_UPDATE_SIZE;
for (Map.Entry<T, ByteBuffer> write : update.writes.entrySet())
{
size += AccordObjectSizes.timestamp(write.getKey());
size += ByteBufferAccessor.instance.size(write.getValue());
}
for (T delete : update.deletes)
size += AccordObjectSizes.timestamp(delete);
return size;
}
private static long EMPTY_UPDATER_SIZE = measure(new CommandsForKeyUpdater.Immutable<>(null));
private static long updaterSize(CommandsForKeyUpdater.Immutable<ByteBuffer> updater)
{
long size = EMPTY_UPDATER_SIZE;
size += immutableTimeseriesUpdate(updater.commands());
return size;
}
private final PartitionKey key;
public CommandsForKeyUpdate(PartitionKey key, CommandsForKeyUpdater.Immutable<ByteBuffer> deps, CommandsForKeyUpdater.Immutable<ByteBuffer> all, CommandsForKeyUpdater.Immutable<ByteBuffer> common)
{
super(deps, all, common);
this.key = key;
}
public static CommandsForKeyUpdate empty(RoutableKey key)
{
return new CommandsForKeyUpdate((PartitionKey) key,
CommandsForKeyUpdater.Immutable.empty(),
CommandsForKeyUpdater.Immutable.empty(),
CommandsForKeyUpdater.Immutable.empty());
}
public PartitionKey key()
{
return key;
}
public long estimatedSizeOnHeap()
{
long size = EMPTY_SIZE;
size += AccordObjectSizes.key(key.asKey());
size += updaterSize(deps());
size += updaterSize(all());
size += updaterSize(common());
return size;
}
}

35
src/java/org/apache/cassandra/service/accord/CommandsForRanges.java
View File

@ -41,7 +41,8 @@ import com.google.common.collect.ImmutableSortedMap;
import accord.api.Key;
import accord.api.RoutingKey;
import accord.impl.CommandTimeseries;
import accord.impl.CommandTimeseriesHolder;
import accord.impl.DomainCommands;
import accord.impl.DomainTimestamps;
import accord.local.Command;
import accord.local.PreLoadContext;
import accord.local.SafeCommand;
@ -66,6 +67,8 @@ import org.apache.cassandra.utils.IntervalTree;
public class CommandsForRanges
{
public interface DomainInfo extends DomainCommands, DomainTimestamps {}
public enum TxnType
{
UNKNOWN, LOCAL, REMOTE;
@ -402,19 +405,19 @@ public class CommandsForRanges
return localCommands.contains(txnId);
}
public Iterable<CommandTimeseriesHolder> search(AbstractKeys<Key> keys)
public Iterable<DomainInfo> search(AbstractKeys<Key> keys)
{
// group by the keyspace, as ranges are based off TokenKey, which is scoped to a range
Map<String, List<Key>> groupByKeyspace = new TreeMap<>();
for (Key key : keys)
groupByKeyspace.computeIfAbsent(((PartitionKey) key).keyspace(), ignore -> new ArrayList<>()).add(key);
return () -> new AbstractIterator<CommandTimeseriesHolder>()
return () -> new AbstractIterator<DomainInfo>()
{
Iterator<String> ksIt = groupByKeyspace.keySet().iterator();
Iterator<Map.Entry<Range, Set<RangeCommandSummary>>> rangeIt;
@Override
protected CommandTimeseriesHolder computeNext()
protected DomainInfo computeNext()
{
while (true)
{
@ -456,14 +459,14 @@ public class CommandsForRanges
}
@Nullable
public CommandTimeseriesHolder search(Range range)
public DomainInfo search(Range range)
{
List<RangeCommandSummary> matches = rangesToCommands.search(Interval.create(normalize(range.start(), range.startInclusive(), true),
normalize(range.end(), range.endInclusive(), false)));
return result(range, matches);
}
private CommandTimeseriesHolder result(Seekable seekable, Collection<RangeCommandSummary> matches)
private DomainInfo result(Seekable seekable, Collection<RangeCommandSummary> matches)
{
if (matches.isEmpty())
return null;
@ -506,7 +509,7 @@ public class CommandsForRanges
}
}
private static class Holder implements CommandTimeseriesHolder
private static class Holder implements DomainInfo
{
private final Seekable keyOrRange;
private final Collection<RangeCommandSummary> matches;
@ -518,31 +521,25 @@ public class CommandsForRanges
}
@Override
public CommandTimeseries<?> byId()
public CommandTimeseries<?> commands()
{
return build(m -> m.txnId);
}
@Override
public CommandTimeseries<?> byExecuteAt()
{
return build(m -> m.executeAt != null ? m.executeAt : m.txnId);
return build();
}
@Override
public Timestamp max()
{
return byExecuteAt().maxTimestamp();
return commands().maxTimestamp();
}
private CommandTimeseries<?> build(Function<RangeCommandSummary, Timestamp> fn)
private CommandTimeseries<?> build()
{
CommandTimeseries.Update<RangeCommandSummary> builder = new CommandTimeseries.Update<>(keyOrRange, RangeCommandSummaryLoader.INSTANCE);
CommandTimeseries.Builder<RangeCommandSummary> builder = new CommandTimeseries.Builder<>(keyOrRange, RangeCommandSummaryLoader.INSTANCE);
for (RangeCommandSummary m : matches)
{
if (m.status == SaveStatus.Invalidated)
continue;
builder.add(fn.apply(m), m);
builder.add(m.txnId, m);
}
return builder.build();
}

74
src/java/org/apache/cassandra/service/accord/async/AsyncLoader.java
View File

@ -34,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import accord.api.RoutingKey;
import accord.local.KeyHistory;
import accord.local.PreLoadContext;
import accord.primitives.Range;
import accord.primitives.Ranges;
@ -71,14 +72,16 @@ public class AsyncLoader
private final Iterable<TxnId> txnIds;
private final Seekables<?, ?> keysOrRanges;
private final KeyHistory keyHistory;
protected AsyncResult<?> readResult;
public AsyncLoader(AccordCommandStore commandStore, Iterable<TxnId> txnIds, Seekables<?, ?> keysOrRanges)
public AsyncLoader(AccordCommandStore commandStore, Iterable<TxnId> txnIds, Seekables<?, ?> keysOrRanges, KeyHistory keyHistory)
{
this.commandStore = commandStore;
this.txnIds = txnIds;
this.keysOrRanges = keysOrRanges;
this.keyHistory = keyHistory;
}
protected static Iterable<TxnId> txnIds(PreLoadContext context)
@ -90,34 +93,51 @@ public class AsyncLoader
return Iterables.concat(Collections.singleton(primaryid), additionalIds);
}
private static <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReadsForKey(K key,
Map<K, S> context,
AccordStateCache.Instance<K, V, S> cache,
List<AsyncChain<?>> listenChains)
{
S safeRef = cache.acquire(key);
context.put(key, safeRef);
AccordCachingState.Status status = safeRef.globalStatus(); // globalStatus() completes
switch (status)
{
default: throw new IllegalStateException("Unhandled global state: " + status);
case LOADING:
listenChains.add(safeRef.loading());
break;
case SAVING:
// make sure we work with a completed state that supports get() and set()
listenChains.add(safeRef.saving());
break;
case LOADED:
case MODIFIED:
case FAILED_TO_SAVE:
break;
case FAILED_TO_LOAD:
throw new RuntimeException(safeRef.failure());
}
}
private void referenceAndAssembleReadsForKey(RoutableKey key,
AsyncOperation.Context context,
List<AsyncChain<?>> listenChains)
{
referenceAndAssembleReadsForKey(key, context.timestampsForKey, commandStore.timestampsForKeyCache(), listenChains);
if (keyHistory == KeyHistory.DEPS)
referenceAndAssembleReadsForKey(key, context.depsCommandsForKeys, commandStore.depsCommandsForKeyCache(), listenChains);
if (keyHistory == KeyHistory.ALL)
referenceAndAssembleReadsForKey(key, context.allCommandsForKeys, commandStore.allCommandsForKeyCache(), listenChains);
referenceAndAssembleReadsForKey(key, context.updatesForKeys, commandStore.updatesForKeyCache(), listenChains);
}
private <K, V, S extends AccordSafeState<K, V>> void referenceAndAssembleReads(Iterable<? extends K> keys,
Map<K, S> context,
AccordStateCache.Instance<K, V, S> cache,
List<AsyncChain<?>> listenChains)
{
for (K key : keys)
{
S safeRef = cache.acquire(key);
context.put(key, safeRef);
AccordCachingState.Status status = safeRef.globalStatus(); // globalStatus() completes
switch (status)
{
default: throw new IllegalStateException("Unhandled global state: " + status);
case LOADING:
listenChains.add(safeRef.loading());
break;
case SAVING:
// make sure we work with a completed state that supports get() and set()
listenChains.add(safeRef.saving());
break;
case LOADED:
case MODIFIED:
case FAILED_TO_SAVE:
break;
case FAILED_TO_LOAD:
throw new RuntimeException(safeRef.failure());
}
}
keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, cache, listenChains));
}
private AsyncResult<?> referenceAndDispatchReads(AsyncOperation.Context context)
@ -131,7 +151,7 @@ public class AsyncLoader
case Key:
// cast to Keys fails...
Iterable<RoutableKey> keys = (Iterable<RoutableKey>) keysOrRanges;
referenceAndAssembleReads(keys, context.commandsForKeys, commandStore.commandsForKeyCache(), chains);
keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
break;
case Range:
chains.add(referenceAndDispatchReadsForRange(context));
@ -151,7 +171,7 @@ public class AsyncLoader
if (keys.isEmpty())
return AsyncChains.success(null);
List<AsyncChain<?>> chains = new ArrayList<>();
referenceAndAssembleReads(keys, context.commandsForKeys, commandStore.commandsForKeyCache(), chains);
keys.forEach(key -> referenceAndAssembleReadsForKey(key, context, chains));
return chains.isEmpty() ? AsyncChains.success(null) : AsyncChains.reduce(chains, (a, b) -> null);
}, commandStore);
}
@ -168,7 +188,7 @@ public class AsyncLoader
private AsyncChain<Set<PartitionKey>> findOverlappingKeys(Range range)
{
Set<PartitionKey> cached = commandStore.commandsForKeyCache().stream()
Set<PartitionKey> cached = commandStore.depsCommandsForKeyCache().stream()
.map(n -> (PartitionKey) n.key())
.filter(range::contains)
.collect(Collectors.toSet());

23
src/java/org/apache/cassandra/service/accord/async/AsyncOperation.java
View File

@ -42,7 +42,9 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordSafeCommand;
import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.AccordSafeCommandsForKeyUpdate;
import org.apache.cassandra.service.accord.AccordSafeState;
import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
import static org.apache.cassandra.service.accord.async.AsyncLoader.txnIds;
import static org.apache.cassandra.service.accord.async.AsyncOperation.State.INITIALIZED;
@ -66,18 +68,27 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
static class Context
{
final HashMap<TxnId, AccordSafeCommand> commands = new HashMap<>();
final TreeMap<RoutableKey, AccordSafeCommandsForKey> commandsForKeys = new TreeMap<>();
final TreeMap<RoutableKey, AccordSafeTimestampsForKey> timestampsForKey = new TreeMap<>();
final TreeMap<RoutableKey, AccordSafeCommandsForKey> depsCommandsForKeys = new TreeMap<>();
final TreeMap<RoutableKey, AccordSafeCommandsForKey> allCommandsForKeys = new TreeMap<>();
final TreeMap<RoutableKey, AccordSafeCommandsForKeyUpdate> updatesForKeys = new TreeMap<>();
void releaseResources(AccordCommandStore commandStore)
{
commands.values().forEach(commandStore.commandCache()::release);
commandsForKeys.values().forEach(commandStore.commandsForKeyCache()::release);
timestampsForKey.values().forEach(commandStore.timestampsForKeyCache()::release);
depsCommandsForKeys.values().forEach(commandStore.depsCommandsForKeyCache()::release);
allCommandsForKeys.values().forEach(commandStore.allCommandsForKeyCache()::release);
updatesForKeys.values().forEach(commandStore.updatesForKeyCache()::release);
}
void revertChanges()
{
commands.values().forEach(AccordSafeState::revert);
commandsForKeys.values().forEach(AccordSafeState::revert);
timestampsForKey.values().forEach(AccordSafeState::revert);
depsCommandsForKeys.values().forEach(AccordSafeState::revert);
allCommandsForKeys.values().forEach(AccordSafeState::revert);
updatesForKeys.values().forEach(AccordSafeState::revert);
}
}
@ -136,7 +147,7 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
{
return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys());
return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys(), preLoadContext.keyHistory());
}
private void onLoaded(Object o, Throwable throwable)
@ -249,11 +260,11 @@ public abstract class AsyncOperation<R> extends AsyncChains.Head<R> implements R
return;
state(PREPARING);
case PREPARING:
safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.commandsForKeys);
safeStore = commandStore.beginOperation(preLoadContext, context.commands, context.timestampsForKey, context.depsCommandsForKeys, context.allCommandsForKeys, context.updatesForKeys);
state(RUNNING);
case RUNNING:
result = apply(safeStore);
safeStore.postExecute(context.commands, context.commandsForKeys);
safeStore.postExecute(context.commands, context.timestampsForKey, context.depsCommandsForKeys, context.allCommandsForKeys, context.updatesForKeys);
context.releaseResources(commandStore);
commandStore.completeOperation(safeStore);
commandStore.executionOrder().unregister(this);

27
src/java/org/apache/cassandra/service/accord/serializers/ApplySerializers.java
View File

@ -29,6 +29,7 @@ import accord.primitives.Seekables;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Writes;
import accord.utils.Invariants;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -37,12 +38,32 @@ import org.apache.cassandra.io.util.DataOutputPlus;
public class ApplySerializers
{
private static final IVersionedSerializer<Apply.Kind> kind = new IVersionedSerializer<Apply.Kind>()
{
public void serialize(Apply.Kind kind, DataOutputPlus out, int version) throws IOException
{
Invariants.checkArgument(kind == Apply.Kind.Maximal || kind == Apply.Kind.Minimal);
out.writeBoolean(kind == Apply.Kind.Maximal);
}
public Apply.Kind deserialize(DataInputPlus in, int version) throws IOException
{
return in.readBoolean() ? Apply.Kind.Maximal : Apply.Kind.Minimal;
}
public long serializedSize(Apply.Kind t, int version)
{
return TypeSizes.BOOL_SIZE;
}
};
// public static final IVersionedSerializer<Apply> request = new TxnRequestSerializer<Apply>()
public abstract static class ApplySerializer<A extends Apply> extends TxnRequestSerializer<A>
{
@Override
public void serializeBody(A apply, DataOutputPlus out, int version) throws IOException
{
out.writeBoolean(apply.kind == Apply.Kind.Maximal);
kind.serialize(apply.kind, out, version);
KeySerializers.seekables.serialize(apply.keys(), out, version);
CommandSerializers.timestamp.serialize(apply.executeAt, out, version);
DepsSerializer.partialDeps.serialize(apply.deps, out, version);
@ -57,7 +78,7 @@ public class ApplySerializers
public A deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch) throws IOException
{
return deserializeApply(txnId, scope, waitForEpoch,
in.readBoolean() ? Apply.Kind.Maximal : Apply.Kind.Minimal,
kind.deserialize(in, version),
KeySerializers.seekables.deserialize(in, version),
CommandSerializers.timestamp.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version),
@ -69,7 +90,7 @@ public class ApplySerializers
@Override
public long serializedBodySize(A apply, int version)
{
return TypeSizes.BOOL_SIZE
return kind.serializedSize(apply.kind, version)
+ KeySerializers.seekables.serializedSize(apply.keys(), version)
+ CommandSerializers.timestamp.serializedSize(apply.executeAt, version)
+ DepsSerializer.partialDeps.serializedSize(apply.deps, version)

1
src/java/org/apache/cassandra/service/accord/serializers/CheckStatusSerializers.java
View File

@ -224,6 +224,7 @@ public class CheckStatusSerializers
return createOk(map, maxKnowledgeStatus, maxStatus, promised, accepted, executeAt,
isCoordinating, durability, route, homeKey, partialTxn, committedDeps, writes, result);
}
}

27
src/java/org/apache/cassandra/service/accord/serializers/CommitSerializers.java
View File

@ -30,6 +30,7 @@ import accord.primitives.PartialTxn;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
import accord.primitives.Unseekables;
import accord.utils.Invariants;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -42,6 +43,26 @@ import static org.apache.cassandra.utils.NullableSerializer.serializedNullableSi
public class CommitSerializers
{
private static final IVersionedSerializer<Commit.Kind> kind = new IVersionedSerializer<Commit.Kind>()
{
public void serialize(Commit.Kind kind, DataOutputPlus out, int version) throws IOException
{
Invariants.checkArgument(kind == Commit.Kind.Minimal || kind == Commit.Kind.Maximal);
out.writeBoolean(kind == Commit.Kind.Maximal);
}
public Commit.Kind deserialize(DataInputPlus in, int version) throws IOException
{
return in.readBoolean() ? Commit.Kind.Maximal : Commit.Kind.Minimal;
}
public long serializedSize(Commit.Kind kind, int version)
{
return TypeSizes.BOOL_SIZE;
}
};
public abstract static class CommitSerializer<C extends Commit, R extends ReadData> extends TxnRequestSerializer<C>
{
private final IVersionedSerializer<ReadData> read;
@ -54,7 +75,7 @@ public class CommitSerializers
@Override
public void serializeBody(C msg, DataOutputPlus out, int version) throws IOException
{
out.writeBoolean(msg.kind == Commit.Kind.Maximal);
kind.serialize(msg.kind, out, version);
CommandSerializers.timestamp.serialize(msg.executeAt, out, version);
CommandSerializers.nullablePartialTxn.serialize(msg.partialTxn, out, version);
DepsSerializer.partialDeps.serialize(msg.partialDeps, out, version);
@ -70,7 +91,7 @@ public class CommitSerializers
public C deserializeBody(DataInputPlus in, int version, TxnId txnId, PartialRoute<?> scope, long waitForEpoch) throws IOException
{
return deserializeCommit(txnId, scope, waitForEpoch,
in.readBoolean() ? Commit.Kind.Maximal : Commit.Kind.Minimal,
kind.deserialize(in, version),
CommandSerializers.timestamp.deserialize(in, version),
CommandSerializers.nullablePartialTxn.deserialize(in, version),
DepsSerializer.partialDeps.deserialize(in, version),
@ -82,7 +103,7 @@ public class CommitSerializers
@Override
public long serializedBodySize(C msg, int version)
{
return TypeSizes.BOOL_SIZE
return kind.serializedSize(msg.kind, version)
+ CommandSerializers.timestamp.serializedSize(msg.executeAt, version)
+ CommandSerializers.nullablePartialTxn.serializedSize(msg.partialTxn, version)
+ DepsSerializer.partialDeps.serializedSize(msg.partialDeps, version)

25
src/java/org/apache/cassandra/service/accord/serializers/FetchSerializers.java
View File

@ -166,7 +166,7 @@ public class FetchSerializers
{
TxnId txnId = CommandSerializers.txnId.deserialize(in, version);
Route<?> route = KeySerializers.route.deserialize(in, version);
SaveStatus saveStatus = CommandSerializers.saveStatus.deserialize(in, version);
SaveStatus maxKnowledgeSaveStatus = CommandSerializers.saveStatus.deserialize(in, version);
SaveStatus maxSaveStatus = CommandSerializers.saveStatus.deserialize(in, version);
Durability durability = CommandSerializers.durability.deserialize(in, version);
RoutingKey homeKey = KeySerializers.nullableRoutingKey.deserialize(in, version);
@ -175,13 +175,13 @@ public class FetchSerializers
CheckStatus.FoundKnownMap known = CheckStatusSerializers.foundKnownMap.deserialize(in, version);
boolean isTruncated = in.readBoolean();
PartialTxn partialTxn = CommandSerializers.nullablePartialTxn.deserialize(in, version);
PartialDeps partialDeps = DepsSerializer.nullablePartialDeps.deserialize(in, version);
PartialDeps committedDeps = DepsSerializer.nullablePartialDeps.deserialize(in, version);
long toEpoch = in.readLong();
Timestamp executeAt = CommandSerializers.nullableTimestamp.deserialize(in, version);
Timestamp committedExecuteAt = CommandSerializers.nullableTimestamp.deserialize(in, version);
Writes writes = CommandSerializers.nullableWrites.deserialize(in, version);
Result result = null;
switch (saveStatus)
switch (maxSaveStatus)
{
case PreApplied:
case Applying:
@ -193,7 +193,22 @@ public class FetchSerializers
break;
}
return Propagate.SerializerSupport.create(txnId, route, saveStatus, maxSaveStatus, durability, homeKey, progressKey, achieved, known, isTruncated, partialTxn, partialDeps, toEpoch, executeAt, writes, result);
return Propagate.SerializerSupport.create(txnId,
route,
maxKnowledgeSaveStatus,
maxSaveStatus,
durability,
homeKey,
progressKey,
achieved,
known,
isTruncated,
partialTxn,
committedDeps,
toEpoch,
committedExecuteAt,
writes,
result);
}
@Override

13
src/java/org/apache/cassandra/service/accord/txn/TxnWrite.java
View File

@ -36,6 +36,9 @@ import org.slf4j.LoggerFactory;
import accord.api.DataStore;
import accord.api.Write;
import accord.impl.AbstractSafeCommandStore;
import accord.impl.CommandsForKeys;
import accord.impl.TimestampsForKey;
import accord.local.SafeCommandStore;
import accord.primitives.PartialTxn;
import accord.primitives.RoutableKey;
@ -60,8 +63,7 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.service.accord.AccordSafeCommandStore;
import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.utils.BooleanSerializer;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -373,11 +375,10 @@ public class TxnWrite extends AbstractKeySorted<TxnWrite.Update> implements Writ
// TODO (expected, efficiency): 99.9999% of the time we can just use executeAt.hlc(), so can avoid bringing
// cfk into memory by retaining at all times in memory key ranges that are dirty and must use this logic;
// any that aren't can just use executeAt.hlc
AccordSafeCommandsForKey cfk = ((AccordSafeCommandStore) safeStore).commandsForKey((RoutableKey) key);
cfk.updateLastExecutionTimestamps(safeStore, executeAt, true);
long timestamp = cfk.timestampMicrosFor(executeAt, true);
TimestampsForKey cfk = CommandsForKeys.updateLastExecutionTimestamps((AbstractSafeCommandStore<?,?,?,?>) safeStore, (RoutableKey) key, executeAt, true);
long timestamp = AccordSafeTimestampsForKey.timestampMicrosFor(cfk, executeAt, true);
// TODO (low priority - do we need to compute nowInSeconds, or can we just use executeAt?)
int nowInSeconds = cfk.nowInSecondsFor(executeAt, true);
int nowInSeconds = AccordSafeTimestampsForKey.nowInSecondsFor(cfk, executeAt, true);
List<AsyncChain<Void>> results = new ArrayList<>();

46
test/unit/org/apache/cassandra/db/compaction/CompactionAccordIteratorsTest.java
View File

@ -85,13 +85,11 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandRows;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsColumns;
import org.apache.cassandra.service.accord.AccordKeyspace.CommandsForKeyRows;
import org.apache.cassandra.service.accord.AccordTestUtils;
import org.apache.cassandra.service.accord.IAccordService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import static accord.impl.CommandsForKey.NO_LAST_EXECUTED_HLC;
import static accord.local.PreLoadContext.contextFor;
import static accord.utils.async.AsyncChains.getUninterruptibly;
import static org.apache.cassandra.Util.spinAssertEquals;
@ -101,10 +99,10 @@ import static org.apache.cassandra.db.compaction.CompactionAccordIteratorsTest.D
import static org.apache.cassandra.db.compaction.CompactionAccordIteratorsTest.DurableBeforeType.UNIVERSAL;
import static org.apache.cassandra.schema.SchemaConstants.ACCORD_KEYSPACE_NAME;
import static org.apache.cassandra.service.accord.AccordKeyspace.COMMANDS;
import static org.apache.cassandra.service.accord.AccordKeyspace.COMMANDS_FOR_KEY;
import static org.apache.cassandra.service.accord.AccordKeyspace.DEPS_COMMANDS_FOR_KEY;
import static org.apache.cassandra.service.accord.AccordKeyspace.DepsCommandsForKeysAccessor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@ -129,7 +127,7 @@ public class CompactionAccordIteratorsTest
private static final TxnId GT_SECOND_TXN_ID = AccordTestUtils.txnId(EPOCH, SECOND_TXN_ID.hlc() + 1, NODE);
static ColumnFamilyStore commands;
static ColumnFamilyStore commandsForKey;
static ColumnFamilyStore depsCommandsForKey;
static TableMetadata table;
static FullRoute<?> route;
Random random;
@ -150,8 +148,8 @@ public class CompactionAccordIteratorsTest
StorageService.instance.initServer();
commands = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, COMMANDS);
commands.disableAutoCompaction();
commandsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, COMMANDS_FOR_KEY);
commandsForKey.disableAutoCompaction();
depsCommandsForKey = ColumnFamilyStore.getIfExists(SchemaConstants.ACCORD_KEYSPACE_NAME, DEPS_COMMANDS_FOR_KEY);
depsCommandsForKey.disableAutoCompaction();
table = ColumnFamilyStore.getIfExists("ks", "tbl").metadata();
route = AccordTestUtils.keys(table, 42).toRoute(AccordTestUtils.key(table, 42).toUnseekable());
}
@ -211,6 +209,12 @@ public class CompactionAccordIteratorsTest
}, false);
}
@Test
public void testAccordTimestampsForKeyPurger()
{
throw new AssertionError("TODO -> see commented out parts of CFK tests");
}
@Test
public void testAccordCommandsForKeyPurgerSingleCompaction() throws Throwable
{
@ -240,16 +244,16 @@ public class CompactionAccordIteratorsTest
Partition partition = partitions.get(0);
Row staticRow = partition.getRow(Clustering.STATIC_CLUSTERING);
assertEquals(4, Iterables.size(staticRow));
assertEquals(SECOND_TXN_ID, CommandsForKeyRows.getMaxTimestamp(staticRow));
assertEquals(TXN_ID, CommandsForKeyRows.getLastExecutedTimestamp(staticRow));
assertEquals(TXN_ID, CommandsForKeyRows.getLastWriteTimestamp(staticRow));
assertEquals(TXN_ID.hlc(), CommandsForKeyRows.getLastExecutedMicros(staticRow));
// assertEquals(SECOND_TXN_ID, CommandsForKeyRows.getMaxTimestamp(staticRow));
// assertEquals(TXN_ID, CommandsForKeyRows.getLastExecutedTimestamp(staticRow));
// assertEquals(TXN_ID, CommandsForKeyRows.getLastWriteTimestamp(staticRow));
// assertEquals(TXN_ID.hlc(), CommandsForKeyRows.getLastExecutedMicros(staticRow));
assertEquals(4, Iterators.size(partition.unfilteredIterator()));
UnfilteredRowIterator rows = partition.unfilteredIterator();
// One row per txn per series
for (int i = 0; i < 2; i++)
for (TxnId txnId : TXN_IDS)
assertEquals(txnId, CommandsForKeyRows.getTimestamp((Row)rows.next()));
assertEquals(txnId, DepsCommandsForKeysAccessor.getTimestamp((Row)rows.next()));
};
}
@ -261,14 +265,14 @@ public class CompactionAccordIteratorsTest
Row staticRow = partition.getRow(Clustering.STATIC_CLUSTERING);
// Only expect one column to remain because the second transaction is a read
assertEquals(1, Iterables.size(staticRow));
assertEquals(SECOND_TXN_ID, CommandsForKeyRows.getMaxTimestamp(staticRow));
assertNull(CommandsForKeyRows.getLastExecutedTimestamp(staticRow));
assertNull(CommandsForKeyRows.getLastWriteTimestamp(staticRow));
assertEquals(NO_LAST_EXECUTED_HLC, CommandsForKeyRows.getLastExecutedMicros(staticRow));
// assertEquals(SECOND_TXN_ID, CommandsForKeyRows.getMaxTimestamp(staticRow));
// assertNull(CommandsForKeyRows.getLastExecutedTimestamp(staticRow));
// assertNull(CommandsForKeyRows.getLastWriteTimestamp(staticRow));
// assertEquals(NO_LAST_EXECUTED_HLC, CommandsForKeyRows.getLastExecutedMicros(staticRow));
assertEquals(2, Iterators.size(partition.unfilteredIterator()));
UnfilteredRowIterator rows = partition.unfilteredIterator();
assertEquals(TXN_IDS[1], CommandsForKeyRows.getTimestamp((Row)rows.next()));
assertEquals(TXN_IDS[1], CommandsForKeyRows.getTimestamp((Row)rows.next()));
assertEquals(TXN_IDS[1], DepsCommandsForKeysAccessor.getTimestamp((Row)rows.next()));
assertEquals(TXN_IDS[1], DepsCommandsForKeysAccessor.getTimestamp((Row)rows.next()));
};
}
@ -281,7 +285,7 @@ public class CompactionAccordIteratorsTest
{
testWithCommandStore((commandStore) -> {
IAccordService mockAccordService = mockAccordService(commandStore, redundantBefore, DurableBefore.EMPTY);
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, COMMANDS_FOR_KEY);
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(ACCORD_KEYSPACE_NAME, DEPS_COMMANDS_FOR_KEY);
List<Partition> result = compactCFS(mockAccordService, cfs);
expectedResult.accept(result);
}, true);
@ -405,7 +409,7 @@ public class CompactionAccordIteratorsTest
commandStore.cache().awaitSaveResults();
});
commands.forceBlockingFlush(FlushReason.UNIT_TESTS);
commandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
depsCommandsForKey.forceBlockingFlush(FlushReason.UNIT_TESTS);
}
private void testWithCommandStore(TestWithCommandStore test, boolean additionalCommand) throws Throwable
@ -464,7 +468,7 @@ public class CompactionAccordIteratorsTest
Iterator<UntypedResultSet.Row> commandsTableIterator = commandsTable.iterator();
for (TxnId txnId : txnIds)
assertEquals(txnId, AccordKeyspace.deserializeTimestampOrNull(commandsTableIterator.next().getBytes("txn_id"), TxnId::fromBits));
UntypedResultSet commandsForKeyTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + COMMANDS_FOR_KEY + ";");
UntypedResultSet commandsForKeyTable = QueryProcessor.executeInternal("SELECT * FROM " + ACCORD_KEYSPACE_NAME + "." + DEPS_COMMANDS_FOR_KEY + ";");
logger.info(commandsForKeyTable.toStringUnsafe());
assertEquals(txnIds.length * 2, commandsForKeyTable.size());
Iterator<UntypedResultSet.Row> commandsForKeyTableIterator = commandsTable.iterator();

7
test/unit/org/apache/cassandra/service/accord/AccordCachingStateTest.java
View File

@ -29,9 +29,14 @@ public class AccordCachingStateTest
{
static class CachingState extends AccordCachingState<String, String>
{
public CachingState(String key, int index)
{
super(key, index);
}
public CachingState(String key)
{
super(key);
this(key, 0);
}
}

202
test/unit/org/apache/cassandra/service/accord/AccordCommandStoreTest.java
View File

@ -18,9 +18,12 @@
package org.apache.cassandra.service.accord;
import java.nio.ByteBuffer;
import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.collect.Iterables;
import com.google.common.collect.ImmutableSortedMap;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@ -30,16 +33,23 @@ import org.slf4j.LoggerFactory;
import accord.api.Key;
import accord.api.Result;
import accord.impl.CommandTimeseries;
import accord.impl.CommandsForKey;
import accord.impl.CommandsForKeys;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.CommonAttributes;
import accord.local.KeyHistory;
import accord.local.PreLoadContext;
import accord.local.SaveStatus;
import accord.messages.Apply;
import accord.primitives.Ballot;
import accord.primitives.Keys;
import accord.primitives.PartialDeps;
import accord.primitives.PartialTxn;
import accord.primitives.Ranges;
import accord.primitives.Route;
import accord.primitives.RoutableKey;
import accord.primitives.RoutingKeys;
import accord.primitives.Timestamp;
import accord.primitives.TxnId;
@ -53,12 +63,14 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordCachingState.Modified;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.serializers.CommandSerializers;
import org.apache.cassandra.service.accord.serializers.CommandsForKeySerializer;
import org.apache.cassandra.utils.Pair;
import static accord.local.Status.Durability.Majority;
import static com.google.common.collect.Iterables.getOnlyElement;
import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted;
import static org.apache.cassandra.service.accord.AccordTestUtils.ballot;
@ -115,6 +127,7 @@ public class AccordCommandStoreTest
attrs.partialTxn(txn);
attrs.route(route);
attrs.durability(Majority);
attrs.partialTxn(txn);
Ballot promised = ballot(1, clock.incrementAndGet(), 1);
Ballot accepted = ballot(1, clock.incrementAndGet(), 1);
Timestamp executeAt = timestamp(1, clock.incrementAndGet(), 1);
@ -154,42 +167,193 @@ public class AccordCommandStoreTest
}
@Test
public void commandsForKeyLoadSave()
public void timestampsForKeyLoadSave()
{
AtomicLong clock = new AtomicLong(0);
AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
Timestamp maxTimestamp = timestamp(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(1);
PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1);
TxnId txnId2 = txnId(1, clock.incrementAndGet(), 1);
Command command1 = preaccepted(txnId1, txn, timestamp(1, clock.incrementAndGet(), 1));
Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1));
AccordSafeTimestampsForKey tfk = new AccordSafeTimestampsForKey(loaded(key, null));
tfk.initialize();
tfk.updateMax(maxTimestamp);
CommandsForKeys.updateLastExecutionTimestamps(commandStore, tfk, txnId1, true);
Assert.assertEquals(txnId1.hlc(), AccordSafeTimestampsForKey.timestampMicrosFor(tfk.current(), txnId1, true));
CommandsForKeys.updateLastExecutionTimestamps(commandStore, tfk, txnId2, true);
Assert.assertEquals(txnId2.hlc(), AccordSafeTimestampsForKey.timestampMicrosFor(tfk.current(), txnId2, true));
Assert.assertEquals(txnId2, tfk.current().lastExecutedTimestamp());
Assert.assertEquals(txnId2.hlc(), tfk.lastExecutedMicros());
AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null));
cfk.initialize(CommandsForKeySerializer.loader);
cfk.updateMax(maxTimestamp);
cfk.updateLastExecutionTimestamps(null, txnId1, true);
Assert.assertEquals(txnId1.hlc(), cfk.timestampMicrosFor(txnId1, true));
AccordSafeCommandsForKeyUpdate ufk = new AccordSafeCommandsForKeyUpdate(loaded(key, null));
ufk.initialize();
cfk.updateLastExecutionTimestamps(null, txnId2, true);
Assert.assertEquals(txnId2.hlc(), cfk.timestampMicrosFor(txnId2, true));
CommandsForKeys.registerCommand(tfk, ufk, command1);
CommandsForKeys.registerCommand(tfk, ufk, command2);
Assert.assertEquals(txnId2, cfk.current().lastExecutedTimestamp());
Assert.assertEquals(txnId2.hlc(), cfk.lastExecutedMicros());
cfk.register(command1);
cfk.register(command2);
AccordKeyspace.getCommandsForKeyMutation(commandStore, cfk, commandStore.nextSystemTimestampMicros()).apply();
logger.info("E: {}", cfk);
CommandsForKey actual = AccordKeyspace.loadCommandsForKey(commandStore, key);
AccordKeyspace.getTimestampsForKeyMutation(commandStore, tfk, commandStore.nextSystemTimestampMicros()).apply();
logger.info("E: {}", tfk);
TimestampsForKey actual = AccordKeyspace.loadTimestampsForKey(commandStore, key);
logger.info("A: {}", actual);
Assert.assertEquals(cfk.current(), actual);
Assert.assertEquals(tfk.current(), actual);
}
@Test
public void commandsForKeyLoadSave()
{
AtomicLong clock = new AtomicLong(0);
AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
PartialTxn txn = createPartialTxn(1);
PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
TxnId txnId1 = txnId(1, clock.incrementAndGet(), 1);
TxnId txnId2 = txnId(1, clock.incrementAndGet(), 1);
Command command1 = preaccepted(txnId1, txn, timestamp(1, clock.incrementAndGet(), 1));
Command command2 = preaccepted(txnId2, txn, timestamp(1, clock.incrementAndGet(), 1));
AccordSafeTimestampsForKey tfk = new AccordSafeTimestampsForKey(loaded(key, null));
tfk.initialize();
AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null));
cfk.initialize(CommandsForKeySerializer.loader);
AccordSafeCommandsForKeyUpdate ufk = new AccordSafeCommandsForKeyUpdate(loaded(key, null));
ufk.initialize();
CommandsForKeys.registerCommand(tfk, ufk, command1);
CommandsForKeys.registerCommand(tfk, ufk, command2);
AccordKeyspace.getCommandsForKeyMutation(commandStore.id(), ufk.setUpdates(), commandStore.nextSystemTimestampMicros()).apply();
logger.info("E: {}", cfk);
CommandsForKey actual = AccordKeyspace.loadDepsCommandsForKey(commandStore, key);
logger.info("A: {}", actual);
Assert.assertEquals(ufk.applyToDeps(cfk.current()), actual);
}
private static <K, V extends AccordSafeState<K, ?>> NavigableMap<K, V> toNavigableMap(V safeState)
{
TreeMap<K, V> map = new TreeMap<>();
map.put(safeState.key(), safeState);
return map;
}
@Test
public void commandsForKeyUpdateTest()
{
// check that updates are reflected in CFKs without marking them modified
AtomicLong clock = new AtomicLong(0);
AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
PartialTxn txn = createPartialTxn(1);
PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
AccordSafeCommand safeCommand = commandStore.commandCache().acquireOrInitialize(txnId, t -> preaccepted(txnId, txn, timestamp(1, clock.incrementAndGet(), 1)));
AccordSafeTimestampsForKey timestamps = commandStore.timestampsForKeyCache().acquireOrInitialize(key, k -> new TimestampsForKey((Key) k));
AccordSafeCommandsForKey commands = commandStore.depsCommandsForKeyCache().acquireOrInitialize(key, k -> new CommandsForKey((Key) k, CommandsForKeySerializer.loader));
AccordSafeCommandsForKeyUpdate update = commandStore.updatesForKeyCache().acquireOrInitialize(key, CommandsForKeyUpdate::empty);
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.commandCache().getUnsafe(txnId).status());
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.timestampsForKeyCache().getUnsafe(key).status());
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.depsCommandsForKeyCache().getUnsafe(key).status());
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.updatesForKeyCache().getUnsafe(key).status());
AccordSafeCommandStore safeStore = commandStore.beginOperation(PreLoadContext.contextFor(txnId, Keys.of(key), KeyHistory.DEPS),
toNavigableMap(safeCommand),
toNavigableMap(timestamps),
toNavigableMap(commands),
new TreeMap<>(),
toNavigableMap(update));
AccordSafeCommandsForKeyUpdate updates = safeStore.getOrCreateCommandsForKeyUpdate(key);
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.updatesForKeyCache().getUnsafe(key).status());
Command initialCommand = safeCommand.current();
CommandsForKey initialCFK = commands.current();
CommandsForKeyUpdate initialUpdate = updates.current();
updates.common().commands().add(txnId, initialCommand);
CommandsForKeyUpdate expected = new CommandsForKeyUpdate(key, updates.deps().toImmutable(), updates.all().toImmutable(), updates.common().toImmutable());
Assert.assertEquals(1, expected.common().commands().numChanges());
Assert.assertTrue(expected.deps().isEmpty());
Assert.assertTrue(expected.all().isEmpty());
Assert.assertSame(initialCFK, commands.current());
Assert.assertSame(initialUpdate, updates.current());
safeStore.postExecute(toNavigableMap(safeCommand),
toNavigableMap(timestamps),
toNavigableMap(commands),
new TreeMap<>(),
toNavigableMap(updates));
safeStore.complete();
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.commandCache().getUnsafe(txnId).status());
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.timestampsForKeyCache().getUnsafe(key).status());
Assert.assertEquals(AccordCachingState.Status.LOADED, commandStore.depsCommandsForKeyCache().getUnsafe(key).status());
Assert.assertEquals(AccordCachingState.Status.MODIFIED, commandStore.updatesForKeyCache().getUnsafe(key).status());
CommandsForKey finalCFK = commandStore.depsCommandsForKeyCache().getUnsafe(key).get();
Assert.assertEquals(txnId, getOnlyElement(finalCFK.commands().commands.keySet()));
Modified<RoutableKey, CommandsForKeyUpdate> loadedUpdate = (Modified<RoutableKey, CommandsForKeyUpdate>) commandStore.updatesForKeyCache().getUnsafe(key).state();
Assert.assertNull(loadedUpdate.original);
Assert.assertEquals(expected, loadedUpdate.get());
}
/**
* Test that in memory cfk updates are applied to
*/
@Test
public void commandsForKeyUpdateOnLoadTest()
{
AtomicLong clock = new AtomicLong(0);
AccordCommandStore commandStore = createAccordCommandStore(clock::incrementAndGet, "ks", "tbl");
PartialTxn txn = createPartialTxn(1);
PartitionKey key = (PartitionKey) getOnlyElement(txn.keys());
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
Command command = preaccepted(txnId, txn, timestamp(1, clock.incrementAndGet(), 1));
// make a cached update
AccordSafeCommandsForKeyUpdate updates = commandStore.updatesForKeyCache().acquireOrInitialize(key, k -> null);
updates.preExecute();
updates.common().commands().remove(command.txnId());
updates.setUpdates(); // apply the updates applied to the safe state to the cached value
updates.postExecute(); // apply the cached value to the global state
commandStore.updatesForKeyCache().release(updates);
// make an out of date CFK
CommandTimeseries.CommandLoader<ByteBuffer> loader = CommandsForKeySerializer.loader;
CommandsForKey staleCFK = CommandsForKey.SerializerSupport.create(key, loader,
ImmutableSortedMap.of(command.txnId(), loader.saveForCFK(command)));
Assert.assertEquals(txnId, getOnlyElement(staleCFK.commands().commands.keySet()));
// on loading the cfk into the cache, the in memory update should be applied
AccordSafeCommandsForKey commands = commandStore.depsCommandsForKeyCache().acquireOrInitialize(key, k -> new CommandsForKey((Key) k, CommandsForKeySerializer.loader));
commands.preExecute();
Assert.assertEquals(txnId, getOnlyElement(staleCFK.commands().commands.keySet()));
}
}

27
test/unit/org/apache/cassandra/service/accord/AccordCommandTest.java
View File

@ -27,7 +27,9 @@ import org.junit.Test;
import accord.api.Key;
import accord.api.RoutingKey;
import accord.impl.CommandsForKey;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.KeyHistory;
import accord.local.Node;
import accord.local.PreLoadContext;
import accord.local.Status;
@ -116,10 +118,11 @@ public class AccordCommandTest
Assert.assertEquals(Status.PreAccepted, command.status());
Assert.assertTrue(command.partialDeps() == null || command.partialDeps().isEmpty());
CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
Assert.assertEquals(txnId, cfk.max());
Assert.assertNotNull((cfk.byId()).get(txnId));
Assert.assertNotNull((cfk.byExecuteAt()).get(txnId));
TimestampsForKey tfk = ((AccordSafeCommandStore) instance).timestampsForKey(key(1)).current();
Assert.assertEquals(txnId, tfk.max());
CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
Assert.assertNotNull((cfk.commands()).get(txnId));
}));
// check accept
@ -146,10 +149,11 @@ public class AccordCommandTest
Assert.assertEquals(Status.Accepted, command.status());
Assert.assertEquals(deps, command.partialDeps());
CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
Assert.assertEquals(executeAt, cfk.max());
Assert.assertNotNull((cfk.byId()).get(txnId));
Assert.assertNotNull((cfk.byExecuteAt()).get(txnId));
TimestampsForKey tfk = ((AccordSafeCommandStore) instance).timestampsForKey(key(1)).current();
Assert.assertEquals(executeAt, tfk.max());
CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
Assert.assertNotNull((cfk.commands()).get(txnId));
}));
// check commit
@ -157,15 +161,14 @@ public class AccordCommandTest
commandStore.appendToJournal(commit);
getUninterruptibly(commandStore.execute(commit, commit::apply));
getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key)), instance -> {
getUninterruptibly(commandStore.execute(PreLoadContext.contextFor(txnId, Keys.of(key), KeyHistory.DEPS), instance -> {
Command command = instance.ifInitialised(txnId).current();
Assert.assertEquals(commit.executeAt, command.executeAt());
Assert.assertTrue(command.hasBeen(Status.Committed));
Assert.assertEquals(commit.partialDeps, command.partialDeps());
CommandsForKey cfk = ((AccordSafeCommandStore) instance).commandsForKey(key(1)).current();
Assert.assertNotNull((cfk.byId()).get(txnId));
Assert.assertNotNull((cfk.byExecuteAt()).get(commit.executeAt));
CommandsForKey cfk = ((AccordSafeCommandStore) instance).depsCommandsForKey(key(1)).current();
Assert.assertNotNull((cfk.commands()).get(txnId));
}));
}

34
test/unit/org/apache/cassandra/service/accord/AccordStateCacheTest.java
View File

@ -188,7 +188,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString safeString1 = instance.acquire("1");
@ -220,9 +220,9 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> stringInstance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true,String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true,String::length);
AccordStateCache.Instance<Integer, Integer, SafeInt> intInstance =
cache.instance(Integer.class, Integer.class, SafeInt::new, key -> key, (original, current) -> null, (k, v) -> true,ignored -> Integer.BYTES);
cache.instance(Integer.class, SafeInt.class, SafeInt::new, key -> key, (original, current) -> null, (k, v) -> true,ignored -> Integer.BYTES);
assertCacheState(cache, 0, 0, 0);
SafeString safeString1 = stringInstance.acquire("1");
@ -260,7 +260,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, DEFAULT_NODE_SIZE * 5, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString[] items = new SafeString[3];
@ -300,7 +300,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 5, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString[] items = new SafeString[5];
@ -326,8 +326,8 @@ public class AccordStateCacheTest
assertCacheState(cache, 1, 5, nodeSize(1) * 4 + nodeSize(0));
Assert.assertSame(items[1].global, cache.head());
Assert.assertSame(items[4].global, cache.tail());
Assert.assertFalse(cache.keyIsCached("0"));
Assert.assertFalse(cache.keyIsReferenced("0"));
Assert.assertFalse(instance.keyIsCached("0", SafeString.class));
Assert.assertFalse(instance.keyIsReferenced("0", SafeString.class));
assertCacheMetrics(cache.metrics, 0, 6, 6);
assertCacheMetrics(instance.instanceMetrics, 0, 6, 6);
@ -346,7 +346,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 4, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString[] items = new SafeString[5];
@ -385,7 +385,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, DEFAULT_NODE_SIZE * 4, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString safeString1 = instance.acquire("0");
@ -394,12 +394,12 @@ public class AccordStateCacheTest
assertCacheMetrics(cache.metrics, 0, 1, 1);
assertCacheMetrics(instance.instanceMetrics, 0, 1, 1);
Assert.assertEquals(1, cache.references("0"));
Assert.assertEquals(1, instance.references("0", SafeString.class));
assertCacheState(cache, 1, 1, nodeSize(0));
SafeString safeString2 = instance.acquire("0");
Assert.assertEquals(Status.LOADED, safeString1.globalStatus());
Assert.assertEquals(2, cache.references("0"));
Assert.assertEquals(2, instance.references("0", SafeString.class));
assertCacheState(cache, 1, 1, nodeSize(0));
assertCacheMetrics(cache.metrics, 1, 1, 2);
assertCacheMetrics(instance.instanceMetrics, 1, 1, 2);
@ -416,7 +416,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, nodeSize(1) * 3 + nodeSize(3), cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString item = instance.acquire(Integer.toString(0));
@ -443,10 +443,10 @@ public class AccordStateCacheTest
// all should have been evicted except 0
assertCacheState(cache, 0, 1, nodeSize(2));
Assert.assertTrue(cache.keyIsCached("0"));
Assert.assertFalse(cache.keyIsCached("1"));
Assert.assertFalse(cache.keyIsCached("2"));
Assert.assertFalse(cache.keyIsCached("3"));
Assert.assertTrue(instance.keyIsCached("0", SafeString.class));
Assert.assertFalse(instance.keyIsCached("1", SafeString.class));
Assert.assertFalse(instance.keyIsCached("2", SafeString.class));
Assert.assertFalse(instance.keyIsCached("3", SafeString.class));
}
@Test
@ -455,7 +455,7 @@ public class AccordStateCacheTest
ManualExecutor executor = new ManualExecutor();
AccordStateCache cache = new AccordStateCache(executor, executor, 500, cacheMetrics);
AccordStateCache.Instance<String, String, SafeString> instance =
cache.instance(String.class, String.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
cache.instance(String.class, SafeString.class, SafeString::new, key -> key, (original, current) -> null, (k, v) -> true, String::length);
assertCacheState(cache, 0, 0, 0);
SafeString safeString = instance.acquire("1");

9
test/unit/org/apache/cassandra/service/accord/AccordTestUtils.java
View File

@ -139,9 +139,9 @@ public class AccordTestUtils
return new CommandsForKey(key, CommandsForKeySerializer.loader);
}
public static <K, V> AccordCachingState<K, V> loaded(K key, V value)
public static <K, V> AccordCachingState<K, V> loaded(K key, V value, int index)
{
AccordCachingState<K, V> global = new AccordCachingState<>(key);
AccordCachingState<K, V> global = new AccordCachingState<>(key, index);
global.load(ImmediateExecutor.INSTANCE, k -> {
Assert.assertEquals(key, k);
return value;
@ -150,6 +150,11 @@ public class AccordTestUtils
return global;
}
public static <K, V> AccordCachingState<K, V> loaded(K key, V value)
{
return loaded(key, value, 0);
}
public static AccordSafeCommand safeCommand(Command command)
{
AccordCachingState<TxnId, Command> global = loaded(command.txnId(), command);

185
test/unit/org/apache/cassandra/service/accord/async/AsyncLoaderTest.java
View File

@ -28,8 +28,10 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import accord.impl.CommandsForKey;
import accord.api.Key;
import accord.impl.TimestampsForKey;
import accord.local.Command;
import accord.local.KeyHistory;
import accord.primitives.Keys;
import accord.primitives.PartialTxn;
import accord.primitives.RoutableKey;
@ -45,16 +47,19 @@ import org.apache.cassandra.service.accord.AccordCommandStore;
import org.apache.cassandra.service.accord.AccordKeyspace;
import org.apache.cassandra.service.accord.AccordCachingState;
import org.apache.cassandra.service.accord.AccordSafeCommand;
import org.apache.cassandra.service.accord.AccordSafeCommandsForKey;
import org.apache.cassandra.service.accord.AccordSafeCommandsForKeyUpdate;
import org.apache.cassandra.service.accord.AccordSafeTimestampsForKey;
import org.apache.cassandra.service.accord.AccordStateCache;
import org.apache.cassandra.service.accord.CommandsForKeyUpdate;
import org.apache.cassandra.service.accord.api.PartitionKey;
import org.apache.cassandra.service.accord.async.AsyncOperation.Context;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import static java.util.Collections.emptyList;
import static java.util.Collections.singleton;
import static org.apache.cassandra.cql3.statements.schema.CreateTableStatement.parse;
import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.notDefined;
import static org.apache.cassandra.service.accord.AccordTestUtils.commandsForKey;
import static org.apache.cassandra.service.accord.AccordTestUtils.Commands.preaccepted;
import static org.apache.cassandra.service.accord.AccordTestUtils.createAccordCommandStore;
import static org.apache.cassandra.service.accord.AccordTestUtils.createPartialTxn;
import static org.apache.cassandra.service.accord.AccordTestUtils.execute;
@ -86,7 +91,7 @@ public class AsyncLoaderTest
AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
commandStore.executeBlocking(() -> commandStore.setCapacity(1024));
AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> cfkCache = commandStore.commandsForKeyCache();
AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(0);
PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
@ -98,24 +103,24 @@ public class AsyncLoaderTest
testLoad(executor, safeCommand, notDefined(txnId, txn));
commandCache.release(safeCommand);
cfkCache.unsafeSetLoadFunction(k -> commandsForKey((PartitionKey) k));
AccordSafeCommandsForKey safeCfk = cfkCache.acquire(key);
testLoad(executor, safeCfk, commandsForKey(key));
cfkCache.release(safeCfk);
timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((PartitionKey) k));
AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquire(key);
testLoad(executor, safeTimestamps, new TimestampsForKey(key));
timestampsCache.release(safeTimestamps);
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key));
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.NONE);
// everything is cached, so the loader should return immediately
commandStore.executeBlocking(() -> {
Context context = new Context();
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertEquals(safeCommand.global(), context.commands.get(txnId).global());
Assert.assertEquals(safeCfk.global(), context.commandsForKeys.get(key).global());
Assert.assertEquals(safeTimestamps.global(), context.timestampsForKey.get(key).global());
Assert.assertTrue(result);
});
Assert.assertSame(safeCommand.global(), commandCache.getUnsafe(txnId));
Assert.assertSame(safeCfk.global(), cfkCache.getUnsafe(key));
Assert.assertSame(safeTimestamps.global(), timestampsCache.getUnsafe(key));
}
/**
@ -136,20 +141,21 @@ public class AsyncLoaderTest
safeCommand.set(notDefined(txnId, txn));
AccordKeyspace.getCommandMutation(commandStore, safeCommand, commandStore.nextSystemTimestampMicros()).apply();
AccordSafeCommandsForKey cfk = new AccordSafeCommandsForKey(loaded(key, null));
safeCommand.preExecute();
cfk.set(commandsForKey(key));
AccordKeyspace.getCommandsForKeyMutation(commandStore, cfk, commandStore.nextSystemTimestampMicros()).apply();
AccordSafeTimestampsForKey timestamps = new AccordSafeTimestampsForKey(loaded(key, null));
timestamps.preExecute();
timestamps.initialize();
AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), null, timestamps.current(), commandStore.nextSystemTimestampMicros()).apply();
// resources are on disk only, so the loader should suspend...
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key));
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.DEPS);
AsyncPromise<Void> cbFired = new AsyncPromise<>();
Context context = new Context();
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> {
Assert.assertNull(t);
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
cbFired.setSuccess(null);
});
Assert.assertFalse(result);
@ -161,7 +167,7 @@ public class AsyncLoaderTest
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
Assert.assertTrue(result);
});
}
@ -187,20 +193,17 @@ public class AsyncLoaderTest
testLoad(executor, safeCommand, notDefined(txnId, txn));
commandCache.release(safeCommand);
AccordSafeCommandsForKey safeCfk = new AccordSafeCommandsForKey(loaded(key, null));
safeCfk.set(commandsForKey(key));
AccordKeyspace.getCommandsForKeyMutation(commandStore, safeCfk, commandStore.nextSystemTimestampMicros()).apply();
AccordKeyspace.getTimestampsForKeyMutation(commandStore.id(), null, new TimestampsForKey(key), commandStore.nextSystemTimestampMicros()).apply();
// resources are on disk only, so the loader should suspend...
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key));
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.NONE);
AsyncPromise<Void> cbFired = new AsyncPromise<>();
Context context = new Context();
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> {
Assert.assertNull(t);
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
cbFired.setSuccess(null);
});
Assert.assertFalse(result);
@ -214,7 +217,7 @@ public class AsyncLoaderTest
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
Assert.assertTrue(result);
});
}
@ -231,16 +234,16 @@ public class AsyncLoaderTest
createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
commandStore.executor().submit(() -> commandStore.setCapacity(1024)).get();
AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
AccordStateCache.Instance<RoutableKey, CommandsForKey, AccordSafeCommandsForKey> cfkCache = commandStore.commandsForKeyCache();
AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(0);
PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
// acquire / release
cfkCache.unsafeSetLoadFunction(k -> commandsForKey((PartitionKey) k));
AccordSafeCommandsForKey safeCfk = cfkCache.acquire(key);
testLoad(executor, safeCfk, commandsForKey(key));
cfkCache.release(safeCfk);
timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((PartitionKey) k));
AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquire(key);
testLoad(executor, safeTimestamps, new TimestampsForKey(key));
timestampsCache.release(safeTimestamps);
commandCache.unsafeSetLoadFunction(id -> { Assert.assertEquals(txnId, id); return notDefined(id, txn); });
AccordSafeCommand safeCommand = commandCache.acquire(txnId);
@ -248,7 +251,7 @@ public class AsyncLoaderTest
Assert.assertTrue(commandCache.isReferenced(txnId));
Assert.assertFalse(commandCache.isLoaded(txnId));
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key));
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(key), KeyHistory.NONE);
// since there's a read future associated with the txnId, we'll wait for it to load
AsyncPromise<Void> cbFired = new AsyncPromise<>();
@ -257,7 +260,7 @@ public class AsyncLoaderTest
boolean result = loader.load(context, (o, t) -> {
Assert.assertNull(t);
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
cbFired.setSuccess(null);
});
Assert.assertFalse(result);
@ -273,7 +276,7 @@ public class AsyncLoaderTest
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(context.commandsForKeys.containsKey(key));
Assert.assertTrue(context.timestampsForKey.containsKey(key));
Assert.assertTrue(result);
});
}
@ -303,7 +306,7 @@ public class AsyncLoaderTest
throw new AssertionError("Unknown txnId: " + txnId);
});
AsyncLoader loader = new AsyncLoader(commandStore, ImmutableList.of(txnId1, txnId2), Keys.EMPTY);
AsyncLoader loader = new AsyncLoader(commandStore, ImmutableList.of(txnId1, txnId2), Keys.EMPTY, KeyHistory.DEPS);
boolean result = loader.load(new Context(), (u, t) -> {
Assert.assertFalse(callback.isDone());
@ -318,4 +321,118 @@ public class AsyncLoaderTest
promise.tryFailure(failure);
AsyncChains.getUninterruptibly(callback);
}
@Test
public void inProgressCommandSaveTest()
{
AtomicLong clock = new AtomicLong(0);
ManualExecutor executor = new ManualExecutor();
AccordCommandStore commandStore =
createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
AccordStateCache.Instance<TxnId, Command, AccordSafeCommand> commandCache = commandStore.commandCache();
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(0);
// acquire / release
commandCache.unsafeSetLoadFunction(id -> notDefined(id, txn));
commandCache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
AccordSafeCommand safeCommand = commandCache.acquire(txnId);
testLoad(executor, safeCommand, notDefined(txnId, txn));
safeCommand.set(preaccepted(txnId, txn, safeCommand.txnId()));
commandCache.release(safeCommand);
Assert.assertEquals(AccordCachingState.Status.MODIFIED, commandCache.getUnsafe(txnId).status());
commandCache.getUnsafe(txnId).save(executor, (before, after) -> () -> {});
Assert.assertEquals(AccordCachingState.Status.SAVING, commandCache.getUnsafe(txnId).status());
// since the command is still saving, the loader shouldn't be able to acquire a reference
AsyncLoader loader = new AsyncLoader(commandStore, singleton(txnId), Keys.of(), KeyHistory.NONE);
AsyncPromise<Void> cbFired = new AsyncPromise<>();
Context context = new Context();
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> {
Assert.assertNull(t);
Assert.assertTrue(context.commands.containsKey(txnId));
cbFired.setSuccess(null);
});
Assert.assertFalse(result);
});
Assert.assertEquals(AccordCachingState.Status.SAVING, commandCache.getUnsafe(txnId).status());
executor.runOne();
cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
Assert.assertEquals(AccordCachingState.Status.LOADED, commandCache.getUnsafe(txnId).status());
// then return immediately after the callback has fired
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertTrue(context.commands.containsKey(txnId));
Assert.assertTrue(result);
});
}
@Test
public void inProgressCFKSaveTest()
{
AtomicLong clock = new AtomicLong(0);
ManualExecutor executor = new ManualExecutor();
AccordCommandStore commandStore =
createAccordCommandStore(clock::incrementAndGet, "ks", "tbl", executor, executor);
AccordStateCache.Instance<RoutableKey, TimestampsForKey, AccordSafeTimestampsForKey> timestampsCache = commandStore.timestampsForKeyCache();
timestampsCache.unsafeSetLoadFunction(k -> new TimestampsForKey((PartitionKey) k));
timestampsCache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
AccordStateCache.Instance<RoutableKey, CommandsForKeyUpdate, AccordSafeCommandsForKeyUpdate> updateCache = commandStore.updatesForKeyCache();
updateCache.unsafeSetLoadFunction(k -> { throw new AssertionError("updates shouldn't be loaded"); });
updateCache.unsafeSetSaveFunction((before, after) -> () -> { throw new AssertionError("nodes expected to be saved manually"); });
TxnId txnId = txnId(1, clock.incrementAndGet(), 1);
PartialTxn txn = createPartialTxn(0);
PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
Command preaccepted = preaccepted(txnId, txn, txnId);
// acquire / release
AccordSafeTimestampsForKey safeTimestamps = timestampsCache.acquireOrInitialize(key, k -> new TimestampsForKey((Key) k));
timestampsCache.release(safeTimestamps);
Assert.assertEquals(AccordCachingState.Status.LOADED, timestampsCache.getUnsafe(key).status());
AccordSafeCommandsForKeyUpdate safeUpdate = updateCache.acquireOrInitialize(key, CommandsForKeyUpdate::empty);
safeUpdate.common().commands().add(txnId, preaccepted);
safeUpdate.setUpdates();
updateCache.release(safeUpdate);
Assert.assertEquals(AccordCachingState.Status.MODIFIED, updateCache.getUnsafe(key).status());
updateCache.getUnsafe(key).save(executor, (before, after) -> () -> {});
Assert.assertEquals(AccordCachingState.Status.SAVING, updateCache.getUnsafe(key).status());
// since the command is still saving, the loader shouldn't be able to acquire a reference
AsyncLoader loader = new AsyncLoader(commandStore, emptyList(), Keys.of(key), KeyHistory.NONE);
AsyncPromise<Void> cbFired = new AsyncPromise<>();
Context context = new Context();
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> {
Assert.assertNull(t);
Assert.assertTrue(context.timestampsForKey.containsKey(key));
cbFired.setSuccess(null);
});
Assert.assertFalse(result);
});
Assert.assertEquals(AccordCachingState.Status.SAVING, updateCache.getUnsafe(key).status());
executor.runOne();
cbFired.awaitUninterruptibly(1, TimeUnit.SECONDS);
Assert.assertEquals(AccordCachingState.Status.LOADED, updateCache.getUnsafe(key).status());
// then return immediately after the callback has fired
commandStore.executeBlocking(() -> {
boolean result = loader.load(context, (o, t) -> Assert.fail());
Assert.assertTrue(context.timestampsForKey.containsKey(key));
Assert.assertTrue(result);
});
}
}

17
test/unit/org/apache/cassandra/service/accord/async/AsyncOperationTest.java
View File

@ -68,6 +68,7 @@ import org.apache.cassandra.db.SinglePartitionReadCommand;
import org.apache.cassandra.db.transform.FilteredPartitions;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.AccordCachingState;
import org.apache.cassandra.service.accord.AccordCommandStore;
@ -111,8 +112,10 @@ public class AsyncOperationTest
@Before
public void before()
{
QueryProcessor.executeInternal("TRUNCATE system_accord.commands");
QueryProcessor.executeInternal("TRUNCATE system_accord.commands_for_key");
QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.COMMANDS));
QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.TIMESTAMPS_FOR_KEY));
QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.DEPS_COMMANDS_FOR_KEY));
QueryProcessor.executeInternal(String.format("TRUNCATE %s.%s", SchemaConstants.ACCORD_KEYSPACE_NAME, AccordKeyspace.ALL_COMMANDS_FOR_KEY));
}
/**
@ -146,12 +149,12 @@ public class AsyncOperationTest
PartitionKey key = (PartitionKey) Iterables.getOnlyElement(txn.keys());
getUninterruptibly(commandStore.execute(contextFor(key), instance -> {
SafeCommandsForKey cfk = ((AccordSafeCommandStore) instance).maybeCommandsForKey(key);
SafeCommandsForKey cfk = ((AccordSafeCommandStore) instance).maybeDepsCommandsForKey(key);
Assert.assertNull(cfk);
}));
long nowInSeconds = FBUtilities.nowInSeconds();
SinglePartitionReadCommand command = AccordKeyspace.getCommandsForKeyRead(commandStore.id(), key, nowInSeconds);
SinglePartitionReadCommand command = AccordKeyspace.getDepsCommandsForKeyRead(commandStore.id(), key, (int) nowInSeconds);
try(ReadExecutionController controller = command.executionController();
FilteredPartitions partitions = FilteredPartitions.filter(command.executeLocally(controller), nowInSeconds))
{
@ -277,7 +280,7 @@ public class AsyncOperationTest
@Override
AsyncLoader createAsyncLoader(AccordCommandStore commandStore, PreLoadContext preLoadContext)
{
return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys())
return new AsyncLoader(commandStore, txnIds(preLoadContext), preLoadContext.keys(), preLoadContext.keyHistory())
{
@Override
void state(State state)
@ -422,7 +425,7 @@ public class AsyncOperationTest
try
{
//TODO this is due to bad typing for Instance, it doesn't use ? extends RoutableKey
assertNoReferences(commandStore.commandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
assertNoReferences(commandStore.depsCommandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
}
catch (AssertionError e)
{
@ -464,7 +467,7 @@ public class AsyncOperationTest
{
awaitDone(commandStore.commandCache(), ids);
//TODO this is due to bad typing for Instance, it doesn't use ? extends RoutableKey
awaitDone(commandStore.commandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
awaitDone(commandStore.depsCommandsForKeyCache(), (Iterable<RoutableKey>) (Iterable<?>) keys);
}
private static <T> void awaitDone(AccordStateCache.Instance<T, ?, ?> cache, Iterable<T> keys)