natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Serialize batchlog mutations with the version of the target node 

patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for
CASSANDRA-6931
This commit is contained in:
Aleksey Yeschenko 2014-03-31 13:09:36 +03:00
parent f6daf4e307
commit b7ac8f96c1
5 changed files with 50 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS.txt
View File

@ -33,7 +33,7 @@ New features
Upgrading
---------
- Rolling upgrades from anything pre-2.0.6 is not supported.
- Rolling upgrade from anything pre-2.0.7 is not supported.
- For leveled compaction users, 2.0 must be atleast started before
upgrading to 2.1 due to the fact that the old JSON leveled
manifest is migrated into the sstable metadata files on startup

28
src/java/org/apache/cassandra/db/BatchlogManager.java
View File

@ -44,9 +44,7 @@ import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.composites.CellName;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.LongType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.WriteTimeoutException;
@ -121,26 +119,23 @@ public class BatchlogManager implements BatchlogManagerMBean
batchlogTasks.execute(runnable);
}
public static Mutation getBatchlogMutationFor(Collection<Mutation> mutations, UUID uuid)
public static Mutation getBatchlogMutationFor(Collection<Mutation> mutations, UUID uuid, int version)
{
return getBatchlogMutationFor(mutations, uuid, FBUtilities.timestampMicros());
return getBatchlogMutationFor(mutations, uuid, version, FBUtilities.timestampMicros());
}
@VisibleForTesting
static Mutation getBatchlogMutationFor(Collection<Mutation> mutations, UUID uuid, long now)
static Mutation getBatchlogMutationFor(Collection<Mutation> mutations, UUID uuid, int version, long now)
{
ByteBuffer writtenAt = LongType.instance.decompose(now / 1000);
ByteBuffer data = serializeMutations(mutations);
ColumnFamily cf = ArrayBackedSortedColumns.factory.create(CFMetaData.BatchlogCf);
cf.addColumn(new Cell(cellName(""), ByteBufferUtil.EMPTY_BYTE_BUFFER, now));
cf.addColumn(new Cell(cellName("data"), data, now));
cf.addColumn(new Cell(cellName("written_at"), writtenAt, now));
CFRowAdder adder = new CFRowAdder(cf, CFMetaData.BatchlogCf.comparator.builder().build(), now);
adder.add("data", serializeMutations(mutations, version))
.add("written_at", new Date(now / 1000))
.add("version", version);
return new Mutation(Keyspace.SYSTEM_KS, UUIDType.instance.decompose(uuid), cf);
}
private static ByteBuffer serializeMutations(Collection<Mutation> mutations)
private static ByteBuffer serializeMutations(Collection<Mutation> mutations, int version)
{
DataOutputBuffer buf = new DataOutputBuffer();
@ -148,7 +143,7 @@ public class BatchlogManager implements BatchlogManagerMBean
{
buf.writeInt(mutations.size());
for (Mutation mutation : mutations)
Mutation.serializer.serialize(mutation, buf, MessagingService.VERSION_12);
Mutation.serializer.serialize(mutation, buf, version);
}
catch (IOException e)
{
@ -331,11 +326,6 @@ public class BatchlogManager implements BatchlogManagerMBean
return (int) ((HintedHandOffManager.calculateHintTTL(mutation) * 1000 - (System.currentTimeMillis() - writtenAt)) / 1000);
}
private static CellName cellName(String name)
{
return CFMetaData.BatchlogCf.comparator.makeCellName(name);
}
// force flush + compaction to reclaim space from the replayed batches
private void cleanup() throws ExecutionException, InterruptedException
{

6
src/java/org/apache/cassandra/net/MessagingService.java
View File

@ -70,9 +70,9 @@ public final class MessagingService implements MessagingServiceMBean
public static final String MBEAN_NAME = "org.apache.cassandra.net:type=MessagingService";
// 8 bits version, so don't waste versions
public static final int VERSION_12 = 6;
public static final int VERSION_20 = 7;
public static final int VERSION_21 = 8;
public static final int VERSION_12 = 6;
public static final int VERSION_20 = 7;
public static final int VERSION_21 = 8;
public static final int current_version = VERSION_21;
/**

51
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -607,40 +607,53 @@ public class StorageProxy implements StorageProxyMBean
Keyspace.open(Keyspace.SYSTEM_KS),
null,
WriteType.BATCH_LOG);
updateBatchlog(BatchlogManager.getBatchlogMutationFor(mutations, uuid), endpoints, handler);
MessageOut<Mutation> message = BatchlogManager.getBatchlogMutationFor(mutations, uuid, MessagingService.current_version)
.createMessage();
for (InetAddress target : endpoints)
{
int targetVersion = MessagingService.instance().getVersion(target);
if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS)
{
insertLocal(message.payload, handler);
}
else if (targetVersion == MessagingService.current_version)
{
MessagingService.instance().sendRR(message, target, handler);
}
else
{
MessagingService.instance().sendRR(BatchlogManager.getBatchlogMutationFor(mutations, uuid, targetVersion)
.createMessage(),
target,
handler);
}
}
handler.get();
}
private static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid)
{
ColumnFamily cf = ArrayBackedSortedColumns.factory.create(Schema.instance.getCFMetaData(Keyspace.SYSTEM_KS, SystemKeyspace.BATCHLOG_CF));
cf.delete(new DeletionInfo(FBUtilities.timestampMicros(), (int) (System.currentTimeMillis() / 1000)));
AbstractWriteResponseHandler handler = new WriteResponseHandler(endpoints,
Collections.<InetAddress>emptyList(),
ConsistencyLevel.ANY,
Keyspace.open(Keyspace.SYSTEM_KS),
null,
WriteType.SIMPLE);
updateBatchlog(new Mutation(Keyspace.SYSTEM_KS, UUIDType.instance.decompose(uuid), cf), endpoints, handler);
}
private static void updateBatchlog(Mutation mutation, Collection<InetAddress> endpoints, AbstractWriteResponseHandler handler)
{
if (endpoints.contains(FBUtilities.getBroadcastAddress()))
Mutation mutation = new Mutation(Keyspace.SYSTEM_KS, UUIDType.instance.decompose(uuid));
mutation.delete(SystemKeyspace.BATCHLOG_CF, FBUtilities.timestampMicros());
MessageOut<Mutation> message = mutation.createMessage();
for (InetAddress target : endpoints)
{
assert endpoints.size() == 1;
insertLocal(mutation, handler);
}
else
{
MessageOut<Mutation> message = mutation.createMessage();
for (InetAddress target : endpoints)
if (target.equals(FBUtilities.getBroadcastAddress()) && OPTIMIZE_LOCAL_REQUESTS)
insertLocal(message.payload, handler);
else
MessagingService.instance().sendRR(message, target, handler);
}
}
private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers,
String localDataCenter)
private static void syncWriteBatchedMutations(List<WriteResponseHandlerWrapper> wrappers, String localDataCenter)
throws WriteTimeoutException, OverloadedException
{
for (WriteResponseHandlerWrapper wrapper : wrappers)
@ -650,9 +663,7 @@ public class StorageProxy implements StorageProxyMBean
}
for (WriteResponseHandlerWrapper wrapper : wrappers)
{
wrapper.handler.get();
}
}
/**

7
test/unit/org/apache/cassandra/db/BatchlogManagerTest.java
View File

@ -30,6 +30,7 @@ import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.composites.CellNameType;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.UUIDGen;
@ -66,7 +67,11 @@ public class BatchlogManagerTest extends SchemaLoader
long timestamp = System.currentTimeMillis();
if (i < 500)
timestamp -= DatabaseDescriptor.getWriteRpcTimeout() * 2;
BatchlogManager.getBatchlogMutationFor(Collections.singleton(mutation), UUIDGen.getTimeUUID(), timestamp * 1000).apply();
BatchlogManager.getBatchlogMutationFor(Collections.singleton(mutation),
UUIDGen.getTimeUUID(),
MessagingService.current_version,
timestamp * 1000)
.apply();
}
assertEquals(1000, BatchlogManager.instance.countAllBatches());