natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove pre-3.0 compatibility code for 4.0 

patch by Sylvain Lebresne; reviewed by Aleksey Yeschenko for CASSANDRA-12716
This commit is contained in:
Sylvain Lebresne 2016-09-27 15:26:15 +02:00
parent 3fabc33506
commit 4a2464192e
418 changed files with 811 additions and 8870 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.0
* Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
* Add column definition kind to dropped columns in schema (CASSANDRA-12705)
* Add (automate) Nodetool Documentation (CASSANDRA-12672)
* Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)

4
NEWS.txt
View File

@ -21,6 +21,10 @@ New features
Upgrading
---------
- Cassandra 4.0 removed support for any pre-3.0 format. This means you cannot upgrade from a 2.x version to 4.0
directly, you have to upgrade to a 3.0.x/3.x version first (and run upgradesstable). In particular, this mean
Cassandra 4.0 cannot load or read pre-3.0 sstables in any way: you will need to upgrade those sstable in 3.0.x/3.x
first.
- Cassandra will no longer allow invalid keyspace replication options, such as invalid datacenter names for
NetworkTopologyStrategy. Operators MUST add new nodes to a datacenter before they can set set ALTER or
CREATE keyspace replication policies using that datacenter. Existing keyspaces will continue to operate,

10
src/java/org/apache/cassandra/auth/CassandraRoleManager.java
View File

@ -375,16 +375,6 @@ public class CassandraRoleManager implements IRoleManager
{
public void run()
{
// If not all nodes are on 2.2, we don't want to initialize the role manager as this will confuse 2.1
// nodes (see CASSANDRA-9761 for details). So we re-schedule the setup for later, hoping that the upgrade
// will be finished by then.
if (!MessagingService.instance().areAllNodesAtLeast22())
{
logger.trace("Not all nodes are upgraded to a version that supports Roles yet, rescheduling setup task");
scheduleSetupTask(setupTask);
return;
}
isClusterReady = true;
try
{

199
src/java/org/apache/cassandra/batchlog/LegacyBatchlogMigrator.java
View File

@ -1,199 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.batchlog;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.exceptions.WriteFailureException;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.AbstractWriteResponseHandler;
import org.apache.cassandra.service.WriteResponseHandler;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
public final class LegacyBatchlogMigrator
{
private static final Logger logger = LoggerFactory.getLogger(LegacyBatchlogMigrator.class);
private LegacyBatchlogMigrator()
{
// static class
}
@SuppressWarnings("deprecation")
public static void migrate()
{
ColumnFamilyStore store = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_BATCHLOG);
// nothing to migrate
if (store.isEmpty())
return;
logger.info("Migrating legacy batchlog to new storage");
int convertedBatches = 0;
String query = String.format("SELECT id, data, written_at, version FROM %s.%s",
SchemaConstants.SYSTEM_KEYSPACE_NAME,
SystemKeyspace.LEGACY_BATCHLOG);
int pageSize = BatchlogManager.calculatePageSize(store);
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize);
for (UntypedResultSet.Row row : rows)
{
if (apply(row, convertedBatches))
convertedBatches++;
}
if (convertedBatches > 0)
Keyspace.openAndGetStore(SystemKeyspace.LegacyBatchlog).truncateBlocking();
}
@SuppressWarnings("deprecation")
public static boolean isLegacyBatchlogMutation(Mutation mutation)
{
return mutation.getKeyspaceName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME)
&& mutation.getPartitionUpdate(SystemKeyspace.LegacyBatchlog.cfId) != null;
}
@SuppressWarnings("deprecation")
public static void handleLegacyMutation(Mutation mutation)
{
PartitionUpdate update = mutation.getPartitionUpdate(SystemKeyspace.LegacyBatchlog.cfId);
logger.trace("Applying legacy batchlog mutation {}", update);
update.forEach(row -> apply(UntypedResultSet.Row.fromInternalRow(update.metadata(), update.partitionKey(), row), -1));
}
private static boolean apply(UntypedResultSet.Row row, long counter)
{
UUID id = row.getUUID("id");
long timestamp = id.version() == 1 ? UUIDGen.unixTimestamp(id) : row.getLong("written_at");
int version = row.has("version") ? row.getInt("version") : MessagingService.VERSION_12;
if (id.version() != 1)
id = UUIDGen.getTimeUUID(timestamp, counter);
logger.trace("Converting mutation at {}", timestamp);
try (DataInputBuffer in = new DataInputBuffer(row.getBytes("data"), false))
{
int numMutations = in.readInt();
List<Mutation> mutations = new ArrayList<>(numMutations);
for (int i = 0; i < numMutations; i++)
mutations.add(Mutation.serializer.deserialize(in, version));
BatchlogManager.store(Batch.createLocal(id, TimeUnit.MILLISECONDS.toMicros(timestamp), mutations));
return true;
}
catch (Throwable t)
{
logger.error("Failed to convert mutation {} at timestamp {}", id, timestamp, t);
return false;
}
}
public static void syncWriteToBatchlog(WriteResponseHandler<?> handler, Batch batch, Collection<InetAddress> endpoints)
throws WriteTimeoutException, WriteFailureException
{
for (InetAddress target : endpoints)
{
logger.trace("Sending legacy batchlog store request {} to {} for {} mutations", batch.id, target, batch.size());
int targetVersion = MessagingService.instance().getVersion(target);
MessagingService.instance().sendRR(getStoreMutation(batch, targetVersion).createMessage(MessagingService.Verb.MUTATION),
target,
handler,
false);
}
}
public static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid, long queryStartNanoTime)
{
AbstractWriteResponseHandler<IMutation> handler = new WriteResponseHandler<>(endpoints,
Collections.<InetAddress>emptyList(),
ConsistencyLevel.ANY,
Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME),
null,
WriteType.SIMPLE,
queryStartNanoTime);
Mutation mutation = getRemoveMutation(uuid);
for (InetAddress target : endpoints)
{
logger.trace("Sending legacy batchlog remove request {} to {}", uuid, target);
MessagingService.instance().sendRR(mutation.createMessage(MessagingService.Verb.MUTATION), target, handler, false);
}
}
static void store(Batch batch, int version)
{
getStoreMutation(batch, version).apply();
}
@SuppressWarnings("deprecation")
static Mutation getStoreMutation(Batch batch, int version)
{
PartitionUpdate.SimpleBuilder builder = PartitionUpdate.simpleBuilder(SystemKeyspace.LegacyBatchlog, batch.id);
builder.row()
.timestamp(batch.creationTime)
.add("written_at", new Date(batch.creationTime / 1000))
.add("data", getSerializedMutations(version, batch.decodedMutations))
.add("version", version);
return builder.buildAsMutation();
}
@SuppressWarnings("deprecation")
private static Mutation getRemoveMutation(UUID uuid)
{
return new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyBatchlog,
UUIDType.instance.decompose(uuid),
FBUtilities.timestampMicros(),
FBUtilities.nowInSeconds()));
}
private static ByteBuffer getSerializedMutations(int version, Collection<Mutation> mutations)
{
try (DataOutputBuffer buf = new DataOutputBuffer())
{
buf.writeInt(mutations.size());
for (Mutation mutation : mutations)
Mutation.serializer.serialize(mutation, buf, version);
return buf.buffer();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}

8
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -94,8 +94,6 @@ public final class CFMetaData
public volatile ClusteringComparator comparator; // bytes, long, timeuuid, utf8, etc. This is built directly from clusteringColumns
public final IPartitioner partitioner; // partitioner the table uses
private final Serializers serializers;
// non-final, for now
public volatile TableParams params = TableParams.DEFAULT;
@ -303,7 +301,6 @@ public final class CFMetaData
rebuild();
this.resource = DataResource.table(ksName, cfName);
this.serializers = new Serializers(this);
}
// This rebuild informations that are intrinsically duplicate of the table definition but
@ -1115,11 +1112,6 @@ public final class CFMetaData
return isView;
}
public Serializers serializers()
{
return serializers;
}
public AbstractType<?> makeLegacyDefaultValidator()
{
return isCounter()

3
src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
View File

@ -556,9 +556,6 @@ public final class StatementRestrictions
VariableSpecifications boundNames,
SecondaryIndexManager indexManager)
{
if (!MessagingService.instance().areAllNodesAtLeast30())
throw new InvalidRequestException("Please upgrade all nodes to at least 3.0 before using custom index expressions");
if (expressions.size() > 1)
throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS);

27
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -747,8 +747,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
descriptor.ksname,
descriptor.cfname,
fileIndexGenerator.incrementAndGet(),
descriptor.formatType,
descriptor.digestComponent);
descriptor.formatType);
}
while (new File(newDescriptor.filenameFor(Component.DATA)).exists());
@ -815,26 +814,24 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
return name;
}
public String getSSTablePath(File directory)
public Descriptor newSSTableDescriptor(File directory)
{
return getSSTablePath(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current());
return newSSTableDescriptor(directory, SSTableFormat.Type.current().info.getLatestVersion(), SSTableFormat.Type.current());
}
public String getSSTablePath(File directory, SSTableFormat.Type format)
public Descriptor newSSTableDescriptor(File directory, SSTableFormat.Type format)
{
return getSSTablePath(directory, format.info.getLatestVersion(), format);
return newSSTableDescriptor(directory, format.info.getLatestVersion(), format);
}
private String getSSTablePath(File directory, Version version, SSTableFormat.Type format)
private Descriptor newSSTableDescriptor(File directory, Version version, SSTableFormat.Type format)
{
Descriptor desc = new Descriptor(version,
directory,
keyspace.getName(),
name,
fileIndexGenerator.incrementAndGet(),
format,
Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType()));
return desc.filenameFor(Component.DATA);
return new Descriptor(version,
directory,
keyspace.getName(),
name,
fileIndexGenerator.incrementAndGet(),
format);
}
/**

40
src/java/org/apache/cassandra/db/Directories.java
View File

@ -260,9 +260,8 @@ public class Directories
if (file.isDirectory())
return false;
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(),
file.getName());
return pair != null && pair.left.ksname.equals(metadata.ksName) && pair.left.cfname.equals(metadata.cfName);
Descriptor desc = SSTable.tryDescriptorFromFilename(file);
return desc != null && desc.ksname.equals(metadata.ksName) && desc.cfname.equals(metadata.cfName);
}
});
@ -308,8 +307,9 @@ public class Directories
{
for (File dir : dataPaths)
{
if (new File(dir, filename).exists())
return Descriptor.fromFilename(dir, filename).left;
File file = new File(dir, filename);
if (file.exists())
return Descriptor.fromFilename(file);
}
return null;
}
@ -755,7 +755,7 @@ public class Directories
return false;
case FINAL:
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file);
if (pair == null)
return false;
@ -769,24 +769,6 @@ public class Directories
previous = new HashSet<>();
components.put(pair.left, previous);
}
else if (pair.right.type == Component.Type.DIGEST)
{
if (pair.right != pair.left.digestComponent)
{
// Need to update the DIGEST component as it might be set to another
// digest type as a guess. This may happen if the first component is
// not the DIGEST (but the Data component for example), so the digest
// type is _guessed_ from the Version.
// Although the Version explicitly defines the digest type, it doesn't
// seem to be true under all circumstances. Generated sstables from a
// post 2.1.8 snapshot produced Digest.sha1 files although Version
// defines Adler32.
// TL;DR this piece of code updates the digest component to be "correct".
components.remove(pair.left);
Descriptor updated = pair.left.withDigestComponent(pair.right);
components.put(updated, previous);
}
}
previous.add(pair.right);
nbFiles++;
return false;
@ -1043,11 +1025,11 @@ public class Directories
public boolean isAcceptable(Path path)
{
File file = path.toFile();
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(path.getParent().toFile(), file.getName());
return pair != null
&& pair.left.ksname.equals(metadata.ksName)
&& pair.left.cfname.equals(metadata.cfName)
&& !toSkip.contains(file);
Descriptor desc = SSTable.tryDescriptorFromFilename(file);
return desc != null
&& desc.ksname.equals(metadata.ksName)
&& desc.cfname.equals(metadata.cfName)
&& !toSkip.contains(file);
}
}
}

488
src/java/org/apache/cassandra/db/LegacyLayout.java
View File

@ -17,33 +17,24 @@
*/
package org.apache.cassandra.db;
import java.io.DataInput;
import java.io.IOException;
import java.io.IOError;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.util.*;
import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.utils.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.PeekingIterator;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
@ -52,18 +43,8 @@ import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
*/
public abstract class LegacyLayout
{
private static final Logger logger = LoggerFactory.getLogger(LegacyLayout.class);
public final static int MAX_CELL_NAME_LENGTH = FBUtilities.MAX_UNSIGNED_SHORT;
public final static int STATIC_PREFIX = 0xFFFF;
public final static int DELETION_MASK = 0x01;
public final static int EXPIRATION_MASK = 0x02;
public final static int COUNTER_MASK = 0x04;
public final static int COUNTER_UPDATE_MASK = 0x08;
private final static int RANGE_TOMBSTONE_MASK = 0x10;
private LegacyLayout() {}
public static AbstractType<?> makeLegacyComparator(CFMetaData metadata)
@ -135,7 +116,7 @@ public abstract class LegacyLayout
return decodeCellName(metadata, cellname, false);
}
public static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname, boolean readAllAsDynamic) throws UnknownColumnException
private static LegacyCellName decodeCellName(CFMetaData metadata, ByteBuffer cellname, boolean readAllAsDynamic) throws UnknownColumnException
{
Clustering clustering = decodeClustering(metadata, cellname);
@ -233,30 +214,6 @@ public abstract class LegacyLayout
return new LegacyBound(cb, metadata.isCompound() && CompositeType.isStaticName(bound), collectionName);
}
public static ByteBuffer encodeBound(CFMetaData metadata, ClusteringBound bound, boolean isStart)
{
if (bound == ClusteringBound.BOTTOM || bound == ClusteringBound.TOP || metadata.comparator.size() == 0)
return ByteBufferUtil.EMPTY_BYTE_BUFFER;
ClusteringPrefix clustering = bound.clustering();
if (!metadata.isCompound())
{
assert clustering.size() == 1;
return clustering.get(0);
}
CompositeType ctype = CompositeType.getInstance(metadata.comparator.subtypes());
CompositeType.Builder builder = ctype.builder();
for (int i = 0; i < clustering.size(); i++)
builder.add(clustering.get(i));
if (isStart)
return bound.isInclusive() ? builder.build() : builder.buildAsEndOfRange();
else
return bound.isInclusive() ? builder.buildAsEndOfRange() : builder.build();
}
public static ByteBuffer encodeCellName(CFMetaData metadata, ClusteringPrefix clustering, ByteBuffer columnName, ByteBuffer collectionElement)
{
boolean isStatic = clustering == Clustering.STATIC_CLUSTERING;
@ -330,213 +287,6 @@ public abstract class LegacyLayout
return Clustering.make(components.subList(0, Math.min(csize, components.size())).toArray(new ByteBuffer[csize]));
}
public static ByteBuffer encodeClustering(CFMetaData metadata, ClusteringPrefix clustering)
{
if (clustering.size() == 0)
return ByteBufferUtil.EMPTY_BYTE_BUFFER;
if (!metadata.isCompound())
{
assert clustering.size() == 1;
return clustering.get(0);
}
ByteBuffer[] values = new ByteBuffer[clustering.size()];
for (int i = 0; i < clustering.size(); i++)
values[i] = clustering.get(i);
return CompositeType.build(values);
}
/**
* The maximum number of cells to include per partition when converting to the old format.
* <p>
* We already apply the limit during the actual query, but for queries that counts cells and not rows (thrift queries
* and distinct queries as far as old nodes are concerned), we may still include a little bit more than requested
* because {@link DataLimits} always include full rows. So if the limit ends in the middle of a queried row, the
* full row will be part of our result. This would confuse old nodes however so we make sure to truncate it to
* what's expected before writting it on the wire.
*
* @param command the read commmand for which to determine the maximum cells per partition. This can be {@code null}
* in which case {@code Integer.MAX_VALUE} is returned.
* @return the maximum number of cells per partition that should be enforced according to the read command if
* post-query limitation are in order (see above). This will be {@code Integer.MAX_VALUE} if no such limits are
* necessary.
*/
private static int maxCellsPerPartition(ReadCommand command)
{
if (command == null)
return Integer.MAX_VALUE;
DataLimits limits = command.limits();
// There is 2 types of DISTINCT queries: those that includes only the partition key, and those that include static columns.
// On old nodes, the latter expects the first row in term of CQL count, which is what we already have and there is no additional
// limit to apply. The former however expect only one cell per partition and rely on it (See CASSANDRA-10762).
if (limits.isDistinct())
return command.columnFilter().fetchedColumns().statics.isEmpty() ? 1 : Integer.MAX_VALUE;
switch (limits.kind())
{
case THRIFT_LIMIT:
case SUPER_COLUMN_COUNTING_LIMIT:
return limits.perPartitionCount();
default:
return Integer.MAX_VALUE;
}
}
// For serializing to old wire format
public static LegacyUnfilteredPartition fromUnfilteredRowIterator(ReadCommand command, UnfilteredRowIterator iterator)
{
// we need to extract the range tombstone so materialize the partition. Since this is
// used for the on-wire format, this is not worst than it used to be.
final ImmutableBTreePartition partition = ImmutableBTreePartition.create(iterator);
DeletionInfo info = partition.deletionInfo();
Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> pair = fromRowIterator(partition.metadata(), partition.iterator(), partition.staticRow());
LegacyLayout.LegacyRangeTombstoneList rtl = pair.left;
// Processing the cell iterator results in the LegacyRangeTombstoneList being populated, so we do this
// before we use the LegacyRangeTombstoneList at all
List<LegacyLayout.LegacyCell> cells = Lists.newArrayList(pair.right);
int maxCellsPerPartition = maxCellsPerPartition(command);
if (cells.size() > maxCellsPerPartition)
cells = cells.subList(0, maxCellsPerPartition);
// The LegacyRangeTombstoneList already has range tombstones for the single-row deletions and complex
// deletions. Go through our normal range tombstones and add then to the LegacyRTL so that the range
// tombstones all get merged and sorted properly.
if (info.hasRanges())
{
Iterator<RangeTombstone> rangeTombstoneIterator = info.rangeIterator(false);
while (rangeTombstoneIterator.hasNext())
{
RangeTombstone rt = rangeTombstoneIterator.next();
Slice slice = rt.deletedSlice();
LegacyLayout.LegacyBound start = new LegacyLayout.LegacyBound(slice.start(), false, null);
LegacyLayout.LegacyBound end = new LegacyLayout.LegacyBound(slice.end(), false, null);
rtl.add(start, end, rt.deletionTime().markedForDeleteAt(), rt.deletionTime().localDeletionTime());
}
}
return new LegacyUnfilteredPartition(info.getPartitionDeletion(), rtl, cells);
}
public static void serializeAsLegacyPartition(ReadCommand command, UnfilteredRowIterator partition, DataOutputPlus out, int version) throws IOException
{
assert version < MessagingService.VERSION_30;
out.writeBoolean(true);
LegacyLayout.LegacyUnfilteredPartition legacyPartition = LegacyLayout.fromUnfilteredRowIterator(command, partition);
UUIDSerializer.serializer.serialize(partition.metadata().cfId, out, version);
DeletionTime.serializer.serialize(legacyPartition.partitionDeletion, out);
legacyPartition.rangeTombstones.serialize(out, partition.metadata());
// begin cell serialization
out.writeInt(legacyPartition.cells.size());
for (LegacyLayout.LegacyCell cell : legacyPartition.cells)
{
ByteBufferUtil.writeWithShortLength(cell.name.encode(partition.metadata()), out);
out.writeByte(cell.serializationFlags());
if (cell.isExpiring())
{
out.writeInt(cell.ttl);
out.writeInt(cell.localDeletionTime);
}
else if (cell.isTombstone())
{
out.writeLong(cell.timestamp);
out.writeInt(TypeSizes.sizeof(cell.localDeletionTime));
out.writeInt(cell.localDeletionTime);
continue;
}
else if (cell.isCounterUpdate())
{
out.writeLong(cell.timestamp);
long count = CounterContext.instance().getLocalCount(cell.value);
ByteBufferUtil.writeWithLength(ByteBufferUtil.bytes(count), out);
continue;
}
else if (cell.isCounter())
{
out.writeLong(Long.MIN_VALUE); // timestampOfLastDelete (not used, and MIN_VALUE is the default)
}
out.writeLong(cell.timestamp);
ByteBufferUtil.writeWithLength(cell.value, out);
}
}
// For the old wire format
// Note: this can return null if an empty partition is serialized!
public static UnfilteredRowIterator deserializeLegacyPartition(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
{
assert version < MessagingService.VERSION_30;
// This is only used in mutation, and mutation have never allowed "null" column families
boolean present = in.readBoolean();
if (!present)
return null;
CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
LegacyDeletionInfo info = LegacyDeletionInfo.deserialize(metadata, in);
int size = in.readInt();
Iterator<LegacyCell> cells = deserializeCells(metadata, in, flag, size);
SerializationHelper helper = new SerializationHelper(metadata, version, flag);
return onWireCellstoUnfilteredRowIterator(metadata, metadata.partitioner.decorateKey(key), info, cells, false, helper);
}
// For the old wire format
public static long serializedSizeAsLegacyPartition(ReadCommand command, UnfilteredRowIterator partition, int version)
{
assert version < MessagingService.VERSION_30;
if (partition.isEmpty())
return TypeSizes.sizeof(false);
long size = TypeSizes.sizeof(true);
LegacyLayout.LegacyUnfilteredPartition legacyPartition = LegacyLayout.fromUnfilteredRowIterator(command, partition);
size += UUIDSerializer.serializer.serializedSize(partition.metadata().cfId, version);
size += DeletionTime.serializer.serializedSize(legacyPartition.partitionDeletion);
size += legacyPartition.rangeTombstones.serializedSize(partition.metadata());
// begin cell serialization
size += TypeSizes.sizeof(legacyPartition.cells.size());
for (LegacyLayout.LegacyCell cell : legacyPartition.cells)
{
size += ByteBufferUtil.serializedSizeWithShortLength(cell.name.encode(partition.metadata()));
size += 1; // serialization flags
if (cell.kind == LegacyLayout.LegacyCell.Kind.EXPIRING)
{
size += TypeSizes.sizeof(cell.ttl);
size += TypeSizes.sizeof(cell.localDeletionTime);
}
else if (cell.kind == LegacyLayout.LegacyCell.Kind.DELETED)
{
size += TypeSizes.sizeof(cell.timestamp);
// localDeletionTime replaces cell.value as the body
size += TypeSizes.sizeof(TypeSizes.sizeof(cell.localDeletionTime));
size += TypeSizes.sizeof(cell.localDeletionTime);
continue;
}
else if (cell.kind == LegacyLayout.LegacyCell.Kind.COUNTER)
{
size += TypeSizes.sizeof(Long.MIN_VALUE); // timestampOfLastDelete
}
size += TypeSizes.sizeof(cell.timestamp);
size += ByteBufferUtil.serializedSizeWithLength(cell.value);
}
return size;
}
// For thrift sake
public static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata,
DecoratedKey key,
@ -547,32 +297,6 @@ public abstract class LegacyLayout
return toUnfilteredRowIterator(metadata, key, delInfo, cells, false, helper);
}
// For deserializing old wire format
public static UnfilteredRowIterator onWireCellstoUnfilteredRowIterator(CFMetaData metadata,
DecoratedKey key,
LegacyDeletionInfo delInfo,
Iterator<LegacyCell> cells,
boolean reversed,
SerializationHelper helper)
{
// If the table is a static compact, the "column_metadata" are now internally encoded as
// static. This has already been recognized by decodeCellName, but it means the cells
// provided are not in the expected order (the "static" cells are not necessarily at the front).
// So sort them to make sure toUnfilteredRowIterator works as expected.
// Further, if the query is reversed, then the on-wire format still has cells in non-reversed
// order, but we need to have them reverse in the final UnfilteredRowIterator. So reverse them.
if (metadata.isStaticCompactTable() || reversed)
{
List<LegacyCell> l = new ArrayList<>();
Iterators.addAll(l, cells);
Collections.sort(l, legacyCellComparator(metadata, reversed));
cells = l.iterator();
}
return toUnfilteredRowIterator(metadata, key, delInfo, cells, reversed, helper);
}
private static UnfilteredRowIterator toUnfilteredRowIterator(CFMetaData metadata,
DecoratedKey key,
LegacyDeletionInfo delInfo,
@ -624,47 +348,6 @@ public abstract class LegacyLayout
true);
}
public static Row extractStaticColumns(CFMetaData metadata, DataInputPlus in, Columns statics) throws IOException
{
assert !statics.isEmpty();
assert metadata.isCompactTable();
if (metadata.isSuper())
// TODO: there is in practice nothing to do here, but we need to handle the column_metadata for super columns somewhere else
throw new UnsupportedOperationException();
Set<ByteBuffer> columnsToFetch = new HashSet<>(statics.size());
for (ColumnDefinition column : statics)
columnsToFetch.add(column.name.bytes);
Row.Builder builder = BTreeRow.unsortedBuilder(FBUtilities.nowInSeconds());
builder.newRow(Clustering.STATIC_CLUSTERING);
boolean foundOne = false;
LegacyAtom atom;
while ((atom = readLegacyAtom(metadata, in, false)) != null)
{
if (atom.isCell())
{
LegacyCell cell = atom.asCell();
if (!columnsToFetch.contains(cell.name.encode(metadata)))
continue;
foundOne = true;
builder.addCell(new BufferCell(cell.name.column, cell.timestamp, cell.ttl, cell.localDeletionTime, cell.value, null));
}
else
{
LegacyRangeTombstone tombstone = atom.asRangeTombstone();
// TODO: we need to track tombstones and potentially ignore cells that are
// shadowed (or even better, replace them by tombstones).
throw new UnsupportedOperationException();
}
}
return foundOne ? builder.build() : Rows.EMPTY_STATIC_ROW;
}
private static Row getNextRow(CellGrouper grouper, PeekingIterator<? extends LegacyAtom> cells)
{
if (!cells.hasNext())
@ -714,7 +397,7 @@ public abstract class LegacyLayout
private Iterator<LegacyCell> initializeRow()
{
if (staticRow == null || staticRow.isEmpty())
return Collections.<LegacyLayout.LegacyCell>emptyIterator();
return Collections.emptyIterator();
Pair<LegacyRangeTombstoneList, Iterator<LegacyCell>> row = fromRow(metadata, staticRow);
deletions.addAll(row.left);
@ -843,7 +526,7 @@ public abstract class LegacyLayout
return legacyCellComparator(metadata, false);
}
public static Comparator<LegacyCell> legacyCellComparator(final CFMetaData metadata, final boolean reversed)
private static Comparator<LegacyCell> legacyCellComparator(final CFMetaData metadata, final boolean reversed)
{
final Comparator<LegacyCellName> cellNameComparator = legacyCellNameComparator(metadata, reversed);
return new Comparator<LegacyCell>()
@ -1013,121 +696,7 @@ public abstract class LegacyLayout
};
}
public static LegacyAtom readLegacyAtom(CFMetaData metadata, DataInputPlus in, boolean readAllAsDynamic) throws IOException
{
while (true)
{
ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
if (!cellname.hasRemaining())
return null; // END_OF_ROW
try
{
int b = in.readUnsignedByte();
return (b & RANGE_TOMBSTONE_MASK) != 0
? readLegacyRangeTombstoneBody(metadata, in, cellname)
: readLegacyCellBody(metadata, in, cellname, b, SerializationHelper.Flag.LOCAL, readAllAsDynamic);
}
catch (UnknownColumnException e)
{
// We can get there if we read a cell for a dropped column, and ff that is the case,
// then simply ignore the cell is fine. But also not that we ignore if it's the
// system keyspace because for those table we actually remove columns without registering
// them in the dropped columns
assert metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null : e.getMessage();
}
}
}
public static LegacyCell readLegacyCell(CFMetaData metadata, DataInput in, SerializationHelper.Flag flag) throws IOException, UnknownColumnException
{
ByteBuffer cellname = ByteBufferUtil.readWithShortLength(in);
int b = in.readUnsignedByte();
return readLegacyCellBody(metadata, in, cellname, b, flag, false);
}
public static LegacyCell readLegacyCellBody(CFMetaData metadata, DataInput in, ByteBuffer cellname, int mask, SerializationHelper.Flag flag, boolean readAllAsDynamic)
throws IOException, UnknownColumnException
{
// Note that we want to call decodeCellName only after we've deserialized other parts, since it can throw
// and we want to throw only after having deserialized the full cell.
if ((mask & COUNTER_MASK) != 0)
{
in.readLong(); // timestampOfLastDelete: this has been unused for a long time so we ignore it
long ts = in.readLong();
ByteBuffer value = ByteBufferUtil.readWithLength(in);
if (flag == SerializationHelper.Flag.FROM_REMOTE || (flag == SerializationHelper.Flag.LOCAL && CounterContext.instance().shouldClearLocal(value)))
value = CounterContext.instance().clearAllLocal(value);
return new LegacyCell(LegacyCell.Kind.COUNTER, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, Cell.NO_DELETION_TIME, Cell.NO_TTL);
}
else if ((mask & EXPIRATION_MASK) != 0)
{
int ttl = in.readInt();
int expiration = in.readInt();
long ts = in.readLong();
ByteBuffer value = ByteBufferUtil.readWithLength(in);
return new LegacyCell(LegacyCell.Kind.EXPIRING, decodeCellName(metadata, cellname, readAllAsDynamic), value, ts, expiration, ttl);
}
else
{
long ts = in.readLong();
ByteBuffer value = ByteBufferUtil.readWithLength(in);
LegacyCellName name = decodeCellName(metadata, cellname, readAllAsDynamic);
return (mask & COUNTER_UPDATE_MASK) != 0
? new LegacyCell(LegacyCell.Kind.COUNTER, name, CounterContext.instance().createLocal(ByteBufferUtil.toLong(value)), ts, Cell.NO_DELETION_TIME, Cell.NO_TTL)
: ((mask & DELETION_MASK) == 0
? new LegacyCell(LegacyCell.Kind.REGULAR, name, value, ts, Cell.NO_DELETION_TIME, Cell.NO_TTL)
: new LegacyCell(LegacyCell.Kind.DELETED, name, ByteBufferUtil.EMPTY_BYTE_BUFFER, ts, ByteBufferUtil.toInt(value), Cell.NO_TTL));
}
}
public static LegacyRangeTombstone readLegacyRangeTombstoneBody(CFMetaData metadata, DataInputPlus in, ByteBuffer boundname) throws IOException
{
LegacyBound min = decodeBound(metadata, boundname, true);
LegacyBound max = decodeBound(metadata, ByteBufferUtil.readWithShortLength(in), false);
DeletionTime dt = DeletionTime.serializer.deserialize(in);
return new LegacyRangeTombstone(min, max, dt);
}
public static Iterator<LegacyCell> deserializeCells(final CFMetaData metadata,
final DataInput in,
final SerializationHelper.Flag flag,
final int size)
{
return new AbstractIterator<LegacyCell>()
{
private int i = 0;
protected LegacyCell computeNext()
{
if (i >= size)
return endOfData();
++i;
try
{
return readLegacyCell(metadata, in, flag);
}
catch (UnknownColumnException e)
{
// We can get there if we read a cell for a dropped column, and if that is the case,
// then simply ignore the cell is fine. But also not that we ignore if it's the
// system keyspace because for those table we actually remove columns without registering
// them in the dropped columns
if (metadata.ksName.equals(SchemaConstants.SYSTEM_KEYSPACE_NAME) || metadata.getDroppedColumnDefinition(e.columnName) != null)
return computeNext();
else
throw new IOError(e);
}
catch (IOException e)
{
throw new IOError(e);
}
}
};
}
public static class CellGrouper
private static class CellGrouper
{
public final CFMetaData metadata;
private final boolean isStatic;
@ -1285,53 +854,6 @@ public abstract class LegacyLayout
}
}
public static class LegacyUnfilteredPartition
{
public final DeletionTime partitionDeletion;
public final LegacyRangeTombstoneList rangeTombstones;
public final List<LegacyCell> cells;
private LegacyUnfilteredPartition(DeletionTime partitionDeletion, LegacyRangeTombstoneList rangeTombstones, List<LegacyCell> cells)
{
this.partitionDeletion = partitionDeletion;
this.rangeTombstones = rangeTombstones;
this.cells = cells;
}
public void digest(CFMetaData metadata, MessageDigest digest)
{
for (LegacyCell cell : cells)
{
digest.update(cell.name.encode(metadata).duplicate());
if (cell.isCounter())
CounterContext.instance().updateDigest(digest, cell.value);
else
digest.update(cell.value.duplicate());
FBUtilities.updateWithLong(digest, cell.timestamp);
FBUtilities.updateWithByte(digest, cell.serializationFlags());
if (cell.isExpiring())
FBUtilities.updateWithInt(digest, cell.ttl);
if (cell.isCounter())
{
// Counters used to have the timestampOfLastDelete field, which we stopped using long ago and has been hard-coded
// to Long.MIN_VALUE but was still taken into account in 2.2 counter digests (to maintain backward compatibility
// in the first place).
FBUtilities.updateWithLong(digest, Long.MIN_VALUE);
}
}
if (partitionDeletion.markedForDeleteAt() != Long.MIN_VALUE)
digest.update(ByteBufferUtil.bytes(partitionDeletion.markedForDeleteAt()));
if (!rangeTombstones.isEmpty())
rangeTombstones.updateDigest(digest);
}
}
public static class LegacyCellName
{
public final Clustering clustering;
@ -1822,7 +1344,7 @@ public abstract class LegacyLayout
* This class is needed to allow us to convert single-row deletions and complex deletions into range tombstones
* and properly merge them into the normal set of range tombstones.
*/
public static class LegacyRangeTombstoneList
private static class LegacyRangeTombstoneList
{
private final LegacyBoundComparator comparator;

12
src/java/org/apache/cassandra/db/Memtable.java
View File

@ -449,9 +449,9 @@ public class Memtable implements Comparable<Memtable>
this.isBatchLogTable = cfs.name.equals(SystemKeyspace.BATCHES) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
if (flushLocation == null)
writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getWriteableLocationAsFile(estimatedSize)), columnsCollector.get(), statsCollector.get());
else
writer = createFlushWriter(txn, cfs.getSSTablePath(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
writer = createFlushWriter(txn, cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(flushLocation)), columnsCollector.get(), statsCollector.get());
}
@ -503,14 +503,14 @@ public class Memtable implements Comparable<Memtable>
}
public SSTableMultiWriter createFlushWriter(LifecycleTransaction txn,
String filename,
PartitionColumns columns,
EncodingStats stats)
Descriptor descriptor,
PartitionColumns columns,
EncodingStats stats)
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.metadata.comparator)
.commitLogIntervals(new IntervalSet<>(commitLogLowerBound.get(), commitLogUpperBound.get()));
return cfs.createSSTableMultiWriter(Descriptor.fromFilename(filename),
return cfs.createSSTableMultiWriter(descriptor,
toFlush.size(),
ActiveRepairService.UNREPAIRED_SSTABLE,
sstableMetadataCollector,

51
src/java/org/apache/cassandra/db/Mutation.java
View File

@ -368,21 +368,9 @@ public class Mutation implements IMutation
{
public void serialize(Mutation mutation, DataOutputPlus out, int version) throws IOException
{
if (version < MessagingService.VERSION_20)
out.writeUTF(mutation.getKeyspaceName());
/* serialize the modifications in the mutation */
int size = mutation.modifications.size();
if (version < MessagingService.VERSION_30)
{
ByteBufferUtil.writeWithShortLength(mutation.key().getKey(), out);
out.writeInt(size);
}
else
{
out.writeUnsignedVInt(size);
}
out.writeUnsignedVInt(size);
assert size > 0;
for (Map.Entry<UUID, PartitionUpdate> entry : mutation.modifications.entrySet())
@ -391,24 +379,10 @@ public class Mutation implements IMutation
public Mutation deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
{
if (version < MessagingService.VERSION_20)
in.readUTF(); // read pre-2.0 keyspace name
ByteBuffer key = null;
int size;
if (version < MessagingService.VERSION_30)
{
key = ByteBufferUtil.readWithShortLength(in);
size = in.readInt();
}
else
{
size = (int)in.readUnsignedVInt();
}
int size = (int)in.readUnsignedVInt();
assert size > 0;
PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, flag, key);
PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, flag);
if (size == 1)
return new Mutation(update);
@ -418,7 +392,7 @@ public class Mutation implements IMutation
modifications.put(update.metadata().cfId, update);
for (int i = 1; i < size; ++i)
{
update = PartitionUpdate.serializer.deserialize(in, version, flag, dk);
update = PartitionUpdate.serializer.deserialize(in, version, flag);
modifications.put(update.metadata().cfId, update);
}
@ -432,22 +406,7 @@ public class Mutation implements IMutation
public long serializedSize(Mutation mutation, int version)
{
int size = 0;
if (version < MessagingService.VERSION_20)
size += TypeSizes.sizeof(mutation.getKeyspaceName());
if (version < MessagingService.VERSION_30)
{
int keySize = mutation.key().getKey().remaining();
size += TypeSizes.sizeof((short) keySize) + keySize;
size += TypeSizes.sizeof(mutation.modifications.size());
}
else
{
size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
}
int size = TypeSizes.sizeofUnsignedVInt(mutation.modifications.size());
for (Map.Entry<UUID, PartitionUpdate> entry : mutation.modifications.entrySet())
size += PartitionUpdate.serializer.serializedSize(entry.getValue(), version);

19
src/java/org/apache/cassandra/db/MutationVerbHandler.java
View File

@ -21,7 +21,6 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetAddress;
import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.io.util.FastByteArrayInputStream;
import org.apache.cassandra.net.*;
@ -59,16 +58,10 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
try
{
if (message.version < MessagingService.VERSION_30 && LegacyBatchlogMigrator.isLegacyBatchlogMutation(message.payload))
{
LegacyBatchlogMigrator.handleLegacyMutation(message.payload);
reply(id, replyTo);
}
else
message.payload.applyFuture().thenAccept(o -> reply(id, replyTo)).exceptionally(wto -> {
failed();
return null;
});
message.payload.applyFuture().thenAccept(o -> reply(id, replyTo)).exceptionally(wto -> {
failed();
return null;
});
}
catch (WriteTimeoutException wto)
{
@ -76,10 +69,6 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
}
}
/**
* Older version (< 1.0) will not send this message at all, hence we don't
* need to check the version of the data.
*/
private static void forwardToLocalNodes(Mutation mutation, MessagingService.Verb verb, byte[] forwardBytes, InetAddress from) throws IOException
{
try (DataInputStream in = new DataInputStream(new FastByteArrayInputStream(forwardBytes)))

6
src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
View File

@ -273,11 +273,9 @@ public class PartitionRangeReadCommand extends ReadCommand
return Transformation.apply(iter, new CacheFilter());
}
public MessageOut<ReadCommand> createMessage(int version)
public MessageOut<ReadCommand> createMessage()
{
return dataRange().isPaging()
? new MessageOut<>(MessagingService.Verb.PAGED_RANGE, this, pagedRangeSerializer)
: new MessageOut<>(MessagingService.Verb.RANGE_SLICE, this, rangeSliceSerializer);
return new MessageOut<>(MessagingService.Verb.RANGE_SLICE, this, serializer);
}
protected void appendCQLWhereClause(StringBuilder sb)

29
src/java/org/apache/cassandra/db/RangeSliceVerbHandler.java
View File

@ -1,29 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import org.apache.cassandra.io.IVersionedSerializer;
public class RangeSliceVerbHandler extends ReadCommandVerbHandler
{
@Override
protected IVersionedSerializer<ReadResponse> serializer()
{
return ReadResponse.rangeSliceSerializer;
}
}

1061
src/java/org/apache/cassandra/db/ReadCommand.java
View File

File diff suppressed because it is too large Load Diff

264
src/java/org/apache/cassandra/db/ReadResponse.java
View File

@ -32,7 +32,6 @@ import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.partitions.*;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.io.ForwardingVersionedSerializer;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
@ -47,20 +46,6 @@ public abstract class ReadResponse
{
// Serializer for single partition read response
public static final IVersionedSerializer<ReadResponse> serializer = new Serializer();
// Serializer for the pre-3.0 rang slice responses.
public static final IVersionedSerializer<ReadResponse> legacyRangeSliceReplySerializer = new LegacyRangeSliceReplySerializer();
// Serializer for partition range read response (this actually delegate to 'serializer' in 3.0 and to
// 'legacyRangeSliceReplySerializer' in older version.
public static final IVersionedSerializer<ReadResponse> rangeSliceSerializer = new ForwardingVersionedSerializer<ReadResponse>()
{
@Override
protected IVersionedSerializer<ReadResponse> delegate(int version)
{
return version < MessagingService.VERSION_30
? legacyRangeSliceReplySerializer
: serializer;
}
};
// This is used only when serializing data responses and we can't it easily in other cases. So this can be null, which is slighly
// hacky, but as this hack doesn't escape this class, and it's easy enough to validate that it's not null when we need, it's "good enough".
@ -95,7 +80,7 @@ public abstract class ReadResponse
protected static ByteBuffer makeDigest(UnfilteredPartitionIterator iterator, ReadCommand command)
{
MessageDigest digest = FBUtilities.threadLocalMD5Digest();
UnfilteredPartitionIterators.digest(command, iterator, digest, command.digestVersion());
UnfilteredPartitionIterators.digest(iterator, digest, command.digestVersion());
return ByteBuffer.wrap(digest.digest());
}
@ -210,130 +195,12 @@ public abstract class ReadResponse
}
}
/**
* A remote response from a pre-3.0 node. This needs a separate class in order to cleanly handle trimming and
* reversal of results when the read command calls for it. Pre-3.0 nodes always return results in the normal
* sorted order, even if the query asks for reversed results. Additionally, pre-3.0 nodes do not have a notion of
* exclusive slices on non-composite tables, so extra rows may need to be trimmed.
*/
@VisibleForTesting
static class LegacyRemoteDataResponse extends ReadResponse
{
private final List<ImmutableBTreePartition> partitions;
@VisibleForTesting
LegacyRemoteDataResponse(List<ImmutableBTreePartition> partitions)
{
super(null); // we never serialize LegacyRemoteDataResponses, so we don't care about the command
this.partitions = partitions;
}
public UnfilteredPartitionIterator makeIterator(final ReadCommand command)
{
// Due to a bug in the serialization of AbstractBounds, anything that isn't a Range is understood by pre-3.0 nodes
// as a Bound, which means IncludingExcludingBounds and ExcludingBounds responses may include keys they shouldn't.
// So filter partitions that shouldn't be included here.
boolean skipFirst = false;
boolean skipLast = false;
if (!partitions.isEmpty() && command instanceof PartitionRangeReadCommand)
{
AbstractBounds<PartitionPosition> keyRange = ((PartitionRangeReadCommand)command).dataRange().keyRange();
boolean isExcludingBounds = keyRange instanceof ExcludingBounds;
skipFirst = isExcludingBounds && !keyRange.contains(partitions.get(0).partitionKey());
skipLast = (isExcludingBounds || keyRange instanceof IncludingExcludingBounds) && !keyRange.contains(partitions.get(partitions.size() - 1).partitionKey());
}
final List<ImmutableBTreePartition> toReturn;
if (skipFirst || skipLast)
{
toReturn = partitions.size() == 1
? Collections.emptyList()
: partitions.subList(skipFirst ? 1 : 0, skipLast ? partitions.size() - 1 : partitions.size());
}
else
{
toReturn = partitions;
}
return new AbstractUnfilteredPartitionIterator()
{
private int idx;
public boolean isForThrift()
{
return true;
}
public CFMetaData metadata()
{
return command.metadata();
}
public boolean hasNext()
{
return idx < toReturn.size();
}
public UnfilteredRowIterator next()
{
ImmutableBTreePartition partition = toReturn.get(idx++);
ClusteringIndexFilter filter = command.clusteringIndexFilter(partition.partitionKey());
// Pre-3.0, we would always request one more row than we actually needed and the command-level "start" would
// be the last-returned cell name, so the response would always include it.
UnfilteredRowIterator iterator = partition.unfilteredIterator(command.columnFilter(), filter.getSlices(command.metadata()), filter.isReversed());
// Wrap results with a ThriftResultMerger only if they're intended for the thrift command.
if (command.isForThrift())
return ThriftResultsMerger.maybeWrap(iterator, command.nowInSec());
else
return iterator;
}
};
}
public ByteBuffer digest(ReadCommand command)
{
try (UnfilteredPartitionIterator iterator = makeIterator(command))
{
return makeDigest(iterator, command);
}
}
public boolean isDigestResponse()
{
return false;
}
}
private static class Serializer implements IVersionedSerializer<ReadResponse>
{
public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException
{
boolean isDigest = response instanceof DigestResponse;
ByteBuffer digest = isDigest ? ((DigestResponse)response).digest : ByteBufferUtil.EMPTY_BYTE_BUFFER;
if (version < MessagingService.VERSION_30)
{
out.writeInt(digest.remaining());
out.write(digest);
out.writeBoolean(isDigest);
if (!isDigest)
{
assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
try (UnfilteredPartitionIterator iter = response.makeIterator(response.command))
{
assert iter.hasNext();
try (UnfilteredRowIterator partition = iter.next())
{
ByteBufferUtil.writeWithShortLength(partition.partitionKey().getKey(), out);
LegacyLayout.serializeAsLegacyPartition(response.command, partition, out, version);
}
assert !iter.hasNext();
}
}
return;
}
ByteBufferUtil.writeWithVIntLength(digest, out);
if (!isDigest)
@ -345,38 +212,12 @@ public abstract class ReadResponse
public ReadResponse deserialize(DataInputPlus in, int version) throws IOException
{
if (version < MessagingService.VERSION_30)
{
byte[] digest = null;
int digestSize = in.readInt();
if (digestSize > 0)
{
digest = new byte[digestSize];
in.readFully(digest, 0, digestSize);
}
boolean isDigest = in.readBoolean();
assert isDigest == digestSize > 0;
if (isDigest)
{
assert digest != null;
return new DigestResponse(ByteBuffer.wrap(digest));
}
// ReadResponses from older versions are always single-partition (ranges are handled by RangeSliceReply)
ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
try (UnfilteredRowIterator rowIterator = LegacyLayout.deserializeLegacyPartition(in, version, SerializationHelper.Flag.FROM_REMOTE, key))
{
if (rowIterator == null)
return new LegacyRemoteDataResponse(Collections.emptyList());
return new LegacyRemoteDataResponse(Collections.singletonList(ImmutableBTreePartition.create(rowIterator)));
}
}
ByteBuffer digest = ByteBufferUtil.readWithVIntLength(in);
if (digest.hasRemaining())
return new DigestResponse(digest);
// Note that we can only get there if version == 3.0, which is the current_version. When we'll change the
// version, we'll have to deserialize/re-serialize the data to be in the proper version.
assert version == MessagingService.VERSION_30;
ByteBuffer data = ByteBufferUtil.readWithVIntLength(in);
return new RemoteDataResponse(data);
@ -387,28 +228,6 @@ public abstract class ReadResponse
boolean isDigest = response instanceof DigestResponse;
ByteBuffer digest = isDigest ? ((DigestResponse)response).digest : ByteBufferUtil.EMPTY_BYTE_BUFFER;
if (version < MessagingService.VERSION_30)
{
long size = TypeSizes.sizeof(digest.remaining())
+ digest.remaining()
+ TypeSizes.sizeof(isDigest);
if (!isDigest)
{
assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
try (UnfilteredPartitionIterator iter = response.makeIterator(response.command))
{
assert iter.hasNext();
try (UnfilteredRowIterator partition = iter.next())
{
size += ByteBufferUtil.serializedSizeWithShortLength(partition.partitionKey().getKey());
size += LegacyLayout.serializedSizeAsLegacyPartition(response.command, partition, version);
}
assert !iter.hasNext();
}
}
return size;
}
long size = ByteBufferUtil.serializedSizeWithVIntLength(digest);
if (!isDigest)
{
@ -421,81 +240,4 @@ public abstract class ReadResponse
return size;
}
}
private static class LegacyRangeSliceReplySerializer implements IVersionedSerializer<ReadResponse>
{
public void serialize(ReadResponse response, DataOutputPlus out, int version) throws IOException
{
assert version < MessagingService.VERSION_30;
// determine the number of partitions upfront for serialization
int numPartitions = 0;
assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
{
while (iterator.hasNext())
{
try (UnfilteredRowIterator atomIterator = iterator.next())
{
numPartitions++;
// we have to fully exhaust the subiterator
while (atomIterator.hasNext())
atomIterator.next();
}
}
}
out.writeInt(numPartitions);
try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
{
while (iterator.hasNext())
{
try (UnfilteredRowIterator partition = iterator.next())
{
ByteBufferUtil.writeWithShortLength(partition.partitionKey().getKey(), out);
LegacyLayout.serializeAsLegacyPartition(response.command, partition, out, version);
}
}
}
}
public ReadResponse deserialize(DataInputPlus in, int version) throws IOException
{
assert version < MessagingService.VERSION_30;
int partitionCount = in.readInt();
ArrayList<ImmutableBTreePartition> partitions = new ArrayList<>(partitionCount);
for (int i = 0; i < partitionCount; i++)
{
ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
try (UnfilteredRowIterator partition = LegacyLayout.deserializeLegacyPartition(in, version, SerializationHelper.Flag.FROM_REMOTE, key))
{
partitions.add(ImmutableBTreePartition.create(partition));
}
}
return new LegacyRemoteDataResponse(partitions);
}
public long serializedSize(ReadResponse response, int version)
{
assert version < MessagingService.VERSION_30;
long size = TypeSizes.sizeof(0); // number of partitions
assert response.command != null; // we only serialize LocalDataResponse, which always has the command set
try (UnfilteredPartitionIterator iterator = response.makeIterator(response.command))
{
while (iterator.hasNext())
{
try (UnfilteredRowIterator partition = iterator.next())
{
size += ByteBufferUtil.serializedSizeWithShortLength(partition.partitionKey().getKey());
size += LegacyLayout.serializedSizeAsLegacyPartition(response.command, partition, version);
}
}
}
return size;
}
}
}

189
src/java/org/apache/cassandra/db/RowIndexEntry.java
View File

@ -253,7 +253,7 @@ public class RowIndexEntry<T> implements IMeasurableMemory
public Serializer(CFMetaData metadata, Version version, SerializationHeader header)
{
this.idxInfoSerializer = metadata.serializers().indexInfoSerializer(version, header);
this.idxInfoSerializer = IndexInfo.serializer(version, header);
this.version = version;
}
@ -264,22 +264,16 @@ public class RowIndexEntry<T> implements IMeasurableMemory
public void serialize(RowIndexEntry<IndexInfo> rie, DataOutputPlus out, ByteBuffer indexInfo) throws IOException
{
assert version.storeRows() : "We read old index files but we should never write them";
rie.serialize(out, idxInfoSerializer, indexInfo);
}
public void serializeForCache(RowIndexEntry<IndexInfo> rie, DataOutputPlus out) throws IOException
{
assert version.storeRows();
rie.serializeForCache(out);
}
public RowIndexEntry<IndexInfo> deserializeForCache(DataInputPlus in) throws IOException
{
assert version.storeRows();
long position = in.readUnsignedVInt();
switch (in.readByte())
@ -297,8 +291,6 @@ public class RowIndexEntry<T> implements IMeasurableMemory
public static void skipForCache(DataInputPlus in, Version version) throws IOException
{
assert version.storeRows();
/* long position = */in.readUnsignedVInt();
switch (in.readByte())
{
@ -317,9 +309,6 @@ public class RowIndexEntry<T> implements IMeasurableMemory
public RowIndexEntry<IndexInfo> deserialize(DataInputPlus in, long indexFilePosition) throws IOException
{
if (!version.storeRows())
return LegacyShallowIndexedEntry.deserialize(in, indexFilePosition, idxInfoSerializer);
long position = in.readUnsignedVInt();
int size = (int)in.readUnsignedVInt();
@ -354,9 +343,6 @@ public class RowIndexEntry<T> implements IMeasurableMemory
public long deserializePositionAndSkip(DataInputPlus in) throws IOException
{
if (!version.storeRows())
return LegacyShallowIndexedEntry.deserializePositionAndSkip(in);
return ShallowIndexedEntry.deserializePositionAndSkip(in);
}
@ -367,7 +353,7 @@ public class RowIndexEntry<T> implements IMeasurableMemory
*/
public static long readPosition(DataInputPlus in, Version version) throws IOException
{
return version.storeRows() ? in.readUnsignedVInt() : in.readLong();
return in.readUnsignedVInt();
}
public static void skip(DataInputPlus in, Version version) throws IOException
@ -378,7 +364,7 @@ public class RowIndexEntry<T> implements IMeasurableMemory
private static void skipPromotedIndex(DataInputPlus in, Version version) throws IOException
{
int size = version.storeRows() ? (int)in.readUnsignedVInt() : in.readInt();
int size = (int)in.readUnsignedVInt();
if (size <= 0)
return;
@ -413,164 +399,6 @@ public class RowIndexEntry<T> implements IMeasurableMemory
out.writeByte(CACHE_NOT_INDEXED);
}
private static final class LegacyShallowIndexedEntry extends RowIndexEntry<IndexInfo>
{
private static final long BASE_SIZE;
static
{
BASE_SIZE = ObjectSizes.measure(new LegacyShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, new int[0], null, 0));
}
private final long indexFilePosition;
private final int[] offsets;
@Unmetered
private final IndexInfo.Serializer idxInfoSerializer;
private final DeletionTime deletionTime;
private final long headerLength;
private final int serializedSize;
private LegacyShallowIndexedEntry(long dataFilePosition, long indexFilePosition,
DeletionTime deletionTime, long headerLength,
int[] offsets, IndexInfo.Serializer idxInfoSerializer,
int serializedSize)
{
super(dataFilePosition);
this.deletionTime = deletionTime;
this.headerLength = headerLength;
this.indexFilePosition = indexFilePosition;
this.offsets = offsets;
this.idxInfoSerializer = idxInfoSerializer;
this.serializedSize = serializedSize;
}
@Override
public DeletionTime deletionTime()
{
return deletionTime;
}
@Override
public long headerLength()
{
return headerLength;
}
@Override
public long unsharedHeapSize()
{
return BASE_SIZE + offsets.length * TypeSizes.sizeof(0);
}
@Override
public int columnsIndexCount()
{
return offsets.length;
}
@Override
public void serialize(DataOutputPlus out, IndexInfo.Serializer idxInfoSerializer, ByteBuffer indexInfo)
{
throw new UnsupportedOperationException("serializing legacy index entries is not supported");
}
@Override
public void serializeForCache(DataOutputPlus out)
{
throw new UnsupportedOperationException("serializing legacy index entries is not supported");
}
@Override
public IndexInfoRetriever openWithIndex(FileHandle indexFile)
{
int fieldsSize = (int) DeletionTime.serializer.serializedSize(deletionTime)
+ TypeSizes.sizeof(0); // columnIndexCount
indexEntrySizeHistogram.update(serializedSize);
indexInfoCountHistogram.update(offsets.length);
return new LegacyIndexInfoRetriever(indexFilePosition +
TypeSizes.sizeof(0L) + // position
TypeSizes.sizeof(0) + // indexInfoSize
fieldsSize,
offsets, indexFile.createReader(), idxInfoSerializer);
}
public static RowIndexEntry<IndexInfo> deserialize(DataInputPlus in, long indexFilePosition,
IndexInfo.Serializer idxInfoSerializer) throws IOException
{
long dataFilePosition = in.readLong();
int size = in.readInt();
if (size == 0)
{
return new RowIndexEntry<>(dataFilePosition);
}
else if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
{
return new IndexedEntry(dataFilePosition, in, idxInfoSerializer);
}
else
{
DeletionTime deletionTime = DeletionTime.serializer.deserialize(in);
// For legacy sstables (i.e. sstables pre-"ma", pre-3.0) we have to scan all serialized IndexInfo
// objects to calculate the offsets array. However, it might be possible to deserialize all
// IndexInfo objects here - but to just skip feels more gentle to the heap/GC.
int entries = in.readInt();
int[] offsets = new int[entries];
TrackedDataInputPlus tracked = new TrackedDataInputPlus(in);
long start = tracked.getBytesRead();
long headerLength = 0L;
for (int i = 0; i < entries; i++)
{
offsets[i] = (int) (tracked.getBytesRead() - start);
if (i == 0)
{
IndexInfo info = idxInfoSerializer.deserialize(tracked);
headerLength = info.offset;
}
else
idxInfoSerializer.skip(tracked);
}
return new LegacyShallowIndexedEntry(dataFilePosition, indexFilePosition, deletionTime, headerLength, offsets, idxInfoSerializer, size);
}
}
static long deserializePositionAndSkip(DataInputPlus in) throws IOException
{
long position = in.readLong();
int size = in.readInt();
if (size > 0)
in.skipBytesFully(size);
return position;
}
}
private static final class LegacyIndexInfoRetriever extends FileIndexInfoRetriever
{
private final int[] offsets;
private LegacyIndexInfoRetriever(long indexFilePosition, int[] offsets, FileDataInput reader, IndexInfo.Serializer idxInfoSerializer)
{
super(indexFilePosition, reader, idxInfoSerializer);
this.offsets = offsets;
}
IndexInfo fetchIndex(int index) throws IOException
{
retrievals++;
// seek to posision of IndexInfo
indexReader.seek(indexInfoFilePosition + offsets[index]);
// deserialize IndexInfo
return idxInfoSerializer.deserialize(indexReader);
}
}
/**
* An entry in the row index for a row whose columns are indexed - used for both legacy and current formats.
*/
@ -622,14 +450,9 @@ public class RowIndexEntry<T> implements IMeasurableMemory
for (int i = 0; i < columnsIndexCount; i++)
this.columnsIndex[i] = idxInfoSerializer.deserialize(in);
int[] offsets = null;
if (version.storeRows())
{
offsets = new int[this.columnsIndex.length];
for (int i = 0; i < offsets.length; i++)
offsets[i] = in.readInt();
}
this.offsets = offsets;
this.offsets = new int[this.columnsIndex.length];
for (int i = 0; i < offsets.length; i++)
offsets[i] = in.readInt();
this.indexedPartSize = indexedPartSize;

183
src/java/org/apache/cassandra/db/Serializers.java
View File

@ -1,183 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.IndexInfo;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.utils.ByteBufferUtil;
/**
* Holds references on serializers that depend on the table definition.
*/
public class Serializers
{
private final CFMetaData metadata;
private Map<Version, IndexInfo.Serializer> otherVersionClusteringSerializers;
private final IndexInfo.Serializer latestVersionIndexSerializer;
public Serializers(CFMetaData metadata)
{
this.metadata = metadata;
this.latestVersionIndexSerializer = new IndexInfo.Serializer(BigFormat.latestVersion,
indexEntryClusteringPrefixSerializer(BigFormat.latestVersion, SerializationHeader.makeWithoutStats(metadata)));
}
IndexInfo.Serializer indexInfoSerializer(Version version, SerializationHeader header)
{
// null header indicates streaming from pre-3.0 sstables
if (version.equals(BigFormat.latestVersion) && header != null)
return latestVersionIndexSerializer;
if (otherVersionClusteringSerializers == null)
otherVersionClusteringSerializers = new ConcurrentHashMap<>();
IndexInfo.Serializer serializer = otherVersionClusteringSerializers.get(version);
if (serializer == null)
{
serializer = new IndexInfo.Serializer(version,
indexEntryClusteringPrefixSerializer(version, header));
otherVersionClusteringSerializers.put(version, serializer);
}
return serializer;
}
// TODO: Once we drop support for old (pre-3.0) sstables, we can drop this method and inline the calls to
// ClusteringPrefix.serializer directly. At which point this whole class probably becomes
// unecessary (since IndexInfo.Serializer won't depend on the metadata either).
private ISerializer<ClusteringPrefix> indexEntryClusteringPrefixSerializer(Version version, SerializationHeader header)
{
if (!version.storeRows() || header == null) //null header indicates streaming from pre-3.0 sstables
{
return oldFormatSerializer(version);
}
return new NewFormatSerializer(version, header.clusteringTypes());
}
private ISerializer<ClusteringPrefix> oldFormatSerializer(Version version)
{
return new ISerializer<ClusteringPrefix>()
{
List<AbstractType<?>> clusteringTypes = SerializationHeader.makeWithoutStats(metadata).clusteringTypes();
public void serialize(ClusteringPrefix clustering, DataOutputPlus out) throws IOException
{
//we deserialize in the old format and serialize in the new format
ClusteringPrefix.serializer.serialize(clustering, out,
version.correspondingMessagingVersion(),
clusteringTypes);
}
@Override
public void skip(DataInputPlus in) throws IOException
{
ByteBufferUtil.skipShortLength(in);
}
public ClusteringPrefix deserialize(DataInputPlus in) throws IOException
{
// We're reading the old cellname/composite
ByteBuffer bb = ByteBufferUtil.readWithShortLength(in);
assert bb.hasRemaining(); // empty cellnames were invalid
int clusteringSize = metadata.clusteringColumns().size();
// If the table has no clustering column, then the cellname will just be the "column" name, which we ignore here.
if (clusteringSize == 0)
return Clustering.EMPTY;
if (!metadata.isCompound())
return Clustering.make(bb);
List<ByteBuffer> components = CompositeType.splitName(bb);
byte eoc = CompositeType.lastEOC(bb);
if (eoc == 0 || components.size() >= clusteringSize)
{
// That's a clustering.
if (components.size() > clusteringSize)
components = components.subList(0, clusteringSize);
return Clustering.make(components.toArray(new ByteBuffer[clusteringSize]));
}
else
{
// It's a range tombstone bound. It is a start since that's the only part we've ever included
// in the index entries.
ClusteringPrefix.Kind boundKind = eoc > 0
? ClusteringPrefix.Kind.EXCL_START_BOUND
: ClusteringPrefix.Kind.INCL_START_BOUND;
return ClusteringBound.create(boundKind, components.toArray(new ByteBuffer[components.size()]));
}
}
public long serializedSize(ClusteringPrefix clustering)
{
return ClusteringPrefix.serializer.serializedSize(clustering, version.correspondingMessagingVersion(),
clusteringTypes);
}
};
}
private static class NewFormatSerializer implements ISerializer<ClusteringPrefix>
{
private final Version version;
private final List<AbstractType<?>> clusteringTypes;
NewFormatSerializer(Version version, List<AbstractType<?>> clusteringTypes)
{
this.version = version;
this.clusteringTypes = clusteringTypes;
}
public void serialize(ClusteringPrefix clustering, DataOutputPlus out) throws IOException
{
ClusteringPrefix.serializer.serialize(clustering, out, version.correspondingMessagingVersion(), clusteringTypes);
}
@Override
public void skip(DataInputPlus in) throws IOException
{
ClusteringPrefix.serializer.skip(in, version.correspondingMessagingVersion(), clusteringTypes);
}
public ClusteringPrefix deserialize(DataInputPlus in) throws IOException
{
return ClusteringPrefix.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes);
}
public long serializedSize(ClusteringPrefix clustering)
{
return ClusteringPrefix.serializer.serializedSize(clustering, version.correspondingMessagingVersion(), clusteringTypes);
}
}
}

4
src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
View File

@ -935,9 +935,9 @@ public class SinglePartitionReadCommand extends ReadCommand
nowInSec());
}
public MessageOut<ReadCommand> createMessage(int version)
public MessageOut<ReadCommand> createMessage()
{
return new MessageOut<>(MessagingService.Verb.READ, this, readSerializer);
return new MessageOut<>(MessagingService.Verb.READ, this, serializer);
}
protected void appendCQLWhereClause(StringBuilder sb)

229
src/java/org/apache/cassandra/db/SystemKeyspace.java
View File

@ -102,16 +102,6 @@ public final class SystemKeyspace
public static final String BUILT_VIEWS = "built_views";
public static final String PREPARED_STATEMENTS = "prepared_statements";
@Deprecated public static final String LEGACY_HINTS = "hints";
@Deprecated public static final String LEGACY_BATCHLOG = "batchlog";
@Deprecated public static final String LEGACY_KEYSPACES = "schema_keyspaces";
@Deprecated public static final String LEGACY_COLUMNFAMILIES = "schema_columnfamilies";
@Deprecated public static final String LEGACY_COLUMNS = "schema_columns";
@Deprecated public static final String LEGACY_TRIGGERS = "schema_triggers";
@Deprecated public static final String LEGACY_USERTYPES = "schema_usertypes";
@Deprecated public static final String LEGACY_FUNCTIONS = "schema_functions";
@Deprecated public static final String LEGACY_AGGREGATES = "schema_aggregates";
public static final CFMetaData Batches =
compile(BATCHES,
"batches awaiting replay",
@ -288,148 +278,6 @@ public final class SystemKeyspace
+ "query_string text,"
+ "PRIMARY KEY ((prepared_id)))");
@Deprecated
public static final CFMetaData LegacyHints =
compile(LEGACY_HINTS,
"*DEPRECATED* hints awaiting delivery",
"CREATE TABLE %s ("
+ "target_id uuid,"
+ "hint_id timeuuid,"
+ "message_version int,"
+ "mutation blob,"
+ "PRIMARY KEY ((target_id), hint_id, message_version)) "
+ "WITH COMPACT STORAGE")
.compaction(CompactionParams.scts(singletonMap("enabled", "false")))
.gcGraceSeconds(0);
@Deprecated
public static final CFMetaData LegacyBatchlog =
compile(LEGACY_BATCHLOG,
"*DEPRECATED* batchlog entries",
"CREATE TABLE %s ("
+ "id uuid,"
+ "data blob,"
+ "version int,"
+ "written_at timestamp,"
+ "PRIMARY KEY ((id)))")
.compaction(CompactionParams.scts(singletonMap("min_threshold", "2")))
.gcGraceSeconds(0);
@Deprecated
public static final CFMetaData LegacyKeyspaces =
compile(LEGACY_KEYSPACES,
"*DEPRECATED* keyspace definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "durable_writes boolean,"
+ "strategy_class text,"
+ "strategy_options text,"
+ "PRIMARY KEY ((keyspace_name))) "
+ "WITH COMPACT STORAGE");
@Deprecated
public static final CFMetaData LegacyColumnfamilies =
compile(LEGACY_COLUMNFAMILIES,
"*DEPRECATED* table definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "columnfamily_name text,"
+ "bloom_filter_fp_chance double,"
+ "caching text,"
+ "cf_id uuid," // post-2.1 UUID cfid
+ "comment text,"
+ "compaction_strategy_class text,"
+ "compaction_strategy_options text,"
+ "comparator text,"
+ "compression_parameters text,"
+ "default_time_to_live int,"
+ "default_validator text,"
+ "dropped_columns map<text, bigint>,"
+ "gc_grace_seconds int,"
+ "is_dense boolean,"
+ "key_validator text,"
+ "local_read_repair_chance double,"
+ "max_compaction_threshold int,"
+ "max_index_interval int,"
+ "memtable_flush_period_in_ms int,"
+ "min_compaction_threshold int,"
+ "min_index_interval int,"
+ "read_repair_chance double,"
+ "speculative_retry text,"
+ "subcomparator text,"
+ "type text,"
+ "PRIMARY KEY ((keyspace_name), columnfamily_name))");
@Deprecated
public static final CFMetaData LegacyColumns =
compile(LEGACY_COLUMNS,
"*DEPRECATED* column definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "columnfamily_name text,"
+ "column_name text,"
+ "component_index int,"
+ "index_name text,"
+ "index_options text,"
+ "index_type text,"
+ "type text,"
+ "validator text,"
+ "PRIMARY KEY ((keyspace_name), columnfamily_name, column_name))");
@Deprecated
public static final CFMetaData LegacyTriggers =
compile(LEGACY_TRIGGERS,
"*DEPRECATED* trigger definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "columnfamily_name text,"
+ "trigger_name text,"
+ "trigger_options map<text, text>,"
+ "PRIMARY KEY ((keyspace_name), columnfamily_name, trigger_name))");
@Deprecated
public static final CFMetaData LegacyUsertypes =
compile(LEGACY_USERTYPES,
"*DEPRECATED* user defined type definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "type_name text,"
+ "field_names list<text>,"
+ "field_types list<text>,"
+ "PRIMARY KEY ((keyspace_name), type_name))");
@Deprecated
public static final CFMetaData LegacyFunctions =
compile(LEGACY_FUNCTIONS,
"*DEPRECATED* user defined function definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "function_name text,"
+ "signature frozen<list<text>>,"
+ "argument_names list<text>,"
+ "argument_types list<text>,"
+ "body text,"
+ "language text,"
+ "return_type text,"
+ "called_on_null_input boolean,"
+ "PRIMARY KEY ((keyspace_name), function_name, signature))");
@Deprecated
public static final CFMetaData LegacyAggregates =
compile(LEGACY_AGGREGATES,
"*DEPRECATED* user defined aggregate definitions",
"CREATE TABLE %s ("
+ "keyspace_name text,"
+ "aggregate_name text,"
+ "signature frozen<list<text>>,"
+ "argument_types list<text>,"
+ "final_func text,"
+ "initcond blob,"
+ "return_type text,"
+ "state_func text,"
+ "state_type text,"
+ "PRIMARY KEY ((keyspace_name), aggregate_name, signature))");
private static CFMetaData compile(String name, String description, String schema)
{
return CFMetaData.compile(String.format(schema, name), SchemaConstants.SYSTEM_KEYSPACE_NAME)
@ -457,16 +305,7 @@ public final class SystemKeyspace
TransferredRanges,
ViewsBuildsInProgress,
BuiltViews,
LegacyHints,
LegacyBatchlog,
PreparedStatements,
LegacyKeyspaces,
LegacyColumnfamilies,
LegacyColumns,
LegacyTriggers,
LegacyUsertypes,
LegacyFunctions,
LegacyAggregates);
PreparedStatements);
}
private static Functions functions()
@ -1131,18 +970,27 @@ public final class SystemKeyspace
if (results.isEmpty())
return new PaxosState(key, metadata);
UntypedResultSet.Row row = results.one();
// Note: Pre-3.0, we used to not store the versions at which things were serialized. As 3.0 is a mandatory
// upgrade to 4.0+ and the paxos table is TTLed, it's _very_ unlikely we'll ever read a proposal or MRC without
// a version. But if we do (say gc_grace, on which the TTL is based, happens to be super large), we consider
// the commit too old and ignore it.
if (!row.has("proposal_version") || !row.has("most_recent_commit_version"))
return new PaxosState(key, metadata);
Commit promised = row.has("in_progress_ballot")
? new Commit(row.getUUID("in_progress_ballot"), new PartitionUpdate(metadata, key, metadata.partitionColumns(), 1))
: Commit.emptyCommit(key, metadata);
// either we have both a recently accepted ballot and update or we have neither
int proposalVersion = row.has("proposal_version") ? row.getInt("proposal_version") : MessagingService.VERSION_21;
Commit accepted = row.has("proposal")
? new Commit(row.getUUID("proposal_ballot"), PartitionUpdate.fromBytes(row.getBytes("proposal"), proposalVersion, key))
Commit accepted = row.has("proposal_version") && row.has("proposal")
? new Commit(row.getUUID("proposal_ballot"),
PartitionUpdate.fromBytes(row.getBytes("proposal"), row.getInt("proposal_version")))
: Commit.emptyCommit(key, metadata);
// either most_recent_commit and most_recent_commit_at will both be set, or neither
int mostRecentVersion = row.has("most_recent_commit_version") ? row.getInt("most_recent_commit_version") : MessagingService.VERSION_21;
Commit mostRecent = row.has("most_recent_commit")
? new Commit(row.getUUID("most_recent_commit_at"), PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), mostRecentVersion, key))
Commit mostRecent = row.has("most_recent_commit_version") && row.has("most_recent_commit")
? new Commit(row.getUUID("most_recent_commit_at"),
PartitionUpdate.fromBytes(row.getBytes("most_recent_commit"), row.getInt("most_recent_commit_version")))
: Commit.emptyCommit(key, metadata);
return new PaxosState(promised, accepted, mostRecent);
}
@ -1404,45 +1252,17 @@ public final class SystemKeyspace
return result.one().getString("release_version");
}
/**
* Check data directories for old files that can be removed when migrating from 2.1 or 2.2 to 3.0,
* these checks can be removed in 4.0, see CASSANDRA-7066
*/
public static void migrateDataDirs()
{
Iterable<String> dirs = Arrays.asList(DatabaseDescriptor.getAllDataFileLocations());
for (String dataDir : dirs)
{
logger.trace("Checking {} for old files", dataDir);
File dir = new File(dataDir);
assert dir.exists() : dir + " should have been created by startup checks";
for (File ksdir : dir.listFiles((d, n) -> new File(d, n).isDirectory()))
{
logger.trace("Checking {} for old files", ksdir);
for (File cfdir : ksdir.listFiles((d, n) -> new File(d, n).isDirectory()))
{
logger.trace("Checking {} for old files", cfdir);
if (Descriptor.isLegacyFile(cfdir))
{
FileUtils.deleteRecursive(cfdir);
}
else
{
FileUtils.delete(cfdir.listFiles((d, n) -> Descriptor.isLegacyFile(new File(d, n))));
}
}
}
}
}
private static ByteBuffer rangeToBytes(Range<Token> range)
{
try (DataOutputBuffer out = new DataOutputBuffer())
{
Range.tokenSerializer.serialize(range, out, MessagingService.VERSION_22);
// The format with which token ranges are serialized in the system tables is the pre-3.0 serialization
// formot for ranges, so we should maintain that for now. And while we don't really support pre-3.0
// messaging versions, we know AbstractBounds.Serializer still support it _exactly_ for this use case, so we
// pass 0 as the version to trigger that legacy code.
// In the future, it might be worth switching to a stable text format for the ranges to 1) save that and 2)
// be more user friendly (the serialization format we currently use is pretty custom).
Range.tokenSerializer.serialize(range, out, 0);
return out.buffer();
}
catch (IOException e)
@ -1456,9 +1276,10 @@ public final class SystemKeyspace
{
try
{
// See rangeToBytes above for why version is 0.
return (Range<Token>) Range.tokenSerializer.deserialize(ByteStreams.newDataInput(ByteBufferUtil.getArray(rawRange)),
partitioner,
MessagingService.VERSION_22);
0);
}
catch (IOException e)
{

658
src/java/org/apache/cassandra/db/UnfilteredDeserializer.java
View File

@ -39,7 +39,7 @@ import org.apache.cassandra.net.MessagingService;
* we don't do more work than necessary (i.e. we don't allocate/deserialize
* objects for things we don't care about).
*/
public abstract class UnfilteredDeserializer
public class UnfilteredDeserializer
{
private static final Logger logger = LoggerFactory.getLogger(UnfilteredDeserializer.class);
@ -47,32 +47,67 @@ public abstract class UnfilteredDeserializer
protected final DataInputPlus in;
protected final SerializationHelper helper;
protected UnfilteredDeserializer(CFMetaData metadata,
DataInputPlus in,
SerializationHelper helper)
private final ClusteringPrefix.Deserializer clusteringDeserializer;
private final SerializationHeader header;
private int nextFlags;
private int nextExtendedFlags;
private boolean isReady;
private boolean isDone;
private final Row.Builder builder;
private UnfilteredDeserializer(CFMetaData metadata,
DataInputPlus in,
SerializationHeader header,
SerializationHelper helper)
{
this.metadata = metadata;
this.in = in;
this.helper = helper;
this.header = header;
this.clusteringDeserializer = new ClusteringPrefix.Deserializer(metadata.comparator, in, header);
this.builder = BTreeRow.sortedBuilder();
}
public static UnfilteredDeserializer create(CFMetaData metadata,
DataInputPlus in,
SerializationHeader header,
SerializationHelper helper,
DeletionTime partitionDeletion,
boolean readAllAsDynamic)
SerializationHelper helper)
{
if (helper.version >= MessagingService.VERSION_30)
return new CurrentDeserializer(metadata, in, header, helper);
else
return new OldFormatDeserializer(metadata, in, helper, partitionDeletion, readAllAsDynamic);
return new UnfilteredDeserializer(metadata, in, header, helper);
}
/**
* Whether or not there is more atom to read.
*/
public abstract boolean hasNext() throws IOException;
public boolean hasNext() throws IOException
{
if (isReady)
return true;
prepareNext();
return !isDone;
}
private void prepareNext() throws IOException
{
if (isDone)
return;
nextFlags = in.readUnsignedByte();
if (UnfilteredSerializer.isEndOfPartition(nextFlags))
{
isDone = true;
isReady = false;
return;
}
nextExtendedFlags = UnfilteredSerializer.readExtendedFlags(in, nextFlags);
clusteringDeserializer.prepare(nextFlags, nextExtendedFlags);
isReady = true;
}
/**
* Compare the provided bound to the next atom to read on disk.
@ -81,585 +116,68 @@ public abstract class UnfilteredDeserializer
* comparison. Whenever we know what to do with this atom (read it or skip it),
* readNext or skipNext should be called.
*/
public abstract int compareNextTo(ClusteringBound bound) throws IOException;
public int compareNextTo(ClusteringBound bound) throws IOException
{
if (!isReady)
prepareNext();
assert !isDone;
return clusteringDeserializer.compareNextTo(bound);
}
/**
* Returns whether the next atom is a row or not.
*/
public abstract boolean nextIsRow() throws IOException;
public boolean nextIsRow() throws IOException
{
if (!isReady)
prepareNext();
/**
* Returns whether the next atom is the static row or not.
*/
public abstract boolean nextIsStatic() throws IOException;
return UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.ROW;
}
/**
* Returns the next atom.
*/
public abstract Unfiltered readNext() throws IOException;
public Unfiltered readNext() throws IOException
{
isReady = false;
if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
ClusteringBoundOrBoundary bound = clusteringDeserializer.deserializeNextBound();
return UnfilteredSerializer.serializer.deserializeMarkerBody(in, header, bound);
}
else
{
builder.newRow(clusteringDeserializer.deserializeNextClustering());
return UnfilteredSerializer.serializer.deserializeRowBody(in, header, helper, nextFlags, nextExtendedFlags, builder);
}
}
/**
* Clears any state in this deserializer.
*/
public abstract void clearState() throws IOException;
public void clearState()
{
isReady = false;
isDone = false;
}
/**
* Skips the next atom.
*/
public abstract void skipNext() throws IOException;
/**
* For the legacy layout deserializer, we have to deal with the fact that a row can span multiple index blocks and that
* the call to hasNext() reads the next element upfront. We must take that into account when we check in AbstractSSTableIterator if
* we're past the end of an index block boundary as that check expect to account for only consumed data (that is, if hasNext has
* been called and made us cross an index boundary but neither readNext() or skipNext() as yet been called, we shouldn't consider
* the index block boundary crossed yet).
*
* TODO: we don't care about this for the current file format because a row can never span multiple index blocks (further, hasNext()
* only just basically read 2 bytes from disk in that case). So once we drop backward compatibility with pre-3.0 sstable, we should
* remove this.
*/
public abstract long bytesReadForUnconsumedData();
private static class CurrentDeserializer extends UnfilteredDeserializer
public void skipNext() throws IOException
{
private final ClusteringPrefix.Deserializer clusteringDeserializer;
private final SerializationHeader header;
private int nextFlags;
private int nextExtendedFlags;
private boolean isReady;
private boolean isDone;
private final Row.Builder builder;
private CurrentDeserializer(CFMetaData metadata,
DataInputPlus in,
SerializationHeader header,
SerializationHelper helper)
isReady = false;
clusteringDeserializer.skipNext();
if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
super(metadata, in, helper);
this.header = header;
this.clusteringDeserializer = new ClusteringPrefix.Deserializer(metadata.comparator, in, header);
this.builder = BTreeRow.sortedBuilder();
UnfilteredSerializer.serializer.skipMarkerBody(in);
}
public boolean hasNext() throws IOException
else
{
if (isReady)
return true;
prepareNext();
return !isDone;
}
private void prepareNext() throws IOException
{
if (isDone)
return;
nextFlags = in.readUnsignedByte();
if (UnfilteredSerializer.isEndOfPartition(nextFlags))
{
isDone = true;
isReady = false;
return;
}
nextExtendedFlags = UnfilteredSerializer.readExtendedFlags(in, nextFlags);
clusteringDeserializer.prepare(nextFlags, nextExtendedFlags);
isReady = true;
}
public int compareNextTo(ClusteringBound bound) throws IOException
{
if (!isReady)
prepareNext();
assert !isDone;
return clusteringDeserializer.compareNextTo(bound);
}
public boolean nextIsRow() throws IOException
{
if (!isReady)
prepareNext();
return UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.ROW;
}
public boolean nextIsStatic() throws IOException
{
// This exists only for the sake of the OldFormatDeserializer
throw new UnsupportedOperationException();
}
public Unfiltered readNext() throws IOException
{
isReady = false;
if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
ClusteringBoundOrBoundary bound = clusteringDeserializer.deserializeNextBound();
return UnfilteredSerializer.serializer.deserializeMarkerBody(in, header, bound);
}
else
{
builder.newRow(clusteringDeserializer.deserializeNextClustering());
return UnfilteredSerializer.serializer.deserializeRowBody(in, header, helper, nextFlags, nextExtendedFlags, builder);
}
}
public void skipNext() throws IOException
{
isReady = false;
clusteringDeserializer.skipNext();
if (UnfilteredSerializer.kind(nextFlags) == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
{
UnfilteredSerializer.serializer.skipMarkerBody(in);
}
else
{
UnfilteredSerializer.serializer.skipRowBody(in);
}
}
public void clearState()
{
isReady = false;
isDone = false;
}
public long bytesReadForUnconsumedData()
{
// In theory, hasNext() does consume 2-3 bytes, but we don't care about this for the current file format so returning
// 0 to mean "do nothing".
return 0;
}
}
public static class OldFormatDeserializer extends UnfilteredDeserializer
{
private final boolean readAllAsDynamic;
private boolean skipStatic;
// The next Unfiltered to return, computed by hasNext()
private Unfiltered next;
// A temporary storage for an unfiltered that isn't returned next but should be looked at just afterwards
private Unfiltered saved;
private boolean isFirst = true;
// The Unfiltered as read from the old format input
private final UnfilteredIterator iterator;
// The position in the input after the last data consumption (readNext/skipNext).
private long lastConsumedPosition;
private OldFormatDeserializer(CFMetaData metadata,
DataInputPlus in,
SerializationHelper helper,
DeletionTime partitionDeletion,
boolean readAllAsDynamic)
{
super(metadata, in, helper);
this.iterator = new UnfilteredIterator(partitionDeletion);
this.readAllAsDynamic = readAllAsDynamic;
this.lastConsumedPosition = currentPosition();
}
public void setSkipStatic()
{
this.skipStatic = true;
}
private boolean isStatic(Unfiltered unfiltered)
{
return unfiltered.isRow() && ((Row)unfiltered).isStatic();
}
public boolean hasNext() throws IOException
{
try
{
while (next == null)
{
if (saved == null && !iterator.hasNext())
return false;
next = saved == null ? iterator.next() : saved;
saved = null;
// The sstable iterators assume that if there is one, the static row is the first thing this deserializer will return.
// However, in the old format, a range tombstone with an empty start would sort before any static cell. So we should
// detect that case and return the static parts first if necessary.
if (isFirst && iterator.hasNext() && isStatic(iterator.peek()))
{
saved = next;
next = iterator.next();
}
isFirst = false;
// When reading old tables, we sometimes want to skip static data (due to how staticly defined column of compact
// tables are handled).
if (skipStatic && isStatic(next))
next = null;
}
return true;
}
catch (IOError e)
{
if (e.getCause() != null && e.getCause() instanceof IOException)
throw (IOException)e.getCause();
throw e;
}
}
private boolean isRow(LegacyLayout.LegacyAtom atom)
{
if (atom.isCell())
return true;
LegacyLayout.LegacyRangeTombstone tombstone = atom.asRangeTombstone();
return tombstone.isCollectionTombstone() || tombstone.isRowDeletion(metadata);
}
public int compareNextTo(ClusteringBound bound) throws IOException
{
if (!hasNext())
throw new IllegalStateException();
return metadata.comparator.compare(next.clustering(), bound);
}
public boolean nextIsRow() throws IOException
{
if (!hasNext())
throw new IllegalStateException();
return next.isRow();
}
public boolean nextIsStatic() throws IOException
{
return nextIsRow() && ((Row)next).isStatic();
}
private long currentPosition()
{
// We return a bogus value if the input is not file based, but check we never rely
// on that value in that case in bytesReadForUnconsumedData
return in instanceof FileDataInput ? ((FileDataInput)in).getFilePointer() : 0;
}
public Unfiltered readNext() throws IOException
{
if (!hasNext())
throw new IllegalStateException();
Unfiltered toReturn = next;
next = null;
lastConsumedPosition = currentPosition();
return toReturn;
}
public void skipNext() throws IOException
{
if (!hasNext())
throw new UnsupportedOperationException();
next = null;
lastConsumedPosition = currentPosition();
}
public long bytesReadForUnconsumedData()
{
if (!(in instanceof FileDataInput))
throw new AssertionError();
return currentPosition() - lastConsumedPosition;
}
public void clearState()
{
next = null;
saved = null;
iterator.clearState();
lastConsumedPosition = currentPosition();
}
// Groups atoms from the input into proper Unfiltered.
// Note: this could use guava AbstractIterator except that we want to be able to clear
// the internal state of the iterator so it's cleaner to do it ourselves.
private class UnfilteredIterator implements PeekingIterator<Unfiltered>
{
private final AtomIterator atoms;
private final LegacyLayout.CellGrouper grouper;
private final TombstoneTracker tombstoneTracker;
private Unfiltered next;
private UnfilteredIterator(DeletionTime partitionDeletion)
{
this.grouper = new LegacyLayout.CellGrouper(metadata, helper);
this.tombstoneTracker = new TombstoneTracker(partitionDeletion);
this.atoms = new AtomIterator();
}
public boolean hasNext()
{
// Note that we loop on next == null because TombstoneTracker.openNew() could return null below or the atom might be shadowed.
while (next == null)
{
if (atoms.hasNext())
{
// If a range tombstone closes strictly before the next row/RT, we need to return that close (or boundary) marker first.
if (tombstoneTracker.hasClosingMarkerBefore(atoms.peek()))
{
next = tombstoneTracker.popClosingMarker();
}
else
{
LegacyLayout.LegacyAtom atom = atoms.next();
if (!tombstoneTracker.isShadowed(atom))
next = isRow(atom) ? readRow(atom) : tombstoneTracker.openNew(atom.asRangeTombstone());
}
}
else if (tombstoneTracker.hasOpenTombstones())
{
next = tombstoneTracker.popClosingMarker();
}
else
{
return false;
}
}
return true;
}
private Unfiltered readRow(LegacyLayout.LegacyAtom first)
{
LegacyLayout.CellGrouper grouper = first.isStatic()
? LegacyLayout.CellGrouper.staticGrouper(metadata, helper)
: this.grouper;
grouper.reset();
grouper.addAtom(first);
// As long as atoms are part of the same row, consume them. Note that the call to addAtom() uses
// atoms.peek() so that the atom is only consumed (by next) if it's part of the row (addAtom returns true)
while (atoms.hasNext() && grouper.addAtom(atoms.peek()))
{
atoms.next();
}
return grouper.getRow();
}
public Unfiltered next()
{
if (!hasNext())
throw new UnsupportedOperationException();
Unfiltered toReturn = next;
next = null;
return toReturn;
}
public Unfiltered peek()
{
if (!hasNext())
throw new UnsupportedOperationException();
return next;
}
public void clearState()
{
atoms.clearState();
tombstoneTracker.clearState();
next = null;
}
public void remove()
{
throw new UnsupportedOperationException();
}
}
// Wraps the input of the deserializer to provide an iterator (and skip shadowed atoms).
// Note: this could use guava AbstractIterator except that we want to be able to clear
// the internal state of the iterator so it's cleaner to do it ourselves.
private class AtomIterator implements PeekingIterator<LegacyLayout.LegacyAtom>
{
private boolean isDone;
private LegacyLayout.LegacyAtom next;
private AtomIterator()
{
}
public boolean hasNext()
{
if (isDone)
return false;
if (next == null)
{
next = readAtom();
if (next == null)
{
isDone = true;
return false;
}
}
return true;
}
private LegacyLayout.LegacyAtom readAtom()
{
try
{
return LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic);
}
catch (IOException e)
{
throw new IOError(e);
}
}
public LegacyLayout.LegacyAtom next()
{
if (!hasNext())
throw new UnsupportedOperationException();
LegacyLayout.LegacyAtom toReturn = next;
next = null;
return toReturn;
}
public LegacyLayout.LegacyAtom peek()
{
if (!hasNext())
throw new UnsupportedOperationException();
return next;
}
public void clearState()
{
this.next = null;
this.isDone = false;
}
public void remove()
{
throw new UnsupportedOperationException();
}
}
/**
* Tracks which range tombstones are open when deserializing the old format.
*/
private class TombstoneTracker
{
private final DeletionTime partitionDeletion;
// Open tombstones sorted by their closing bound (i.e. first tombstone is the first to close).
// As we only track non-fully-shadowed ranges, the first range is necessarily the currently
// open tombstone (the one with the higher timestamp).
private final SortedSet<LegacyLayout.LegacyRangeTombstone> openTombstones;
public TombstoneTracker(DeletionTime partitionDeletion)
{
this.partitionDeletion = partitionDeletion;
this.openTombstones = new TreeSet<>((rt1, rt2) -> metadata.comparator.compare(rt1.stop.bound, rt2.stop.bound));
}
/**
* Checks if the provided atom is fully shadowed by the open tombstones of this tracker (or the partition deletion).
*/
public boolean isShadowed(LegacyLayout.LegacyAtom atom)
{
assert !hasClosingMarkerBefore(atom);
long timestamp = atom.isCell() ? atom.asCell().timestamp : atom.asRangeTombstone().deletionTime.markedForDeleteAt();
if (partitionDeletion.deletes(timestamp))
return true;
SortedSet<LegacyLayout.LegacyRangeTombstone> coveringTombstones = isRow(atom) ? openTombstones : openTombstones.tailSet(atom.asRangeTombstone());
return Iterables.any(coveringTombstones, tombstone -> tombstone.deletionTime.deletes(timestamp));
}
/**
* Whether the currently open marker closes stricly before the provided row/RT.
*/
public boolean hasClosingMarkerBefore(LegacyLayout.LegacyAtom atom)
{
return !openTombstones.isEmpty()
&& metadata.comparator.compare(openTombstones.first().stop.bound, atom.clustering()) < 0;
}
/**
* Returns the unfiltered corresponding to closing the currently open marker (and update the tracker accordingly).
*/
public Unfiltered popClosingMarker()
{
assert !openTombstones.isEmpty();
Iterator<LegacyLayout.LegacyRangeTombstone> iter = openTombstones.iterator();
LegacyLayout.LegacyRangeTombstone first = iter.next();
iter.remove();
// If that was the last open tombstone, we just want to close it. Otherwise, we have a boundary with the
// next tombstone
if (!iter.hasNext())
return new RangeTombstoneBoundMarker(first.stop.bound, first.deletionTime);
LegacyLayout.LegacyRangeTombstone next = iter.next();
return RangeTombstoneBoundaryMarker.makeBoundary(false, first.stop.bound, first.stop.bound.invert(), first.deletionTime, next.deletionTime);
}
/**
* Update the tracker given the provided newly open tombstone. This return the Unfiltered corresponding to the opening
* of said tombstone: this can be a simple open mark, a boundary (if there was an open tombstone superseded by this new one)
* or even null (if the new tombston start is supersedes by the currently open tombstone).
*
* Note that this method assume the added tombstone is not fully shadowed, i.e. that !isShadowed(tombstone). It also
* assumes no opened tombstone closes before that tombstone (so !hasClosingMarkerBefore(tombstone)).
*/
public Unfiltered openNew(LegacyLayout.LegacyRangeTombstone tombstone)
{
if (openTombstones.isEmpty())
{
openTombstones.add(tombstone);
return new RangeTombstoneBoundMarker(tombstone.start.bound, tombstone.deletionTime);
}
Iterator<LegacyLayout.LegacyRangeTombstone> iter = openTombstones.iterator();
LegacyLayout.LegacyRangeTombstone first = iter.next();
if (tombstone.deletionTime.supersedes(first.deletionTime))
{
// We're supperseding the currently open tombstone, so we should produce a boundary that close the currently open
// one and open the new one. We should also add the tombstone, but if it stop after the first one, we should
// also remove that first tombstone as it won't be useful anymore.
if (metadata.comparator.compare(tombstone.stop.bound, first.stop.bound) >= 0)
iter.remove();
openTombstones.add(tombstone);
return RangeTombstoneBoundaryMarker.makeBoundary(false, tombstone.start.bound.invert(), tombstone.start.bound, first.deletionTime, tombstone.deletionTime);
}
else
{
// If the new tombstone don't supersedes the currently open tombstone, we don't have anything to return, we
// just add the new tombstone (because we know tombstone is not fully shadowed, this imply the new tombstone
// simply extend after the first one and we'll deal with it later)
assert metadata.comparator.compare(tombstone.start.bound, first.stop.bound) > 0;
openTombstones.add(tombstone);
return null;
}
}
public boolean hasOpenTombstones()
{
return !openTombstones.isEmpty();
}
public void clearState()
{
openTombstones.clear();
}
UnfilteredSerializer.serializer.skipRowBody(in);
}
}
}

44
src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java
View File

@ -89,12 +89,6 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
// - we're querying static columns.
boolean needSeekAtPartitionStart = !indexEntry.isIndexed() || !columns.fetchedColumns().statics.isEmpty();
// For CQL queries on static compact tables, we only want to consider static value (only those are exposed),
// but readStaticRow have already read them and might in fact have consumed the whole partition (when reading
// the legacy file format), so set the reader to null so we don't try to read anything more. We can remove this
// once we drop support for the legacy file format
boolean needsReader = sstable.descriptor.version.storeRows() || isForThrift || !sstable.metadata.isStaticCompactTable();
if (needSeekAtPartitionStart)
{
// Not indexed (or is reading static), set to the beginning of the partition and read partition level deletion there
@ -108,14 +102,14 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
// Note that this needs to be called after file != null and after the partitionDeletion has been set, but before readStaticRow
// (since it uses it) so we can't move that up (but we'll be able to simplify as soon as we drop support for the old file format).
this.reader = needsReader ? createReader(indexEntry, file, shouldCloseFile) : null;
this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics, isForThrift, reader == null ? null : reader.deserializer);
this.reader = createReader(indexEntry, file, shouldCloseFile);
this.staticRow = readStaticRow(sstable, file, helper, columns.fetchedColumns().statics, isForThrift, reader.deserializer);
}
else
{
this.partitionLevelDeletion = indexEntry.deletionTime();
this.staticRow = Rows.EMPTY_STATIC_ROW;
this.reader = needsReader ? createReader(indexEntry, file, shouldCloseFile) : null;
this.reader = createReader(indexEntry, file, shouldCloseFile);
}
if (reader != null && !slices.isEmpty())
@ -168,33 +162,6 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
boolean isForThrift,
UnfilteredDeserializer deserializer) throws IOException
{
if (!sstable.descriptor.version.storeRows())
{
if (!sstable.metadata.isCompactTable())
{
assert deserializer != null;
return deserializer.hasNext() && deserializer.nextIsStatic()
? (Row)deserializer.readNext()
: Rows.EMPTY_STATIC_ROW;
}
// For compact tables, we use statics for the "column_metadata" definition. However, in the old format, those
// "column_metadata" are intermingled as any other "cell". In theory, this means that we'd have to do a first
// pass to extract the static values. However, for thrift, we'll use the ThriftResultsMerger right away which
// will re-merge static values with dynamic ones, so we can just ignore static and read every cell as a
// "dynamic" one. For CQL, if the table is a "static compact", then is has only static columns exposed and no
// dynamic ones. So we do a pass to extract static columns here, but will have no more work to do. Otherwise,
// the table won't have static columns.
if (statics.isEmpty() || isForThrift)
return Rows.EMPTY_STATIC_ROW;
assert sstable.metadata.isStaticCompactTable();
// As said above, if it's a CQL query and the table is a "static compact", the only exposed columns are the
// static ones. So we don't have to mark the position to seek back later.
return LegacyLayout.extractStaticColumns(sstable.metadata, file, statics);
}
if (!sstable.header.hasStatic())
return Rows.EMPTY_STATIC_ROW;
@ -345,7 +312,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
private void createDeserializer()
{
assert file != null && deserializer == null;
deserializer = UnfilteredDeserializer.create(sstable.metadata, file, sstable.header, helper, partitionLevelDeletion, isForThrift);
deserializer = UnfilteredDeserializer.create(sstable.metadata, file, sstable.header, helper);
}
protected void seekToPosition(long position) throws IOException
@ -550,8 +517,7 @@ public abstract class AbstractSSTableIterator implements UnfilteredRowIterator
public boolean isPastCurrentBlock() throws IOException
{
assert reader.deserializer != null;
long correction = reader.deserializer.bytesReadForUnconsumedData();
return reader.file.bytesPastMark(mark) - correction >= currentIndex().width;
return reader.file.bytesPastMark(mark) >= currentIndex().width;
}
public int currentBlockIdx()

6
src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java
View File

@ -257,12 +257,10 @@ public class SSTableIterator extends AbstractSSTableIterator
// so if currentIdx == lastBlockIdx and slice.end < indexes[currentIdx].firstName, we're guaranteed that the
// whole slice is between the previous block end and this block start, and thus has no corresponding
// data. One exception is if the previous block ends with an openMarker as it will cover our slice
// and we need to return it (we also don't skip the slice for the old format because we didn't have the openMarker
// info in that case and can't rely on this optimization).
// and we need to return it.
if (indexState.currentBlockIdx() == lastBlockIdx
&& metadata().comparator.compare(slice.end(), indexState.currentIndex().firstName) < 0
&& openMarker == null
&& sstable.descriptor.version.storeRows())
&& openMarker == null)
{
sliceDone = true;
}

18
src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java
View File

@ -310,23 +310,7 @@ public class SSTableReversedIterator extends AbstractSSTableIterator
int currentBlock = indexState.currentBlockIdx();
boolean canIncludeSliceStart = currentBlock == lastBlockIdx;
// When dealing with old format sstable, we have the problem that a row can span 2 index block, i.e. it can
// start at the end of a block and end at the beginning of the next one. That's not a problem per se for
// UnfilteredDeserializer.OldFormatSerializer, since it always read rows entirely, even if they span index
// blocks, but as we reading index block in reverse we must be careful to not read the end of the row at
// beginning of a block before we're reading the beginning of that row. So what we do is that if we detect
// that the row starting this block is also the row ending the previous one, we skip that first result and
// let it be read when we'll read the previous block.
boolean includeFirst = true;
if (!sstable.descriptor.version.storeRows() && currentBlock > 0)
{
ClusteringPrefix lastOfPrevious = indexState.index(currentBlock - 1).lastName;
ClusteringPrefix firstOfCurrent = indexState.index(currentBlock).firstName;
includeFirst = metadata().comparator.compare(lastOfPrevious, firstOfCurrent) != 0;
}
loadFromDisk(canIncludeSliceStart ? slice.start() : null, canIncludeSliceEnd ? slice.end() : null, includeFirst);
loadFromDisk(canIncludeSliceStart ? slice.start() : null, canIncludeSliceEnd ? slice.end() : null, true);
}
@Override

2
src/java/org/apache/cassandra/db/commitlog/CommitLogArchiver.java
View File

@ -233,7 +233,7 @@ public class CommitLogArchiver
throw new IllegalStateException("Cannot safely construct descriptor for segment, either from its name or its header: " + fromFile.getPath());
else if (fromHeader != null && fromName != null && !fromHeader.equalsIgnoringCompression(fromName))
throw new IllegalStateException(String.format("Cannot safely construct descriptor for segment, as name and header descriptors do not match (%s vs %s): %s", fromHeader, fromName, fromFile.getPath()));
else if (fromName != null && fromHeader == null && fromName.version >= CommitLogDescriptor.VERSION_21)
else if (fromName != null && fromHeader == null)
throw new IllegalStateException("Cannot safely construct descriptor for segment, as name descriptor implies a version that should contain a header descriptor, but that descriptor could not be read: " + fromFile.getPath());
else if (fromHeader != null)
descriptor = fromHeader;

47
src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java
View File

@ -57,10 +57,7 @@ public class CommitLogDescriptor
static final String COMPRESSION_PARAMETERS_KEY = "compressionParameters";
static final String COMPRESSION_CLASS_KEY = "compressionClass";
public static final int VERSION_12 = 2;
public static final int VERSION_20 = 3;
public static final int VERSION_21 = 4;
public static final int VERSION_22 = 5;
// We don't support anything pre-3.0
public static final int VERSION_30 = 6;
/**
@ -104,20 +101,15 @@ public class CommitLogDescriptor
out.putLong(descriptor.id);
updateChecksumInt(crc, (int) (descriptor.id & 0xFFFFFFFFL));
updateChecksumInt(crc, (int) (descriptor.id >>> 32));
if (descriptor.version >= VERSION_22)
{
String parametersString = constructParametersString(descriptor.compression, descriptor.encryptionContext, additionalHeaders);
byte[] parametersBytes = parametersString.getBytes(StandardCharsets.UTF_8);
if (parametersBytes.length != (((short) parametersBytes.length) & 0xFFFF))
throw new ConfigurationException(String.format("Compression parameters too long, length %d cannot be above 65535.",
parametersBytes.length));
out.putShort((short) parametersBytes.length);
updateChecksumInt(crc, parametersBytes.length);
out.put(parametersBytes);
crc.update(parametersBytes, 0, parametersBytes.length);
}
else
assert descriptor.compression == null;
String parametersString = constructParametersString(descriptor.compression, descriptor.encryptionContext, additionalHeaders);
byte[] parametersBytes = parametersString.getBytes(StandardCharsets.UTF_8);
if (parametersBytes.length != (((short) parametersBytes.length) & 0xFFFF))
throw new ConfigurationException(String.format("Compression parameters too long, length %d cannot be above 65535.",
parametersBytes.length));
out.putShort((short) parametersBytes.length);
updateChecksumInt(crc, parametersBytes.length);
out.put(parametersBytes);
crc.update(parametersBytes, 0, parametersBytes.length);
out.putInt((int) crc.getValue());
}
@ -157,16 +149,15 @@ public class CommitLogDescriptor
{
CRC32 checkcrc = new CRC32();
int version = input.readInt();
if (version < VERSION_30)
throw new IllegalArgumentException("Unsupported pre-3.0 commit log found; cannot read.");
updateChecksumInt(checkcrc, version);
long id = input.readLong();
updateChecksumInt(checkcrc, (int) (id & 0xFFFFFFFFL));
updateChecksumInt(checkcrc, (int) (id >>> 32));
int parametersLength = 0;
if (version >= VERSION_22)
{
parametersLength = input.readShort() & 0xFFFF;
updateChecksumInt(checkcrc, parametersLength);
}
int parametersLength = input.readShort() & 0xFFFF;
updateChecksumInt(checkcrc, parametersLength);
// This should always succeed as parametersLength cannot be too long even for a
// corrupt segment file.
byte[] parametersBytes = new byte[parametersLength];
@ -213,14 +204,6 @@ public class CommitLogDescriptor
{
switch (version)
{
case VERSION_12:
return MessagingService.VERSION_12;
case VERSION_20:
return MessagingService.VERSION_20;
case VERSION_21:
return MessagingService.VERSION_21;
case VERSION_22:
return MessagingService.VERSION_22;
case VERSION_30:
return MessagingService.VERSION_30;
default:

44
src/java/org/apache/cassandra/db/commitlog/CommitLogReader.java
View File

@ -122,19 +122,6 @@ public class CommitLogReader
try(RandomAccessReader reader = RandomAccessReader.open(file))
{
if (desc.version < CommitLogDescriptor.VERSION_21)
{
if (!shouldSkipSegmentId(file, desc, minPosition))
{
if (minPosition.segmentId == desc.id)
reader.seek(minPosition.position);
ReadStatusTracker statusTracker = new ReadStatusTracker(mutationLimit, tolerateTruncation);
statusTracker.errorContext = desc.fileName();
readSection(handler, reader, minPosition, (int) reader.length(), statusTracker, desc);
}
return;
}
final long segmentIdFromFilename = desc.id;
try
{
@ -430,42 +417,17 @@ public class CommitLogReader
{
public static long calculateClaimedChecksum(FileDataInput input, int commitLogVersion) throws IOException
{
switch (commitLogVersion)
{
case CommitLogDescriptor.VERSION_12:
case CommitLogDescriptor.VERSION_20:
return input.readLong();
// Changed format in 2.1
default:
return input.readInt() & 0xffffffffL;
}
return input.readInt() & 0xffffffffL;
}
public static void updateChecksum(CRC32 checksum, int serializedSize, int commitLogVersion)
{
switch (commitLogVersion)
{
case CommitLogDescriptor.VERSION_12:
checksum.update(serializedSize);
break;
// Changed format in 2.0
default:
updateChecksumInt(checksum, serializedSize);
break;
}
updateChecksumInt(checksum, serializedSize);
}
public static long calculateClaimedCRC32(FileDataInput input, int commitLogVersion) throws IOException
{
switch (commitLogVersion)
{
case CommitLogDescriptor.VERSION_12:
case CommitLogDescriptor.VERSION_20:
return input.readLong();
// Changed format in 2.1
default:
return input.readInt() & 0xffffffffL;
}
return input.readInt() & 0xffffffffL;
}
}

4
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -1242,7 +1242,7 @@ public class CompactionManager implements CompactionManagerMBean
header = SerializationHeader.make(sstable.metadata, Collections.singleton(sstable));
return SSTableWriter.create(cfs.metadata,
Descriptor.fromFilename(cfs.getSSTablePath(compactionFileLocation)),
cfs.newSSTableDescriptor(compactionFileLocation),
expectedBloomFilterSize,
repairedAt,
sstable.getSSTableLevel(),
@ -1274,7 +1274,7 @@ public class CompactionManager implements CompactionManagerMBean
break;
}
}
return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(compactionFileLocation)),
return SSTableWriter.create(cfs.newSSTableDescriptor(compactionFileLocation),
(long) expectedBloomFilterSize,
repairedAt,
cfs.metadata,

2
src/java/org/apache/cassandra/db/compaction/Upgrader.java
View File

@ -70,7 +70,7 @@ public class Upgrader
{
MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(directory)),
return SSTableWriter.create(cfs.newSSTableDescriptor(directory),
estimatedRows,
repairedAt,
cfs.metadata,

3
src/java/org/apache/cassandra/db/compaction/Verifier.java
View File

@ -97,8 +97,7 @@ public class Verifier implements Closeable
{
validator = null;
if (sstable.descriptor.digestComponent != null &&
new File(sstable.descriptor.filenameFor(sstable.descriptor.digestComponent)).exists())
if (new File(sstable.descriptor.filenameFor(Component.DIGEST)).exists())
{
validator = DataIntegrityMetadata.fileDigestValidator(sstable.descriptor);
validator.validate();

2
src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
View File

@ -69,7 +69,7 @@ public class DefaultCompactionWriter extends CompactionAwareWriter
public void switchCompactionLocation(Directories.DataDirectory directory)
{
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(directory))),
SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory)),
estimatedTotalKeys,
minRepairedAt,
cfs.metadata,

2
src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
View File

@ -105,7 +105,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
{
this.sstableDirectory = location;
averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
sstableWriter.switchWriter(SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(sstableDirectory))),
sstableWriter.switchWriter(SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
keysPerSSTable,
minRepairedAt,
cfs.metadata,

2
src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
View File

@ -108,7 +108,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
{
sstableDirectory = location;
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(sstableDirectory))),
SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory)),
estimatedTotalKeys / estimatedSSTables,
minRepairedAt,
cfs.metadata,

2
src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
View File

@ -104,7 +104,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
this.location = location;
long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
@SuppressWarnings("resource")
SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
SSTableWriter writer = SSTableWriter.create(cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location)),
currentPartitionsToWrite,
minRepairedAt,
cfs.metadata,

103
src/java/org/apache/cassandra/db/filter/RowFilter.java
View File

@ -509,16 +509,12 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
{
public void serialize(Expression expression, DataOutputPlus out, int version) throws IOException
{
if (version >= MessagingService.VERSION_30)
out.writeByte(expression.kind().ordinal());
out.writeByte(expression.kind().ordinal());
// Custom expressions include neither a column or operator, but all
// other expressions do. Also, custom expressions are 3.0+ only, so
// the column & operator will always be the first things written for
// any pre-3.0 version
// other expressions do.
if (expression.kind() == Kind.CUSTOM)
{
assert version >= MessagingService.VERSION_30;
IndexMetadata.serializer.serialize(((CustomExpression)expression).targetIndex, out, version);
ByteBufferUtil.writeWithShortLength(expression.value, out);
return;
@ -526,7 +522,6 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
if (expression.kind() == Kind.USER)
{
assert version >= MessagingService.VERSION_30;
UserExpression.serialize((UserExpression)expression, out, version);
return;
}
@ -541,15 +536,8 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
break;
case MAP_EQUALITY:
MapEqualityExpression mexpr = (MapEqualityExpression)expression;
if (version < MessagingService.VERSION_30)
{
ByteBufferUtil.writeWithShortLength(mexpr.getIndexValue(), out);
}
else
{
ByteBufferUtil.writeWithShortLength(mexpr.key, out);
ByteBufferUtil.writeWithShortLength(mexpr.value, out);
}
ByteBufferUtil.writeWithShortLength(mexpr.key, out);
ByteBufferUtil.writeWithShortLength(mexpr.value, out);
break;
case THRIFT_DYN_EXPR:
ByteBufferUtil.writeWithShortLength(((ThriftExpression)expression).value, out);
@ -559,62 +547,33 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
public Expression deserialize(DataInputPlus in, int version, CFMetaData metadata) throws IOException
{
Kind kind = null;
ByteBuffer name;
Operator operator;
ColumnDefinition column;
Kind kind = Kind.values()[in.readByte()];
if (version >= MessagingService.VERSION_30)
// custom expressions (3.0+ only) do not contain a column or operator, only a value
if (kind == Kind.CUSTOM)
{
kind = Kind.values()[in.readByte()];
// custom expressions (3.0+ only) do not contain a column or operator, only a value
if (kind == Kind.CUSTOM)
{
return new CustomExpression(metadata,
IndexMetadata.serializer.deserialize(in, version, metadata),
ByteBufferUtil.readWithShortLength(in));
}
if (kind == Kind.USER)
{
return UserExpression.deserialize(in, version, metadata);
}
return new CustomExpression(metadata,
IndexMetadata.serializer.deserialize(in, version, metadata),
ByteBufferUtil.readWithShortLength(in));
}
name = ByteBufferUtil.readWithShortLength(in);
operator = Operator.readFrom(in);
column = metadata.getColumnDefinition(name);
if (kind == Kind.USER)
return UserExpression.deserialize(in, version, metadata);
ByteBuffer name = ByteBufferUtil.readWithShortLength(in);
Operator operator = Operator.readFrom(in);
ColumnDefinition column = metadata.getColumnDefinition(name);
if (!metadata.isCompactTable() && column == null)
throw new RuntimeException("Unknown (or dropped) column " + UTF8Type.instance.getString(name) + " during deserialization");
if (version < MessagingService.VERSION_30)
{
if (column == null)
kind = Kind.THRIFT_DYN_EXPR;
else if (column.type instanceof MapType && operator == Operator.EQ)
kind = Kind.MAP_EQUALITY;
else
kind = Kind.SIMPLE;
}
assert kind != null;
switch (kind)
{
case SIMPLE:
return new SimpleExpression(column, operator, ByteBufferUtil.readWithShortLength(in));
case MAP_EQUALITY:
ByteBuffer key, value;
if (version < MessagingService.VERSION_30)
{
ByteBuffer composite = ByteBufferUtil.readWithShortLength(in);
key = CompositeType.extractComponent(composite, 0);
value = CompositeType.extractComponent(composite, 0);
}
else
{
key = ByteBufferUtil.readWithShortLength(in);
value = ByteBufferUtil.readWithShortLength(in);
}
ByteBuffer key = ByteBufferUtil.readWithShortLength(in);
ByteBuffer value = ByteBufferUtil.readWithShortLength(in);
return new MapEqualityExpression(column, key, operator, value);
case THRIFT_DYN_EXPR:
return new ThriftExpression(metadata, name, operator, ByteBufferUtil.readWithShortLength(in));
@ -622,16 +581,12 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
throw new AssertionError();
}
public long serializedSize(Expression expression, int version)
{
// version 3.0+ includes a byte for Kind
long size = version >= MessagingService.VERSION_30 ? 1 : 0;
long size = 1; // kind byte
// Custom expressions include neither a column or operator, but all
// other expressions do. Also, custom expressions are 3.0+ only, so
// the column & operator will always be the first things written for
// any pre-3.0 version
// other expressions do.
if (expression.kind() != Kind.CUSTOM && expression.kind() != Kind.USER)
size += ByteBufferUtil.serializedSizeWithShortLength(expression.column().name.bytes)
+ expression.operator.serializedSize();
@ -643,23 +598,19 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
break;
case MAP_EQUALITY:
MapEqualityExpression mexpr = (MapEqualityExpression)expression;
if (version < MessagingService.VERSION_30)
size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.getIndexValue());
else
size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
+ ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
size += ByteBufferUtil.serializedSizeWithShortLength(mexpr.key)
+ ByteBufferUtil.serializedSizeWithShortLength(mexpr.value);
break;
case THRIFT_DYN_EXPR:
size += ByteBufferUtil.serializedSizeWithShortLength(((ThriftExpression)expression).value);
break;
case CUSTOM:
if (version >= MessagingService.VERSION_30)
size += IndexMetadata.serializer.serializedSize(((CustomExpression)expression).targetIndex, version)
+ ByteBufferUtil.serializedSizeWithShortLength(expression.value);
size += IndexMetadata.serializer.serializedSize(((CustomExpression)expression).targetIndex, version)
+ ByteBufferUtil.serializedSizeWithShortLength(expression.value);
break;
case USER:
if (version >= MessagingService.VERSION_30)
size += UserExpression.serializedSize((UserExpression)expression, version);
size += UserExpression.serializedSize((UserExpression)expression, version);
break;
}
return size;
}

60
src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java
View File

@ -238,12 +238,10 @@ public class PartitionUpdate extends AbstractBTreePartition
*
* @param bytes the byte buffer that contains the serialized update.
* @param version the version with which the update is serialized.
* @param key the partition key for the update. This is only used if {@code version &lt 3.0}
* and can be {@code null} otherwise.
*
* @return the deserialized update or {@code null} if {@code bytes == null}.
*/
public static PartitionUpdate fromBytes(ByteBuffer bytes, int version, DecoratedKey key)
public static PartitionUpdate fromBytes(ByteBuffer bytes, int version)
{
if (bytes == null)
return null;
@ -252,8 +250,7 @@ public class PartitionUpdate extends AbstractBTreePartition
{
return serializer.deserialize(new DataInputBuffer(bytes, true),
version,
SerializationHelper.Flag.LOCAL,
version < MessagingService.VERSION_30 ? key : null);
SerializationHelper.Flag.LOCAL);
}
catch (IOException e)
{
@ -780,47 +777,12 @@ public class PartitionUpdate extends AbstractBTreePartition
{
assert !iter.isReverseOrder();
if (version < MessagingService.VERSION_30)
{
LegacyLayout.serializeAsLegacyPartition(null, iter, out, version);
}
else
{
CFMetaData.serializer.serialize(update.metadata(), out, version);
UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
}
CFMetaData.serializer.serialize(update.metadata(), out, version);
UnfilteredRowIteratorSerializer.serializer.serialize(iter, null, out, version, update.rowCount());
}
}
public PartitionUpdate deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
{
if (version >= MessagingService.VERSION_30)
{
assert key == null; // key is only there for the old format
return deserialize30(in, version, flag);
}
else
{
assert key != null;
return deserializePre30(in, version, flag, key);
}
}
// Used to share same decorated key between updates.
public PartitionUpdate deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag, DecoratedKey key) throws IOException
{
if (version >= MessagingService.VERSION_30)
{
return deserialize30(in, version, flag);
}
else
{
assert key != null;
return deserializePre30(in, version, flag, key.getKey());
}
}
private static PartitionUpdate deserialize30(DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
public PartitionUpdate deserialize(DataInputPlus in, int version, SerializationHelper.Flag flag) throws IOException
{
CFMetaData metadata = CFMetaData.serializer.deserialize(in, version);
UnfilteredRowIteratorSerializer.Header header = UnfilteredRowIteratorSerializer.serializer.deserializeHeader(metadata, null, in, version, flag);
@ -854,22 +816,10 @@ public class PartitionUpdate extends AbstractBTreePartition
false);
}
private static PartitionUpdate deserializePre30(DataInputPlus in, int version, SerializationHelper.Flag flag, ByteBuffer key) throws IOException
{
try (UnfilteredRowIterator iterator = LegacyLayout.deserializeLegacyPartition(in, version, flag, key))
{
assert iterator != null; // This is only used in mutation, and mutation have never allowed "null" column families
return PartitionUpdate.fromIterator(iterator, ColumnFilter.all(iterator.metadata()));
}
}
public long serializedSize(PartitionUpdate update, int version)
{
try (UnfilteredRowIterator iter = update.unfilteredIterator())
{
if (version < MessagingService.VERSION_30)
return LegacyLayout.serializedSizeAsLegacyPartition(null, iter, version);
return CFMetaData.serializer.serializedSize(update.metadata(), version)
+ UnfilteredRowIteratorSerializer.serializer.serializedSize(iter, null, version, update.rowCount());
}

9
src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
View File

@ -252,13 +252,11 @@ public abstract class UnfilteredPartitionIterators
/**
* Digests the the provided iterator.
*
* @param command the command that has yield {@code iterator}. This can be null if {@code version >= MessagingService.VERSION_30}
* as this is only used when producing digest to be sent to legacy nodes.
* @param iterator the iterator to digest.
* @param digest the {@code MessageDigest} to use for the digest.
* @param version the messaging protocol to use when producing the digest.
*/
public static void digest(ReadCommand command, UnfilteredPartitionIterator iterator, MessageDigest digest, int version)
public static void digest(UnfilteredPartitionIterator iterator, MessageDigest digest, int version)
{
try (UnfilteredPartitionIterator iter = iterator)
{
@ -266,7 +264,7 @@ public abstract class UnfilteredPartitionIterators
{
try (UnfilteredRowIterator partition = iter.next())
{
UnfilteredRowIterators.digest(command, partition, digest, version);
UnfilteredRowIterators.digest(partition, digest, version);
}
}
}
@ -303,8 +301,6 @@ public abstract class UnfilteredPartitionIterators
{
public void serialize(UnfilteredPartitionIterator iter, ColumnFilter selection, DataOutputPlus out, int version) throws IOException
{
assert version >= MessagingService.VERSION_30; // We handle backward compatibility directy in ReadResponse.LegacyRangeSliceReplySerializer
out.writeBoolean(iter.isForThrift());
while (iter.hasNext())
{
@ -319,7 +315,6 @@ public abstract class UnfilteredPartitionIterators
public UnfilteredPartitionIterator deserialize(final DataInputPlus in, final int version, final CFMetaData metadata, final ColumnFilter selection, final SerializationHelper.Flag flag) throws IOException
{
assert version >= MessagingService.VERSION_30; // We handle backward compatibility directy in ReadResponse.LegacyRangeSliceReplySerializer
final boolean isForThrift = in.readBoolean();
return new AbstractUnfilteredPartitionIterator()

5
src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java
View File

@ -211,12 +211,11 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt
/**
* @return true if we can use the clustering values in the stats of the sstable:
* - we need the latest stats file format (or else the clustering values create clusterings with the wrong size)
* - we cannot create tombstone bounds from these values only and so we rule out sstables with tombstones
* we cannot create tombstone bounds from these values only and so we rule out sstables with tombstones
*/
private boolean canUseMetadataLowerBound()
{
return !sstable.hasTombstones() && sstable.descriptor.version.hasNewStatsFile();
return !sstable.hasTombstones();
}
/**

10
src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
View File

@ -143,20 +143,12 @@ public abstract class UnfilteredRowIterators
/**
* Digests the partition represented by the provided iterator.
*
* @param command the command that has yield {@code iterator}. This can be null if {@code version >= MessagingService.VERSION_30}
* as this is only used when producing digest to be sent to legacy nodes.
* @param iterator the iterator to digest.
* @param digest the {@code MessageDigest} to use for the digest.
* @param version the messaging protocol to use when producing the digest.
*/
public static void digest(ReadCommand command, UnfilteredRowIterator iterator, MessageDigest digest, int version)
public static void digest(UnfilteredRowIterator iterator, MessageDigest digest, int version)
{
if (version < MessagingService.VERSION_30)
{
LegacyLayout.fromUnfilteredRowIterator(command, iterator).digest(iterator.metadata(), digest);
return;
}
digest.update(iterator.partitionKey().getKey().duplicate());
iterator.partitionLevelDeletion().digest(digest);
iterator.columns().regulars.digest(digest);

5
src/java/org/apache/cassandra/dht/AbstractBounds.java
View File

@ -184,6 +184,9 @@ public abstract class AbstractBounds<T extends RingPosition<T>> implements Seria
* The first int tells us if it's a range or bounds (depending on the value) _and_ if it's tokens or keys (depending on the
* sign). We use negative kind for keys so as to preserve the serialization of token from older version.
*/
// !WARNING! While we don't support the pre-3.0 messaging protocol, we serialize the token range in the
// system table (see SystemKeypsace.rangeToBytes) using the old/pre-3.0 format and until we deal with that
// problem, we have to preserve this code.
if (version < MessagingService.VERSION_30)
out.writeInt(kindInt(range));
else
@ -195,6 +198,7 @@ public abstract class AbstractBounds<T extends RingPosition<T>> implements Seria
public AbstractBounds<T> deserialize(DataInput in, IPartitioner p, int version) throws IOException
{
boolean isToken, startInclusive, endInclusive;
// !WARNING! See serialize method above for why we still need to have that condition.
if (version < MessagingService.VERSION_30)
{
int kind = in.readInt();
@ -226,6 +230,7 @@ public abstract class AbstractBounds<T extends RingPosition<T>> implements Seria
public long serializedSize(AbstractBounds<T> ab, int version)
{
// !WARNING! See serialize method above for why we still need to have that condition.
int size = version < MessagingService.VERSION_30
? TypeSizes.sizeof(kindInt(ab))
: 1;

6
src/java/org/apache/cassandra/gms/Gossiper.java
View File

@ -979,12 +979,6 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
private void markAlive(final InetAddress addr, final EndpointState localState)
{
if (MessagingService.instance().getVersion(addr) < MessagingService.VERSION_20)
{
realMarkAlive(addr, localState);
return;
}
localState.markDead();
MessageOut<EchoMessage> echoMessage = new MessageOut<EchoMessage>(MessagingService.Verb.ECHO, EchoMessage.instance, EchoMessage.serializer);

244
src/java/org/apache/cassandra/hints/LegacyHintsMigrator.java
View File

@ -1,244 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.hints;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.SchemaConstants;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.utils.FBUtilities;
/**
* A migrator that goes through the legacy system.hints table and writes all the hints to the new hints storage format.
*/
@SuppressWarnings("deprecation")
public final class LegacyHintsMigrator
{
private static final Logger logger = LoggerFactory.getLogger(LegacyHintsMigrator.class);
private final File hintsDirectory;
private final long maxHintsFileSize;
private final ColumnFamilyStore legacyHintsTable;
private final int pageSize;
public LegacyHintsMigrator(File hintsDirectory, long maxHintsFileSize)
{
this.hintsDirectory = hintsDirectory;
this.maxHintsFileSize = maxHintsFileSize;
legacyHintsTable = Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(SystemKeyspace.LEGACY_HINTS);
pageSize = calculatePageSize(legacyHintsTable);
}
// read fewer columns (mutations) per page if they are very large
private static int calculatePageSize(ColumnFamilyStore legacyHintsTable)
{
int size = 128;
int meanCellCount = legacyHintsTable.getMeanColumns();
double meanPartitionSize = legacyHintsTable.getMeanPartitionSize();
if (meanCellCount != 0 && meanPartitionSize != 0)
{
int avgHintSize = (int) meanPartitionSize / meanCellCount;
size = Math.max(2, Math.min(size, (512 << 10) / avgHintSize));
}
return size;
}
public void migrate()
{
// nothing to migrate
if (legacyHintsTable.isEmpty())
return;
logger.info("Migrating legacy hints to new storage");
// major-compact all of the existing sstables to get rid of the tombstones + expired hints
logger.info("Forcing a major compaction of {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
compactLegacyHints();
// paginate over legacy hints and write them to the new storage
logger.info("Writing legacy hints to the new storage");
migrateLegacyHints();
// truncate the legacy hints table
logger.info("Truncating {}.{} table", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
legacyHintsTable.truncateBlocking();
}
private void compactLegacyHints()
{
Collection<Descriptor> descriptors = new ArrayList<>();
legacyHintsTable.getTracker().getUncompacting().forEach(sstable -> descriptors.add(sstable.descriptor));
if (!descriptors.isEmpty())
forceCompaction(descriptors);
}
private void forceCompaction(Collection<Descriptor> descriptors)
{
try
{
CompactionManager.instance.submitUserDefined(legacyHintsTable, descriptors, FBUtilities.nowInSeconds()).get();
}
catch (InterruptedException | ExecutionException e)
{
throw new RuntimeException(e);
}
}
private void migrateLegacyHints()
{
ByteBuffer buffer = ByteBuffer.allocateDirect(256 * 1024);
String query = String.format("SELECT DISTINCT target_id FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SystemKeyspace.LEGACY_HINTS);
//noinspection ConstantConditions
QueryProcessor.executeInternal(query).forEach(row -> migrateLegacyHints(row.getUUID("target_id"), buffer));
FileUtils.clean(buffer);
}
private void migrateLegacyHints(UUID hostId, ByteBuffer buffer)
{
String query = String.format("SELECT target_id, hint_id, message_version, mutation, ttl(mutation) AS ttl, writeTime(mutation) AS write_time " +
"FROM %s.%s " +
"WHERE target_id = ?",
SchemaConstants.SYSTEM_KEYSPACE_NAME,
SystemKeyspace.LEGACY_HINTS);
// read all the old hints (paged iterator), write them in the new format
UntypedResultSet rows = QueryProcessor.executeInternalWithPaging(query, pageSize, hostId);
migrateLegacyHints(hostId, rows, buffer);
// delete the whole partition in the legacy table; we would truncate the whole table afterwards, but this allows
// to not lose progress in case of a terminated conversion
deleteLegacyHintsPartition(hostId);
}
private void migrateLegacyHints(UUID hostId, UntypedResultSet rows, ByteBuffer buffer)
{
migrateLegacyHints(hostId, rows.iterator(), buffer);
}
private void migrateLegacyHints(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
{
do
{
migrateLegacyHintsInternal(hostId, iterator, buffer);
// if there are hints that didn't fit in the previous file, keep calling the method to write to a new
// file until we get everything written.
}
while (iterator.hasNext());
}
private void migrateLegacyHintsInternal(UUID hostId, Iterator<UntypedResultSet.Row> iterator, ByteBuffer buffer)
{
HintsDescriptor descriptor = new HintsDescriptor(hostId, System.currentTimeMillis());
try (HintsWriter writer = HintsWriter.create(hintsDirectory, descriptor))
{
try (HintsWriter.Session session = writer.newSession(buffer))
{
while (iterator.hasNext())
{
Hint hint = convertLegacyHint(iterator.next());
if (hint != null)
session.append(hint);
if (session.position() >= maxHintsFileSize)
break;
}
}
}
catch (IOException e)
{
throw new FSWriteError(e, descriptor.fileName());
}
}
private static Hint convertLegacyHint(UntypedResultSet.Row row)
{
Mutation mutation = deserializeLegacyMutation(row);
if (mutation == null)
return null;
long creationTime = row.getLong("write_time"); // milliseconds, not micros, for the hints table
int expirationTime = FBUtilities.nowInSeconds() + row.getInt("ttl");
int originalGCGS = expirationTime - (int) TimeUnit.MILLISECONDS.toSeconds(creationTime);
int gcgs = Math.min(originalGCGS, mutation.smallestGCGS());
return Hint.create(mutation, creationTime, gcgs);
}
private static Mutation deserializeLegacyMutation(UntypedResultSet.Row row)
{
try (DataInputBuffer dib = new DataInputBuffer(row.getBlob("mutation"), true))
{
Mutation mutation = Mutation.serializer.deserialize(dib,
row.getInt("message_version"));
mutation.getPartitionUpdates().forEach(PartitionUpdate::validate);
return mutation;
}
catch (IOException e)
{
logger.error("Failed to migrate a hint for {} from legacy {}.{} table",
row.getUUID("target_id"),
SchemaConstants.SYSTEM_KEYSPACE_NAME,
SystemKeyspace.LEGACY_HINTS,
e);
return null;
}
catch (MarshalException e)
{
logger.warn("Failed to validate a hint for {} from legacy {}.{} table - skipping",
row.getUUID("target_id"),
SchemaConstants.SYSTEM_KEYSPACE_NAME,
SystemKeyspace.LEGACY_HINTS,
e);
return null;
}
}
private static void deleteLegacyHintsPartition(UUID hostId)
{
// intentionally use millis, like the rest of the legacy implementation did, just in case
Mutation mutation = new Mutation(PartitionUpdate.fullPartitionDelete(SystemKeyspace.LegacyHints,
UUIDType.instance.decompose(hostId),
System.currentTimeMillis(),
FBUtilities.nowInSeconds()));
mutation.applyUnsafe();
}
}

57
src/java/org/apache/cassandra/io/ForwardingVersionedSerializer.java
View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io;
import java.io.IOException;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
/**
* A serializer which forwards all its method calls to another serializer. Subclasses should override one or more
* methods to modify the behavior of the backing serializer as desired per the decorator pattern.
*/
public abstract class ForwardingVersionedSerializer<T> implements IVersionedSerializer<T>
{
protected ForwardingVersionedSerializer()
{
}
/**
* Returns the backing delegate instance that methods are forwarded to.
*
* @param version the server version
* @return the backing delegate instance that methods are forwarded to.
*/
protected abstract IVersionedSerializer<T> delegate(int version);
public void serialize(T t, DataOutputPlus out, int version) throws IOException
{
delegate(version).serialize(t, out, version);
}
public T deserialize(DataInputPlus in, int version) throws IOException
{
return delegate(version).deserialize(in, version);
}
public long serializedSize(T t, int version)
{
return delegate(version).serializedSize(t, version);
}
}

11
src/java/org/apache/cassandra/io/compress/CompressionMetadata.java
View File

@ -71,7 +71,6 @@ public class CompressionMetadata
private final long chunkOffsetsSize;
public final String indexFilePath;
public final CompressionParams parameters;
public final ChecksumType checksumType;
/**
* Create metadata about given compressed file including uncompressed data length, chunk size
@ -87,14 +86,13 @@ public class CompressionMetadata
public static CompressionMetadata create(String dataFilePath)
{
Descriptor desc = Descriptor.fromFilename(dataFilePath);
return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length(), desc.version.compressedChecksumType());
return new CompressionMetadata(desc.filenameFor(Component.COMPRESSION_INFO), new File(dataFilePath).length());
}
@VisibleForTesting
public CompressionMetadata(String indexFilePath, long compressedLength, ChecksumType checksumType)
public CompressionMetadata(String indexFilePath, long compressedLength)
{
this.indexFilePath = indexFilePath;
this.checksumType = checksumType;
try (DataInputStream stream = new DataInputStream(new FileInputStream(indexFilePath)))
{
@ -133,7 +131,7 @@ public class CompressionMetadata
this.chunkOffsetsSize = chunkOffsets.size();
}
private CompressionMetadata(String filePath, CompressionParams parameters, SafeMemory offsets, long offsetsSize, long dataLength, long compressedLength, ChecksumType checksumType)
private CompressionMetadata(String filePath, CompressionParams parameters, SafeMemory offsets, long offsetsSize, long dataLength, long compressedLength)
{
this.indexFilePath = filePath;
this.parameters = parameters;
@ -141,7 +139,6 @@ public class CompressionMetadata
this.compressedFileLength = compressedLength;
this.chunkOffsets = offsets;
this.chunkOffsetsSize = offsetsSize;
this.checksumType = checksumType;
}
public ICompressor compressor()
@ -417,7 +414,7 @@ public class CompressionMetadata
if (count < this.count)
compressedLength = offsets.getLong(count * 8L);
return new CompressionMetadata(filePath, parameters, offsets, count * 8L, dataLength, compressedLength, ChecksumType.CRC32);
return new CompressionMetadata(filePath, parameters, offsets, count * 8L, dataLength, compressedLength);
}
/**

9
src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
View File

@ -18,7 +18,7 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.FilenameFilter;
import java.io.FileFilter;
import java.io.IOException;
import java.io.Closeable;
import java.nio.ByteBuffer;
@ -90,12 +90,11 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
private static int getNextGeneration(File directory, final String columnFamily)
{
final Set<Descriptor> existing = new HashSet<>();
directory.list(new FilenameFilter()
directory.listFiles(new FileFilter()
{
public boolean accept(File dir, String name)
public boolean accept(File file)
{
Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
Descriptor desc = p == null ? null : p.left;
Descriptor desc = SSTable.tryDescriptorFromFilename(file);
if (desc == null)
return false;

94
src/java/org/apache/cassandra/io/sstable/Component.java
View File

@ -50,8 +50,8 @@ public class Component
COMPRESSION_INFO("CompressionInfo.db"),
// statistical metadata about the content of the sstable
STATS("Statistics.db"),
// holds adler32 checksum of the data file
DIGEST("Digest.crc32", "Digest.adler32", "Digest.sha1"),
// holds CRC32 checksum of the data file
DIGEST("Digest.crc32"),
// holds the CRC32 for chunks in an a uncompressed file.
CRC("CRC.db"),
// holds SSTable Index Summary (sampling of Index component)
@ -61,15 +61,10 @@ public class Component
// built-in secondary index (may be multiple per sstable)
SECONDARY_INDEX("SI_.*.db"),
// custom component, used by e.g. custom compaction strategy
CUSTOM(new String[] { null });
CUSTOM(null);
final String[] repr;
final String repr;
Type(String repr)
{
this(new String[] { repr });
}
Type(String... repr)
{
this.repr = repr;
}
@ -78,9 +73,7 @@ public class Component
{
for (Type type : TYPES)
{
if (type.repr == null || type.repr.length == 0 || type.repr[0] == null)
continue;
if (Pattern.matches(type.repr[0], repr))
if (type.repr != null && Pattern.matches(type.repr, repr))
return type;
}
return CUSTOM;
@ -93,36 +86,18 @@ public class Component
public final static Component FILTER = new Component(Type.FILTER);
public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO);
public final static Component STATS = new Component(Type.STATS);
private static final String digestCrc32 = "Digest.crc32";
private static final String digestAdler32 = "Digest.adler32";
private static final String digestSha1 = "Digest.sha1";
public final static Component DIGEST_CRC32 = new Component(Type.DIGEST, digestCrc32);
public final static Component DIGEST_ADLER32 = new Component(Type.DIGEST, digestAdler32);
public final static Component DIGEST_SHA1 = new Component(Type.DIGEST, digestSha1);
public final static Component DIGEST = new Component(Type.DIGEST);
public final static Component CRC = new Component(Type.CRC);
public final static Component SUMMARY = new Component(Type.SUMMARY);
public final static Component TOC = new Component(Type.TOC);
public static Component digestFor(ChecksumType checksumType)
{
switch (checksumType)
{
case Adler32:
return DIGEST_ADLER32;
case CRC32:
return DIGEST_CRC32;
}
throw new AssertionError();
}
public final Type type;
public final String name;
public final int hashCode;
public Component(Type type)
{
this(type, type.repr[0]);
assert type.repr.length == 1;
this(type, type.repr);
assert type != Type.CUSTOM;
}
@ -143,45 +118,32 @@ public class Component
}
/**
* {@code
* Filename of the form "<ksname>/<cfname>-[tmp-][<version>-]<gen>-<component>",
* }
* @return A Descriptor for the SSTable, and a Component for this particular file.
* TODO move descriptor into Component field
* Parse the component part of a sstable filename into a {@code Component} object.
*
* @param name a string representing a sstable component.
* @return the component corresponding to {@code name}. Note that this always return a component as an unrecognized
* name is parsed into a CUSTOM component.
*/
public static Pair<Descriptor,Component> fromFilename(File directory, String name)
static Component parse(String name)
{
Pair<Descriptor,String> path = Descriptor.fromFilename(directory, name);
Type type = Type.fromRepresentation(name);
// parse the component suffix
Type type = Type.fromRepresentation(path.right);
// build (or retrieve singleton for) the component object
Component component;
switch(type)
// Build (or retrieve singleton for) the component object
switch (type)
{
case DATA: component = Component.DATA; break;
case PRIMARY_INDEX: component = Component.PRIMARY_INDEX; break;
case FILTER: component = Component.FILTER; break;
case COMPRESSION_INFO: component = Component.COMPRESSION_INFO; break;
case STATS: component = Component.STATS; break;
case DIGEST: switch (path.right)
{
case digestCrc32: component = Component.DIGEST_CRC32; break;
case digestAdler32: component = Component.DIGEST_ADLER32; break;
case digestSha1: component = Component.DIGEST_SHA1; break;
default: throw new IllegalArgumentException("Invalid digest component " + path.right);
}
break;
case CRC: component = Component.CRC; break;
case SUMMARY: component = Component.SUMMARY; break;
case TOC: component = Component.TOC; break;
case SECONDARY_INDEX: component = new Component(Type.SECONDARY_INDEX, path.right); break;
case CUSTOM: component = new Component(Type.CUSTOM, path.right); break;
default:
throw new IllegalStateException();
case DATA: return Component.DATA;
case PRIMARY_INDEX: return Component.PRIMARY_INDEX;
case FILTER: return Component.FILTER;
case COMPRESSION_INFO: return Component.COMPRESSION_INFO;
case STATS: return Component.STATS;
case DIGEST: return Component.DIGEST;
case CRC: return Component.CRC;
case SUMMARY: return Component.SUMMARY;
case TOC: return Component.TOC;
case SECONDARY_INDEX: return new Component(Type.SECONDARY_INDEX, name);
case CUSTOM: return new Component(Type.CUSTOM, name);
default: throw new AssertionError();
}
return Pair.create(path.left, component);
}
@Override

274
src/java/org/apache/cassandra/io/sstable/Descriptor.java
View File

@ -26,12 +26,12 @@ import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CharMatcher;
import com.google.common.base.Objects;
import com.google.common.base.Splitter;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
import org.apache.cassandra.utils.Pair;
@ -46,8 +46,13 @@ import static org.apache.cassandra.io.sstable.Component.separator;
*/
public class Descriptor
{
private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
public static String TMP_EXT = ".tmp";
private static final Splitter filenameSplitter = Splitter.on('-');
/** canonicalized path to the directory where SSTable resides */
public final File directory;
/** version has the following format: <code>[a-z]+</code> */
@ -56,8 +61,6 @@ public class Descriptor
public final String cfname;
public final int generation;
public final SSTableFormat.Type formatType;
/** digest component - might be {@code null} for old, legacy sstables */
public final Component digestComponent;
private final int hashCode;
/**
@ -66,7 +69,7 @@ public class Descriptor
@VisibleForTesting
public Descriptor(File directory, String ksname, String cfname, int generation)
{
this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current(), null);
this(SSTableFormat.Type.current().info.getLatestVersion(), directory, ksname, cfname, generation, SSTableFormat.Type.current());
}
/**
@ -74,16 +77,10 @@ public class Descriptor
*/
public Descriptor(File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
{
this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType()));
this(formatType.info.getLatestVersion(), directory, ksname, cfname, generation, formatType);
}
@VisibleForTesting
public Descriptor(String version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
{
this(formatType.info.getVersion(version), directory, ksname, cfname, generation, formatType, Component.digestFor(formatType.info.getLatestVersion().uncompressedChecksumType()));
}
public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType, Component digestComponent)
public Descriptor(Version version, File directory, String ksname, String cfname, int generation, SSTableFormat.Type formatType)
{
assert version != null && directory != null && ksname != null && cfname != null && formatType.info.getLatestVersion().getClass().equals(version.getClass());
this.version = version;
@ -99,24 +96,18 @@ public class Descriptor
this.cfname = cfname;
this.generation = generation;
this.formatType = formatType;
this.digestComponent = digestComponent;
hashCode = Objects.hashCode(version, this.directory, generation, ksname, cfname, formatType);
}
public Descriptor withGeneration(int newGeneration)
{
return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType, digestComponent);
return new Descriptor(version, directory, ksname, cfname, newGeneration, formatType);
}
public Descriptor withFormatType(SSTableFormat.Type newType)
{
return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType, digestComponent);
}
public Descriptor withDigestComponent(Component newDigestComponent)
{
return new Descriptor(version, directory, ksname, cfname, generation, formatType, newDigestComponent);
return new Descriptor(newType.info.getLatestVersion(), directory, ksname, cfname, generation, newType);
}
public String tmpFilenameFor(Component component)
@ -139,15 +130,9 @@ public class Descriptor
private void appendFileName(StringBuilder buff)
{
if (!version.hasNewFileName())
{
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
}
buff.append(version).append(separator);
buff.append(generation);
if (formatType != SSTableFormat.Type.LEGACY)
buff.append(separator).append(formatType.name);
buff.append(separator).append(formatType.name);
}
public String relativeFilenameFor(Component component)
@ -176,155 +161,156 @@ public class Descriptor
return ret;
}
/**
* Files obsoleted by CASSANDRA-7066 : temporary files and compactions_in_progress. We support
* versions 2.1 (ka) and 2.2 (la).
* Temporary files have tmp- or tmplink- at the beginning for 2.2 sstables or after ks-cf- for 2.1 sstables
*/
private final static String LEGACY_COMP_IN_PROG_REGEX_STR = "^compactions_in_progress(\\-[\\d,a-f]{32})?$";
private final static Pattern LEGACY_COMP_IN_PROG_REGEX = Pattern.compile(LEGACY_COMP_IN_PROG_REGEX_STR);
private final static String LEGACY_TMP_REGEX_STR = "^((.*)\\-(.*)\\-)?tmp(link)?\\-((?:l|k).)\\-(\\d)*\\-(.*)$";
private final static Pattern LEGACY_TMP_REGEX = Pattern.compile(LEGACY_TMP_REGEX_STR);
public static boolean isLegacyFile(File file)
public static boolean isValidFile(File file)
{
if (file.isDirectory())
return file.getParentFile() != null &&
file.getParentFile().getName().equalsIgnoreCase("system") &&
LEGACY_COMP_IN_PROG_REGEX.matcher(file.getName()).matches();
else
return LEGACY_TMP_REGEX.matcher(file.getName()).matches();
}
public static boolean isValidFile(String fileName)
{
return fileName.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(fileName).matches();
String filename = file.getName();
return filename.endsWith(".db") && !LEGACY_TMP_REGEX.matcher(filename).matches();
}
/**
* @see #fromFilename(File directory, String name)
* @param filename The SSTable filename
* @return Descriptor of the SSTable initialized from filename
* Parse a sstable filename into a Descriptor.
* <p>
* This is a shortcut for {@code fromFilename(new File(filename))}.
*
* @param filename the filename to a sstable component.
* @return the descriptor for the parsed file.
*
* @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
* mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
* versions.
*/
public static Descriptor fromFilename(String filename)
{
return fromFilename(filename, false);
}
public static Descriptor fromFilename(String filename, SSTableFormat.Type formatType)
{
return fromFilename(filename).withFormatType(formatType);
}
public static Descriptor fromFilename(String filename, boolean skipComponent)
{
File file = new File(filename).getAbsoluteFile();
return fromFilename(file.getParentFile(), file.getName(), skipComponent).left;
}
public static Pair<Descriptor, String> fromFilename(File directory, String name)
{
return fromFilename(directory, name, false);
return fromFilename(new File(filename));
}
/**
* Filename of the form is vary by version:
* Parse a sstable filename into a Descriptor.
* <p>
* SSTables files are all located within subdirectories of the form {@code <keyspace>/<table>/}. Normal sstables are
* are directly within that subdirectory structure while 2ndary index, backups and snapshot are each inside an
* additional subdirectory. The file themselves have the form:
* {@code <version>-<gen>-<format>-<component>}.
* <p>
* Note that this method will only sucessfully parse sstable files of supported versions.
*
* <ul>
* <li>&lt;ksname&gt;-&lt;cfname&gt;-(tmp-)?&lt;version&gt;-&lt;gen&gt;-&lt;component&gt; for cassandra 2.0 and before</li>
* <li>(&lt;tmp marker&gt;-)?&lt;version&gt;-&lt;gen&gt;-&lt;component&gt; for cassandra 3.0 and later</li>
* </ul>
* @param file the {@code File} object for the filename to parse.
* @return the descriptor for the parsed file.
*
* If this is for SSTable of secondary index, directory should ends with index name for 2.1+.
*
* @param directory The directory of the SSTable files
* @param name The name of the SSTable file
* @param skipComponent true if the name param should not be parsed for a component tag
*
* @return A Descriptor for the SSTable, and the Component remainder.
* @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
* mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
* versions.
*/
public static Pair<Descriptor, String> fromFilename(File directory, String name, boolean skipComponent)
public static Descriptor fromFilename(File file)
{
File parentDirectory = directory != null ? directory : new File(".");
return fromFilenameWithComponent(file).left;
}
// tokenize the filename
StringTokenizer st = new StringTokenizer(name, String.valueOf(separator));
String nexttok;
/**
* Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part.
*
* @param file the {@code File} object for the filename to parse.
* @return a pair of the descriptor and component corresponding to the provided {@code file}.
*
* @throws IllegalArgumentException if the provided {@code file} does point to a valid sstable filename. This could
* mean either that the filename doesn't look like a sstable file, or that it is for an old and unsupported
* versions.
*/
public static Pair<Descriptor, Component> fromFilenameWithComponent(File file)
{
// We need to extract the keyspace and table names from the parent directories, so make sure we deal with the
// absolute path.
if (!file.isAbsolute())
file = file.getAbsoluteFile();
// read tokens backwards to determine version
Deque<String> tokenStack = new ArrayDeque<>();
while (st.hasMoreTokens())
String name = file.getName();
List<String> tokens = filenameSplitter.splitToList(name);
int size = tokens.size();
if (size != 4)
{
tokenStack.push(st.nextToken());
// This is an invalid sstable file for this version. But to provide a more helpful error message, we detect
// old format sstable, which had the format:
// <keyspace>-<table>-(tmp-)?<version>-<gen>-<component>
// Note that we assume it's an old format sstable if it has the right number of tokens: this is not perfect
// but we're just trying to be helpful, not perfect.
if (size == 5 || size == 6)
throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.",
name,
tokens.get(size - 3)));
throw new IllegalArgumentException(String.format("Invalid sstable file %s: the name doesn't look like a supported sstable file name", name));
}
// component suffix
String component = skipComponent ? null : tokenStack.pop();
String versionString = tokens.get(0);
if (!Version.validate(versionString))
throw invalidSSTable(name, "invalid version %s", versionString);
nexttok = tokenStack.pop();
// generation OR format type
SSTableFormat.Type fmt = SSTableFormat.Type.LEGACY;
if (!CharMatcher.DIGIT.matchesAllOf(nexttok))
int generation;
try
{
fmt = SSTableFormat.Type.validate(nexttok);
nexttok = tokenStack.pop();
generation = Integer.parseInt(tokens.get(1));
}
catch (NumberFormatException e)
{
throw invalidSSTable(name, "the 'generation' part of the name doesn't parse as a number");
}
// generation
int generation = Integer.parseInt(nexttok);
// version
nexttok = tokenStack.pop();
if (!Version.validate(nexttok))
throw new UnsupportedOperationException("SSTable " + name + " is too old to open. Upgrade to 2.0 first, and run upgradesstables");
Version version = fmt.info.getVersion(nexttok);
// ks/cf names
String ksname, cfname;
if (version.hasNewFileName())
String formatString = tokens.get(2);
SSTableFormat.Type format;
try
{
// for 2.1+ read ks and cf names from directory
File cfDirectory = parentDirectory;
// check if this is secondary index
String indexName = "";
if (cfDirectory.getName().startsWith(Directories.SECONDARY_INDEX_NAME_SEPARATOR))
{
indexName = cfDirectory.getName();
cfDirectory = cfDirectory.getParentFile();
}
if (cfDirectory.getName().equals(Directories.BACKUPS_SUBDIR))
{
cfDirectory = cfDirectory.getParentFile();
}
else if (cfDirectory.getParentFile().getName().equals(Directories.SNAPSHOT_SUBDIR))
{
cfDirectory = cfDirectory.getParentFile().getParentFile();
}
cfname = cfDirectory.getName().split("-")[0] + indexName;
ksname = cfDirectory.getParentFile().getName();
format = SSTableFormat.Type.validate(formatString);
}
else
catch (IllegalArgumentException e)
{
cfname = tokenStack.pop();
ksname = tokenStack.pop();
throw invalidSSTable(name, "unknown 'format' part (%s)", formatString);
}
assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory;
return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, fmt,
// _assume_ version from version
Component.digestFor(version.uncompressedChecksumType())),
component);
Component component = Component.parse(tokens.get(3));
Version version = format.info.getVersion(versionString);
if (!version.isCompatible())
throw invalidSSTable(name, "incompatible sstable version (%s); you should have run upgradesstables before upgrading", versionString);
File directory = parentOf(name, file);
File tableDir = directory;
// Check if it's a 2ndary index directory (not that it doesn't exclude it to be also a backup or snapshot)
String indexName = "";
if (tableDir.getName().startsWith(Directories.SECONDARY_INDEX_NAME_SEPARATOR))
{
indexName = tableDir.getName();
tableDir = parentOf(name, tableDir);
}
// Then it can be a backup or a snapshot
if (tableDir.getName().equals(Directories.BACKUPS_SUBDIR))
tableDir = tableDir.getParentFile();
else if (parentOf(name, tableDir).getName().equals(Directories.SNAPSHOT_SUBDIR))
tableDir = parentOf(name, parentOf(name, tableDir));
String table = tableDir.getName().split("-")[0] + indexName;
String keyspace = parentOf(name, tableDir).getName();
return Pair.create(new Descriptor(version, directory, keyspace, table, generation, format), component);
}
private static File parentOf(String name, File file)
{
File parent = file.getParentFile();
if (parent == null)
throw invalidSSTable(name, "cannot extract keyspace and table name; make sure the sstable is in the proper sub-directories");
return parent;
}
private static IllegalArgumentException invalidSSTable(String name, String msgFormat, Object... parameters)
{
throw new IllegalArgumentException(String.format("Invalid sstable file " + name + ": " + msgFormat, parameters));
}
public IMetadataSerializer getMetadataSerializer()
{
if (version.hasNewStatsFile())
return new MetadataSerializer();
else
return new LegacyMetadataSerializer();
return new MetadataSerializer();
}
/**

78
src/java/org/apache/cassandra/io/sstable/IndexInfo.java
View File

@ -19,11 +19,14 @@
package org.apache.cassandra.io.sstable;
import java.io.IOException;
import java.util.List;
import org.apache.cassandra.db.ClusteringPrefix;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.RowIndexEntry;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
@ -79,6 +82,11 @@ public class IndexInfo
this.endOpenMarker = endOpenMarker;
}
public static IndexInfo.Serializer serializer(Version version, SerializationHeader header)
{
return new IndexInfo.Serializer(version, header.clusteringTypes());
}
public static class Serializer implements ISerializer<IndexInfo>
{
// This is the default index size that we use to delta-encode width when serializing so we get better vint-encoding.
@ -87,21 +95,19 @@ public class IndexInfo
// size so using the default is almost surely better than using no base at all.
public static final long WIDTH_BASE = 64 * 1024;
private final ISerializer<ClusteringPrefix> clusteringSerializer;
private final Version version;
private final int version;
private final List<AbstractType<?>> clusteringTypes;
public Serializer(Version version, ISerializer<ClusteringPrefix> clusteringSerializer)
public Serializer(Version version, List<AbstractType<?>> clusteringTypes)
{
this.clusteringSerializer = clusteringSerializer;
this.version = version;
this.version = version.correspondingMessagingVersion();
this.clusteringTypes = clusteringTypes;
}
public void serialize(IndexInfo info, DataOutputPlus out) throws IOException
{
assert version.storeRows() : "We read old index files but we should never write them";
clusteringSerializer.serialize(info.firstName, out);
clusteringSerializer.serialize(info.lastName, out);
ClusteringPrefix.serializer.serialize(info.firstName, out, version, clusteringTypes);
ClusteringPrefix.serializer.serialize(info.lastName, out, version, clusteringTypes);
out.writeUnsignedVInt(info.offset);
out.writeVInt(info.width - WIDTH_BASE);
@ -112,53 +118,33 @@ public class IndexInfo
public void skip(DataInputPlus in) throws IOException
{
clusteringSerializer.skip(in);
clusteringSerializer.skip(in);
if (version.storeRows())
{
in.readUnsignedVInt();
in.readVInt();
if (in.readBoolean())
DeletionTime.serializer.skip(in);
}
else
{
in.skipBytes(TypeSizes.sizeof(0L));
in.skipBytes(TypeSizes.sizeof(0L));
}
ClusteringPrefix.serializer.skip(in, version, clusteringTypes);
ClusteringPrefix.serializer.skip(in, version, clusteringTypes);
in.readUnsignedVInt();
in.readVInt();
if (in.readBoolean())
DeletionTime.serializer.skip(in);
}
public IndexInfo deserialize(DataInputPlus in) throws IOException
{
ClusteringPrefix firstName = clusteringSerializer.deserialize(in);
ClusteringPrefix lastName = clusteringSerializer.deserialize(in);
long offset;
long width;
ClusteringPrefix firstName = ClusteringPrefix.serializer.deserialize(in, version, clusteringTypes);
ClusteringPrefix lastName = ClusteringPrefix.serializer.deserialize(in, version, clusteringTypes);
long offset = in.readUnsignedVInt();
long width = in.readVInt() + WIDTH_BASE;
DeletionTime endOpenMarker = null;
if (version.storeRows())
{
offset = in.readUnsignedVInt();
width = in.readVInt() + WIDTH_BASE;
if (in.readBoolean())
endOpenMarker = DeletionTime.serializer.deserialize(in);
}
else
{
offset = in.readLong();
width = in.readLong();
}
if (in.readBoolean())
endOpenMarker = DeletionTime.serializer.deserialize(in);
return new IndexInfo(firstName, lastName, offset, width, endOpenMarker);
}
public long serializedSize(IndexInfo info)
{
assert version.storeRows() : "We read old index files but we should never write them";
long size = clusteringSerializer.serializedSize(info.firstName)
+ clusteringSerializer.serializedSize(info.lastName)
+ TypeSizes.sizeofUnsignedVInt(info.offset)
+ TypeSizes.sizeofVInt(info.width - WIDTH_BASE)
+ TypeSizes.sizeof(info.endOpenMarker != null);
long size = ClusteringPrefix.serializer.serializedSize(info.firstName, version, clusteringTypes)
+ ClusteringPrefix.serializer.serializedSize(info.lastName, version, clusteringTypes)
+ TypeSizes.sizeofUnsignedVInt(info.offset)
+ TypeSizes.sizeofVInt(info.width - WIDTH_BASE)
+ TypeSizes.sizeof(info.endOpenMarker != null);
if (info.endOpenMarker != null)
size += DeletionTime.serializer.serializedSize(info.endOpenMarker);

29
src/java/org/apache/cassandra/io/sstable/IndexSummary.java
View File

@ -268,16 +268,13 @@ public class IndexSummary extends WrappedSharedCloseable
public static class IndexSummarySerializer
{
public void serialize(IndexSummary t, DataOutputPlus out, boolean withSamplingLevel) throws IOException
public void serialize(IndexSummary t, DataOutputPlus out) throws IOException
{
out.writeInt(t.minIndexInterval);
out.writeInt(t.offsetCount);
out.writeLong(t.getOffHeapSize());
if (withSamplingLevel)
{
out.writeInt(t.samplingLevel);
out.writeInt(t.sizeAtFullSampling);
}
out.writeInt(t.samplingLevel);
out.writeInt(t.sizeAtFullSampling);
// our on-disk representation treats the offsets and the summary data as one contiguous structure,
// in which the offsets are based from the start of the structure. i.e., if the offsets occupy
// X bytes, the value of the first offset will be X. In memory we split the two regions up, so that
@ -297,7 +294,7 @@ public class IndexSummary extends WrappedSharedCloseable
}
@SuppressWarnings("resource")
public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
{
int minIndexInterval = in.readInt();
if (minIndexInterval != expectedMinIndexInterval)
@ -308,17 +305,8 @@ public class IndexSummary extends WrappedSharedCloseable
int offsetCount = in.readInt();
long offheapSize = in.readLong();
int samplingLevel, fullSamplingSummarySize;
if (haveSamplingLevel)
{
samplingLevel = in.readInt();
fullSamplingSummarySize = in.readInt();
}
else
{
samplingLevel = BASE_SAMPLING_LEVEL;
fullSamplingSummarySize = offsetCount;
}
int samplingLevel = in.readInt();
int fullSamplingSummarySize = in.readInt();
int effectiveIndexInterval = (int) Math.ceil((BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval);
if (effectiveIndexInterval > maxIndexInterval)
@ -355,13 +343,12 @@ public class IndexSummary extends WrappedSharedCloseable
*
* Only for use by offline tools like SSTableMetadataViewer, otherwise SSTable.first/last should be used.
*/
public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel) throws IOException
public Pair<DecoratedKey, DecoratedKey> deserializeFirstLastKey(DataInputStream in, IPartitioner partitioner) throws IOException
{
in.skipBytes(4); // minIndexInterval
int offsetCount = in.readInt();
long offheapSize = in.readLong();
if (haveSamplingLevel)
in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
in.skipBytes(8); // samplingLevel, fullSamplingSummarySize
in.skip(offsetCount * 4);
in.skip(offheapSize - offsetCount * 4);

16
src/java/org/apache/cassandra/io/sstable/IndexSummaryRedistribution.java
View File

@ -73,21 +73,9 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
public List<SSTableReader> redistributeSummaries() throws IOException
{
logger.info("Redistributing index summaries");
List<SSTableReader> oldFormatSSTables = new ArrayList<>();
List<SSTableReader> redistribute = new ArrayList<>();
for (LifecycleTransaction txn : transactions.values())
{
for (SSTableReader sstable : ImmutableList.copyOf(txn.originals()))
{
// We can't change the sampling level of sstables with the old format, because the serialization format
// doesn't include the sampling level. Leave this one as it is. (See CASSANDRA-8993 for details.)
logger.trace("SSTable {} cannot be re-sampled due to old sstable format", sstable);
if (!sstable.descriptor.version.hasSamplingLevel())
{
oldFormatSSTables.add(sstable);
txn.cancel(sstable);
}
}
redistribute.addAll(txn.originals());
}
@ -119,7 +107,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
Collections.sort(sstablesByHotness, new ReadRateComparator(readRates));
long remainingBytes = memoryPoolBytes;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables))
for (SSTableReader sstable : compacting)
remainingBytes -= sstable.getIndexSummaryOffHeapSize();
logger.trace("Index summaries for compacting SSTables are using {} MB of space",
@ -130,7 +118,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
txn.finish();
total = 0;
for (SSTableReader sstable : Iterables.concat(compacting, oldFormatSSTables, newSSTables))
for (SSTableReader sstable : Iterables.concat(compacting, newSSTables))
total += sstable.getIndexSummaryOffHeapSize();
logger.trace("Completed resizing of index summaries; current approximate memory used: {}",
FBUtilities.prettyPrintMemory(total));

48
src/java/org/apache/cassandra/io/sstable/SSTable.java
View File

@ -168,14 +168,40 @@ public abstract class SSTable
}
/**
* @return Descriptor and Component pair. null if given file is not acceptable as SSTable component.
* If component is of unknown type, returns CUSTOM component.
* Parse a sstable filename into both a {@link Descriptor} and {@code Component} object.
*
* @param file the filename to parse.
* @return a pair of the {@code Descriptor} and {@code Component} corresponding to {@code file} if it corresponds to
* a valid and supported sstable filename, {@code null} otherwise. Note that components of an unknown type will be
* returned as CUSTOM ones.
*/
public static Pair<Descriptor, Component> tryComponentFromFilename(File dir, String name)
public static Pair<Descriptor, Component> tryComponentFromFilename(File file)
{
try
{
return Component.fromFilename(dir, name);
return Descriptor.fromFilenameWithComponent(file);
}
catch (Throwable e)
{
return null;
}
}
/**
* Parse a sstable filename into a {@link Descriptor} object.
* <p>
* Note that this method ignores the component part of the filename; if this is not what you want, use
* {@link #tryComponentFromFilename} instead.
*
* @param file the filename to parse.
* @return the {@code Descriptor} corresponding to {@code file} if it corresponds to a valid and supported sstable
* filename, {@code null} otherwise.
*/
public static Descriptor tryDescriptorFromFilename(File file)
{
try
{
return Descriptor.fromFilename(file);
}
catch (Throwable e)
{
@ -218,17 +244,9 @@ public abstract class SSTable
Set<Component> components = Sets.newHashSetWithExpectedSize(knownTypes.size());
for (Component.Type componentType : knownTypes)
{
if (componentType == Component.Type.DIGEST)
{
if (desc.digestComponent != null && new File(desc.filenameFor(desc.digestComponent)).exists())
components.add(desc.digestComponent);
}
else
{
Component component = new Component(componentType);
if (new File(desc.filenameFor(component)).exists())
components.add(component);
}
Component component = new Component(componentType);
if (new File(desc.filenameFor(component)).exists())
components.add(component);
}
return components;
}

2
src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
View File

@ -85,7 +85,7 @@ public class SSTableLoader implements StreamEventHandler
return false;
}
Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(file);
Descriptor desc = p == null ? null : p.left;
if (p == null || !p.right.equals(Component.DATA))
return false;

112
src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java
View File

@ -54,18 +54,12 @@ public abstract class SSTableSimpleIterator extends AbstractIterator<Unfiltered>
public static SSTableSimpleIterator create(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper, DeletionTime partitionDeletion)
{
if (helper.version < MessagingService.VERSION_30)
return new OldFormatIterator(metadata, in, helper, partitionDeletion);
else
return new CurrentFormatIterator(metadata, in, header, helper);
return new CurrentFormatIterator(metadata, in, header, helper);
}
public static SSTableSimpleIterator createTombstoneOnly(CFMetaData metadata, DataInputPlus in, SerializationHeader header, SerializationHelper helper, DeletionTime partitionDeletion)
{
if (helper.version < MessagingService.VERSION_30)
return new OldFormatTombstoneIterator(metadata, in, helper, partitionDeletion);
else
return new CurrentFormatTombstoneIterator(metadata, in, header, helper);
return new CurrentFormatTombstoneIterator(metadata, in, header, helper);
}
public abstract Row readStaticRow() throws IOException;
@ -136,106 +130,4 @@ public abstract class SSTableSimpleIterator extends AbstractIterator<Unfiltered>
}
}
}
private static class OldFormatIterator extends SSTableSimpleIterator
{
private final UnfilteredDeserializer deserializer;
private OldFormatIterator(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, DeletionTime partitionDeletion)
{
super(metadata, in, helper);
// We use an UnfilteredDeserializer because even though we don't need all it's fanciness, it happens to handle all
// the details we need for reading the old format.
this.deserializer = UnfilteredDeserializer.create(metadata, in, null, helper, partitionDeletion, false);
}
public Row readStaticRow() throws IOException
{
if (metadata.isCompactTable())
{
// For static compact tables, in the old format, static columns are intermingled with the other columns, so we
// need to extract them. Which imply 2 passes (one to extract the static, then one for other value).
if (metadata.isStaticCompactTable())
{
assert in instanceof RewindableDataInput;
RewindableDataInput file = (RewindableDataInput)in;
DataPosition mark = file.mark();
Row staticRow = LegacyLayout.extractStaticColumns(metadata, file, metadata.partitionColumns().statics);
file.reset(mark);
// We've extracted the static columns, so we must ignore them on the 2nd pass
((UnfilteredDeserializer.OldFormatDeserializer)deserializer).setSkipStatic();
return staticRow;
}
else
{
return Rows.EMPTY_STATIC_ROW;
}
}
return deserializer.hasNext() && deserializer.nextIsStatic()
? (Row)deserializer.readNext()
: Rows.EMPTY_STATIC_ROW;
}
protected Unfiltered computeNext()
{
while (true)
{
try
{
if (!deserializer.hasNext())
return endOfData();
Unfiltered unfiltered = deserializer.readNext();
if (metadata.isStaticCompactTable() && unfiltered.kind() == Unfiltered.Kind.ROW)
{
Row row = (Row) unfiltered;
ColumnDefinition def = metadata.getColumnDefinition(LegacyLayout.encodeClustering(metadata, row.clustering()));
if (def != null && def.isStatic())
continue;
}
return unfiltered;
}
catch (IOException e)
{
throw new IOError(e);
}
}
}
}
private static class OldFormatTombstoneIterator extends OldFormatIterator
{
private OldFormatTombstoneIterator(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, DeletionTime partitionDeletion)
{
super(metadata, in, helper, partitionDeletion);
}
public Row readStaticRow() throws IOException
{
Row row = super.readStaticRow();
if (!row.deletion().isLive())
return BTreeRow.emptyDeletedRow(row.clustering(), row.deletion());
return Rows.EMPTY_STATIC_ROW;
}
protected Unfiltered computeNext()
{
while (true)
{
Unfiltered unfiltered = super.computeNext();
if (unfiltered == null || unfiltered.isRangeTombstoneMarker())
return unfiltered;
Row row = (Row) unfiltered;
if (!row.deletion().isLive())
return BTreeRow.emptyDeletedRow(row.clustering(), row.deletion());
// Otherwise read next.
}
}
}
}

10
src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
View File

@ -148,14 +148,8 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
return new SSTableTxnWriter(txn, writer);
}
public static SSTableTxnWriter create(ColumnFamilyStore cfs, String filename, long keyCount, long repairedAt, int sstableLevel, SerializationHeader header)
public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor desc, long keyCount, long repairedAt, SerializationHeader header)
{
Descriptor desc = Descriptor.fromFilename(filename);
return create(cfs, desc, keyCount, repairedAt, sstableLevel, header);
}
public static SSTableTxnWriter create(ColumnFamilyStore cfs, String filename, long keyCount, long repairedAt, SerializationHeader header)
{
return create(cfs, filename, keyCount, repairedAt, 0, header);
return create(cfs, desc, keyCount, repairedAt, 0, header);
}
}

4
src/java/org/apache/cassandra/io/sstable/format/RangeAwareSSTableWriter.java
View File

@ -68,7 +68,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
if (localDir == null)
throw new IOException(String.format("Insufficient disk space to store %s",
FBUtilities.prettyPrintMemory(totalSize)));
Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(cfs.getDirectories().getLocationForDisk(localDir), format));
Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, sstableLevel, header, txn);
}
}
@ -90,7 +90,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
if (currentWriter != null)
finishedWriters.add(currentWriter);
Descriptor desc = Descriptor.fromFilename(cfs.getSSTablePath(cfs.getDirectories().getLocationForDisk(directories[currentIndex])), format);
Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories[currentIndex]), format);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, sstableLevel, header, txn);
}
}

8
src/java/org/apache/cassandra/io/sstable/format/SSTableFormat.java
View File

@ -41,10 +41,6 @@ public interface SSTableFormat
public static enum Type
{
//Used internally to refer to files with no
//format flag in the filename
LEGACY("big", BigFormat.instance),
//The original sstable format
BIG("big", BigFormat.instance);
@ -70,10 +66,6 @@ public interface SSTableFormat
{
for (Type valid : Type.values())
{
//This is used internally for old sstables
if (valid == LEGACY)
continue;
if (valid.name.equalsIgnoreCase(name))
return valid;
}

96
src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
View File

@ -253,58 +253,48 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
long count = -1;
// check if cardinality estimator is available for all SSTables
boolean cardinalityAvailable = !Iterables.isEmpty(sstables) && Iterables.all(sstables, new Predicate<SSTableReader>()
if (Iterables.isEmpty(sstables))
return count;
boolean failed = false;
ICardinality cardinality = null;
for (SSTableReader sstable : sstables)
{
public boolean apply(SSTableReader sstable)
if (sstable.openReason == OpenReason.EARLY)
continue;
try
{
return sstable.descriptor.version.hasNewStatsFile();
}
});
// if it is, load them to estimate key count
if (cardinalityAvailable)
{
boolean failed = false;
ICardinality cardinality = null;
for (SSTableReader sstable : sstables)
{
if (sstable.openReason == OpenReason.EARLY)
continue;
try
CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
// If we can't load the CompactionMetadata, we are forced to estimate the keys using the index
// summary. (CASSANDRA-10676)
if (metadata == null)
{
CompactionMetadata metadata = (CompactionMetadata) sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor, MetadataType.COMPACTION);
// If we can't load the CompactionMetadata, we are forced to estimate the keys using the index
// summary. (CASSANDRA-10676)
if (metadata == null)
{
logger.warn("Reading cardinality from Statistics.db failed for {}", sstable.getFilename());
failed = true;
break;
}
if (cardinality == null)
cardinality = metadata.cardinalityEstimator;
else
cardinality = cardinality.merge(metadata.cardinalityEstimator);
}
catch (IOException e)
{
logger.warn("Reading cardinality from Statistics.db failed.", e);
failed = true;
break;
}
catch (CardinalityMergeException e)
{
logger.warn("Cardinality merge failed.", e);
logger.warn("Reading cardinality from Statistics.db failed for {}", sstable.getFilename());
failed = true;
break;
}
if (cardinality == null)
cardinality = metadata.cardinalityEstimator;
else
cardinality = cardinality.merge(metadata.cardinalityEstimator);
}
catch (IOException e)
{
logger.warn("Reading cardinality from Statistics.db failed.", e);
failed = true;
break;
}
catch (CardinalityMergeException e)
{
logger.warn("Cardinality merge failed.", e);
failed = true;
break;
}
if (cardinality != null && !failed)
count = cardinality.cardinality();
}
if (cardinality != null && !failed)
count = cardinality.cardinality();
// if something went wrong above or cardinality is not available, calculate using index summary
if (count < 0)
@ -481,14 +471,14 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
assert !validate || components.contains(Component.PRIMARY_INDEX) : "Primary index component is missing for sstable " + descriptor;
// For the 3.0+ sstable format, the (misnomed) stats component hold the serialization header which we need to deserialize the sstable content
assert !descriptor.version.storeRows() || components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
assert components.contains(Component.STATS) : "Stats component is missing for sstable " + descriptor;
EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
Map<MetadataType, MetadataComponent> sstableMetadata = descriptor.getMetadataSerializer().deserialize(descriptor, types);
ValidationMetadata validationMetadata = (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
StatsMetadata statsMetadata = (StatsMetadata) sstableMetadata.get(MetadataType.STATS);
SerializationHeader.Component header = (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
assert !descriptor.version.storeRows() || header != null;
assert header != null;
// Check if sstable is created using same partitioner.
// Partitioner can be null, which indicates older version of sstable or no stats available.
@ -730,7 +720,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
// bf is enabled and fp chance matches the currently configured value.
load(false, true);
loadBloomFilter(descriptor.version.hasOldBfHashOrder());
loadBloomFilter();
}
}
@ -739,11 +729,11 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
*
* @throws IOException
*/
private void loadBloomFilter(boolean oldBfHashOrder) throws IOException
private void loadBloomFilter() throws IOException
{
try (DataInputStream stream = new DataInputStream(new BufferedInputStream(new FileInputStream(descriptor.filenameFor(Component.FILTER)))))
{
bf = FilterFactory.deserialize(stream, true, oldBfHashOrder);
bf = FilterFactory.deserialize(stream, true);
}
}
@ -829,7 +819,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
: estimateRowsFromIndex(primaryIndex); // statistics is supposed to be optional
if (recreateBloomFilter)
bf = FilterFactory.getFilter(estimatedKeys, metadata.params.bloomFilterFpChance, true, descriptor.version.hasOldBfHashOrder());
bf = FilterFactory.getFilter(estimatedKeys, metadata.params.bloomFilterFpChance, true);
try (IndexSummaryBuilder summaryBuilder = summaryLoaded ? null : new IndexSummaryBuilder(estimatedKeys, metadata.params.minIndexInterval, samplingLevel))
{
@ -883,7 +873,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
{
iStream = new DataInputStream(new FileInputStream(summariesFile));
indexSummary = IndexSummary.serializer.deserialize(
iStream, getPartitioner(), descriptor.version.hasSamplingLevel(),
iStream, getPartitioner(),
metadata.params.minIndexInterval, metadata.params.maxIndexInterval);
first = decorateKey(ByteBufferUtil.readWithLength(iStream));
last = decorateKey(ByteBufferUtil.readWithLength(iStream));
@ -932,7 +922,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
try (DataOutputStreamPlus oStream = new BufferedDataOutputStreamPlus(new FileOutputStream(summariesFile));)
{
IndexSummary.serializer.serialize(summary, oStream, descriptor.version.hasSamplingLevel());
IndexSummary.serializer.serialize(summary, oStream);
ByteBufferUtil.writeWithLength(first.getKey(), oStream);
ByteBufferUtil.writeWithLength(last.getKey(), oStream);
}
@ -1106,8 +1096,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
@SuppressWarnings("resource")
public SSTableReader cloneWithNewSummarySamplingLevel(ColumnFamilyStore parent, int samplingLevel) throws IOException
{
assert descriptor.version.hasSamplingLevel();
synchronized (tidy.global)
{
assert openReason != OpenReason.EARLY;

16
src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
View File

@ -127,26 +127,14 @@ public abstract class SSTableWriter extends SSTable implements Transactional
return create(descriptor, keyCount, repairedAt, metadata, collector, header, indexes, txn);
}
public static SSTableWriter create(String filename,
long keyCount,
long repairedAt,
int sstableLevel,
SerializationHeader header,
Collection<Index> indexes,
LifecycleTransaction txn)
{
return create(Descriptor.fromFilename(filename), keyCount, repairedAt, sstableLevel, header, indexes, txn);
}
@VisibleForTesting
public static SSTableWriter create(String filename,
public static SSTableWriter create(Descriptor descriptor,
long keyCount,
long repairedAt,
SerializationHeader header,
Collection<Index> indexes,
LifecycleTransaction txn)
{
Descriptor descriptor = Descriptor.fromFilename(filename);
return create(descriptor, keyCount, repairedAt, 0, header, indexes, txn);
}
@ -157,7 +145,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
Component.STATS,
Component.SUMMARY,
Component.TOC,
Component.digestFor(BigFormat.latestVersion.uncompressedChecksumType())));
Component.DIGEST));
if (metadata.params.bloomFilterFpChance < 1.0)
components.add(Component.FILTER);

22
src/java/org/apache/cassandra/io/sstable/format/Version.java
View File

@ -46,30 +46,8 @@ public abstract class Version
public abstract boolean isLatestVersion();
public abstract boolean hasSamplingLevel();
public abstract boolean hasNewStatsFile();
public abstract ChecksumType compressedChecksumType();
public abstract ChecksumType uncompressedChecksumType();
public abstract boolean hasRepairedAt();
public abstract boolean tracksLegacyCounterShards();
public abstract boolean hasNewFileName();
public abstract boolean storeRows();
public abstract int correspondingMessagingVersion(); // Only use by storage that 'storeRows' so far
public abstract boolean hasOldBfHashOrder();
public abstract boolean hasCompactionAncestors();
public abstract boolean hasBoundaries();
public abstract boolean hasCommitLogLowerBound();
public abstract boolean hasCommitLogIntervals();

125
src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
View File

@ -111,16 +111,8 @@ public class BigFormat implements SSTableFormat
static class BigVersion extends Version
{
public static final String current_version = "mc";
public static final String earliest_supported_version = "jb";
public static final String earliest_supported_version = "ma";
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
// checksum the compressed data
// ka (2.1.0): new Statistics.db file format
// index summaries can be downsampled and the sampling level is persisted
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
// la (2.2.0): new file name format
// lb (2.2.7): commit log lower bound included
// ma (3.0.0): swap bf hash order
// store rows natively
// mb (3.0.7, 3.7): commit log lower bound included
@ -129,62 +121,17 @@ public class BigFormat implements SSTableFormat
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
private final boolean isLatestVersion;
private final boolean hasSamplingLevel;
private final boolean newStatsFile;
private final ChecksumType compressedChecksumType;
private final ChecksumType uncompressedChecksumType;
private final boolean hasRepairedAt;
private final boolean tracksLegacyCounterShards;
private final boolean newFileName;
public final boolean storeRows;
public final int correspondingMessagingVersion; // Only use by storage that 'storeRows' so far
public final boolean hasBoundaries;
/**
* CASSANDRA-8413: 3.0 bloom filter representation changed (two longs just swapped)
* have no 'static' bits caused by using the same upper bits for both bloom filter and token distribution.
*/
private final boolean hasOldBfHashOrder;
public final int correspondingMessagingVersion;
private final boolean hasCommitLogLowerBound;
private final boolean hasCommitLogIntervals;
/**
* CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
*/
private final boolean hasCompactionAncestors;
BigVersion(String version)
{
super(instance, version);
isLatestVersion = version.compareTo(current_version) == 0;
hasSamplingLevel = version.compareTo("ka") >= 0;
newStatsFile = version.compareTo("ka") >= 0;
correspondingMessagingVersion = MessagingService.VERSION_30;
//For a while Adler32 was in use, now the CRC32 instrinsic is very good especially after Haswell
//PureJavaCRC32 was always faster than Adler32. See CASSANDRA-8684
ChecksumType checksumType = ChecksumType.CRC32;
if (version.compareTo("ka") >= 0 && version.compareTo("ma") < 0)
checksumType = ChecksumType.Adler32;
this.uncompressedChecksumType = checksumType;
checksumType = ChecksumType.CRC32;
if (version.compareTo("jb") >= 0 && version.compareTo("ma") < 0)
checksumType = ChecksumType.Adler32;
this.compressedChecksumType = checksumType;
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("la") >= 0;
hasOldBfHashOrder = version.compareTo("ma") < 0;
hasCompactionAncestors = version.compareTo("ma") < 0;
storeRows = version.compareTo("ma") >= 0;
correspondingMessagingVersion = storeRows
? MessagingService.VERSION_30
: MessagingService.VERSION_21;
hasBoundaries = version.compareTo("ma") < 0;
hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma") < 0)
|| version.compareTo("mb") >= 0;
hasCommitLogIntervals = version.compareTo("mc") >= 0;
@ -196,60 +143,6 @@ public class BigFormat implements SSTableFormat
return isLatestVersion;
}
@Override
public boolean hasSamplingLevel()
{
return hasSamplingLevel;
}
@Override
public boolean hasNewStatsFile()
{
return newStatsFile;
}
@Override
public ChecksumType compressedChecksumType()
{
return compressedChecksumType;
}
@Override
public ChecksumType uncompressedChecksumType()
{
return uncompressedChecksumType;
}
@Override
public boolean hasRepairedAt()
{
return hasRepairedAt;
}
@Override
public boolean tracksLegacyCounterShards()
{
return tracksLegacyCounterShards;
}
@Override
public boolean hasOldBfHashOrder()
{
return hasOldBfHashOrder;
}
@Override
public boolean hasCompactionAncestors()
{
return hasCompactionAncestors;
}
@Override
public boolean hasNewFileName()
{
return newFileName;
}
@Override
public boolean hasCommitLogLowerBound()
{
@ -262,24 +155,12 @@ public class BigFormat implements SSTableFormat
return hasCommitLogIntervals;
}
@Override
public boolean storeRows()
{
return storeRows;
}
@Override
public int correspondingMessagingVersion()
{
return correspondingMessagingVersion;
}
@Override
public boolean hasBoundaries()
{
return hasBoundaries;
}
@Override
public boolean isCompatible()
{

6
src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
View File

@ -84,7 +84,7 @@ public class BigTableWriter extends SSTableWriter
{
dataFile = new CompressedSequentialWriter(new File(getFilename()),
descriptor.filenameFor(Component.COMPRESSION_INFO),
new File(descriptor.filenameFor(descriptor.digestComponent)),
new File(descriptor.filenameFor(Component.DIGEST)),
writerOption,
metadata.params.compression,
metadataCollector);
@ -93,7 +93,7 @@ public class BigTableWriter extends SSTableWriter
{
dataFile = new ChecksummedSequentialWriter(new File(getFilename()),
new File(descriptor.filenameFor(Component.CRC)),
new File(descriptor.filenameFor(descriptor.digestComponent)),
new File(descriptor.filenameFor(Component.DIGEST)),
writerOption);
}
dbuilder = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).compressed(compression)
@ -442,7 +442,7 @@ public class BigTableWriter extends SSTableWriter
builder = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).mmapped(DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap);
chunkCache.ifPresent(builder::withChunkCache);
summary = new IndexSummaryBuilder(keyCount, metadata.params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
bf = FilterFactory.getFilter(keyCount, metadata.params.bloomFilterFpChance, true, descriptor.version.hasOldBfHashOrder());
bf = FilterFactory.getFilter(keyCount, metadata.params.bloomFilterFpChance, true);
// register listeners to be alerted when the data files are flushed
indexFile.setPostFlushListener(() -> summary.markIndexSynced(indexFile.getLastFlushOffset()));
dataFile.setPostFlushListener(() -> summary.markDataSynced(dataFile.getLastFlushOffset()));

13
src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
View File

@ -75,30 +75,17 @@ public class CompactionMetadata extends MetadataComponent
public int serializedSize(Version version, CompactionMetadata component) throws IOException
{
int sz = 0;
if (version.hasCompactionAncestors())
{ // write empty ancestor marker
sz = 4;
}
byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length + sz;
}
public void serialize(Version version, CompactionMetadata component, DataOutputPlus out) throws IOException
{
if (version.hasCompactionAncestors())
{ // write empty ancestor marker
out.writeInt(0);
}
ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
}
public CompactionMetadata deserialize(Version version, DataInputPlus in) throws IOException
{
if (version.hasCompactionAncestors())
{ // skip ancestors
int nbAncestors = in.readInt();
in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
}
ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
return new CompactionMetadata(cardinality);
}

163
src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
View File

@ -1,163 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable.metadata;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.*;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.StreamingHistogram;
import static org.apache.cassandra.io.sstable.metadata.StatsMetadata.commitLogPositionSetSerializer;
/**
* Serializer for SSTable from legacy versions
*/
@Deprecated
public class LegacyMetadataSerializer extends MetadataSerializer
{
/**
* Legacy serialization is only used for SSTable level reset.
*/
@Override
public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException
{
ValidationMetadata validation = (ValidationMetadata) components.get(MetadataType.VALIDATION);
StatsMetadata stats = (StatsMetadata) components.get(MetadataType.STATS);
CompactionMetadata compaction = (CompactionMetadata) components.get(MetadataType.COMPACTION);
assert validation != null && stats != null && compaction != null && validation.partitioner != null;
EstimatedHistogram.serializer.serialize(stats.estimatedPartitionSize, out);
EstimatedHistogram.serializer.serialize(stats.estimatedColumnCount, out);
CommitLogPosition.serializer.serialize(stats.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE), out);
out.writeLong(stats.minTimestamp);
out.writeLong(stats.maxTimestamp);
out.writeInt(stats.maxLocalDeletionTime);
out.writeDouble(validation.bloomFilterFPChance);
out.writeDouble(stats.compressionRatio);
out.writeUTF(validation.partitioner);
out.writeInt(0); // compaction ancestors
StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
out.writeInt(stats.sstableLevel);
out.writeInt(stats.minClusteringValues.size());
for (ByteBuffer value : stats.minClusteringValues)
ByteBufferUtil.writeWithShortLength(value, out);
out.writeInt(stats.maxClusteringValues.size());
for (ByteBuffer value : stats.maxClusteringValues)
ByteBufferUtil.writeWithShortLength(value, out);
if (version.hasCommitLogLowerBound())
CommitLogPosition.serializer.serialize(stats.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE), out);
if (version.hasCommitLogIntervals())
commitLogPositionSetSerializer.serialize(stats.commitLogIntervals, out);
}
/**
* Legacy serializer deserialize all components no matter what types are specified.
*/
@Override
public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, EnumSet<MetadataType> types) throws IOException
{
Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
File statsFile = new File(descriptor.filenameFor(Component.STATS));
if (!statsFile.exists() && types.contains(MetadataType.STATS))
{
components.put(MetadataType.STATS, MetadataCollector.defaultStatsMetadata());
}
else
{
try (DataInputStreamPlus in = new DataInputStreamPlus(new BufferedInputStream(new FileInputStream(statsFile))))
{
EstimatedHistogram partitionSizes = EstimatedHistogram.serializer.deserialize(in);
EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(in);
CommitLogPosition commitLogLowerBound = CommitLogPosition.NONE;
CommitLogPosition commitLogUpperBound = CommitLogPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
int maxLocalDeletionTime = in.readInt();
double bloomFilterFPChance = in.readDouble();
double compressionRatio = in.readDouble();
String partitioner = in.readUTF();
int nbAncestors = in.readInt(); //skip compaction ancestors
in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = 0;
if (in.available() > 0)
sstableLevel = in.readInt();
int colCount = in.readInt();
List<ByteBuffer> minColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
minColumnNames.add(ByteBufferUtil.readWithShortLength(in));
colCount = in.readInt();
List<ByteBuffer> maxColumnNames = new ArrayList<>(colCount);
for (int i = 0; i < colCount; i++)
maxColumnNames.add(ByteBufferUtil.readWithShortLength(in));
if (descriptor.version.hasCommitLogLowerBound())
commitLogLowerBound = CommitLogPosition.serializer.deserialize(in);
IntervalSet<CommitLogPosition> commitLogIntervals;
if (descriptor.version.hasCommitLogIntervals())
commitLogIntervals = commitLogPositionSetSerializer.deserialize(in);
else
commitLogIntervals = new IntervalSet<>(commitLogLowerBound, commitLogUpperBound);
if (types.contains(MetadataType.VALIDATION))
components.put(MetadataType.VALIDATION,
new ValidationMetadata(partitioner, bloomFilterFPChance));
if (types.contains(MetadataType.STATS))
components.put(MetadataType.STATS,
new StatsMetadata(partitionSizes,
columnCounts,
commitLogIntervals,
minTimestamp,
maxTimestamp,
Integer.MAX_VALUE,
maxLocalDeletionTime,
0,
Integer.MAX_VALUE,
compressionRatio,
tombstoneHistogram,
sstableLevel,
minColumnNames,
maxColumnNames,
true,
ActiveRepairService.UNREPAIRED_SSTABLE,
-1,
-1));
if (types.contains(MetadataType.COMPACTION))
components.put(MetadataType.COMPACTION,
new CompactionMetadata(null));
}
}
return components;
}
}

44
src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
View File

@ -236,10 +236,7 @@ public class StatsMetadata extends MetadataComponent
size += EstimatedHistogram.serializer.serializedSize(component.estimatedPartitionSize);
size += EstimatedHistogram.serializer.serializedSize(component.estimatedColumnCount);
size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE));
if (version.storeRows())
size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
else
size += 8 + 8 + 4 + 8 + 8; // mix/max timestamp(long), maxLocalDeletionTime(int), compressionRatio(double), repairedAt (long)
size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long)
size += StreamingHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime);
size += TypeSizes.sizeof(component.sstableLevel);
// min column names
@ -251,8 +248,7 @@ public class StatsMetadata extends MetadataComponent
for (ByteBuffer value : component.maxClusteringValues)
size += 2 + value.remaining(); // with short length
size += TypeSizes.sizeof(component.hasLegacyCounterShards);
if (version.storeRows())
size += 8 + 8; // totalColumnsSet, totalRows
size += 8 + 8; // totalColumnsSet, totalRows
if (version.hasCommitLogLowerBound())
size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE));
if (version.hasCommitLogIntervals())
@ -267,14 +263,10 @@ public class StatsMetadata extends MetadataComponent
CommitLogPosition.serializer.serialize(component.commitLogIntervals.upperBound().orElse(CommitLogPosition.NONE), out);
out.writeLong(component.minTimestamp);
out.writeLong(component.maxTimestamp);
if (version.storeRows())
out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.minLocalDeletionTime);
out.writeInt(component.maxLocalDeletionTime);
if (version.storeRows())
{
out.writeInt(component.minTTL);
out.writeInt(component.maxTTL);
}
out.writeInt(component.minTTL);
out.writeInt(component.maxTTL);
out.writeDouble(component.compressionRatio);
StreamingHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out);
out.writeInt(component.sstableLevel);
@ -287,11 +279,8 @@ public class StatsMetadata extends MetadataComponent
ByteBufferUtil.writeWithShortLength(value, out);
out.writeBoolean(component.hasLegacyCounterShards);
if (version.storeRows())
{
out.writeLong(component.totalColumnsSet);
out.writeLong(component.totalRows);
}
out.writeLong(component.totalColumnsSet);
out.writeLong(component.totalRows);
if (version.hasCommitLogLowerBound())
CommitLogPosition.serializer.serialize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE), out);
@ -307,17 +296,14 @@ public class StatsMetadata extends MetadataComponent
commitLogUpperBound = CommitLogPosition.serializer.deserialize(in);
long minTimestamp = in.readLong();
long maxTimestamp = in.readLong();
// We use MAX_VALUE as that's the default value for "no deletion time"
int minLocalDeletionTime = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
int minLocalDeletionTime = in.readInt();
int maxLocalDeletionTime = in.readInt();
int minTTL = version.storeRows() ? in.readInt() : 0;
int maxTTL = version.storeRows() ? in.readInt() : Integer.MAX_VALUE;
int minTTL = in.readInt();
int maxTTL = in.readInt();
double compressionRatio = in.readDouble();
StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
int sstableLevel = in.readInt();
long repairedAt = 0;
if (version.hasRepairedAt())
repairedAt = in.readLong();
long repairedAt = in.readLong();
int colCount = in.readInt();
List<ByteBuffer> minClusteringValues = new ArrayList<>(colCount);
@ -329,12 +315,10 @@ public class StatsMetadata extends MetadataComponent
for (int i = 0; i < colCount; i++)
maxClusteringValues.add(ByteBufferUtil.readWithShortLength(in));
boolean hasLegacyCounterShards = true;
if (version.tracksLegacyCounterShards())
hasLegacyCounterShards = in.readBoolean();
boolean hasLegacyCounterShards = in.readBoolean();
long totalColumnsSet = version.storeRows() ? in.readLong() : -1L;
long totalRows = version.storeRows() ? in.readLong() : -1L;
long totalColumnsSet = in.readLong();
long totalRows = in.readLong();
if (version.hasCommitLogLowerBound())
commitLogLowerBound = CommitLogPosition.serializer.deserialize(in);

5
src/java/org/apache/cassandra/io/util/CompressedChunkReader.java
View File

@ -29,6 +29,7 @@ import org.apache.cassandra.io.compress.BufferType;
import org.apache.cassandra.io.compress.CompressionMetadata;
import org.apache.cassandra.io.compress.CorruptBlockException;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.utils.ChecksumType;
public abstract class CompressedChunkReader extends AbstractReaderFileProxy implements ChunkReader
{
@ -142,7 +143,7 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl
if (getCrcCheckChance() > ThreadLocalRandom.current().nextDouble())
{
compressed.rewind();
int checksum = (int) metadata.checksumType.of(compressed);
int checksum = (int) ChecksumType.CRC32.of(compressed);
compressed.clear().limit(Integer.BYTES);
if (channel.read(compressed, chunk.offset + chunk.length) != Integer.BYTES
@ -204,7 +205,7 @@ public abstract class CompressedChunkReader extends AbstractReaderFileProxy impl
{
compressedChunk.position(chunkOffset).limit(chunkOffset + chunk.length);
int checksum = (int) metadata.checksumType.of(compressedChunk);
int checksum = (int) ChecksumType.CRC32.of(compressedChunk);
compressedChunk.limit(compressedChunk.capacity());
if (compressedChunk.getInt() != checksum)

6
src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java
View File

@ -44,7 +44,7 @@ public class DataIntegrityMetadata
public ChecksumValidator(Descriptor descriptor) throws IOException
{
this(descriptor.version.uncompressedChecksumType(),
this(ChecksumType.CRC32,
RandomAccessReader.open(new File(descriptor.filenameFor(Component.CRC))),
descriptor.filenameFor(Component.DATA));
}
@ -99,8 +99,8 @@ public class DataIntegrityMetadata
public FileDigestValidator(Descriptor descriptor) throws IOException
{
this.descriptor = descriptor;
checksum = descriptor.version.uncompressedChecksumType().newInstance();
digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.digestFor(descriptor.version.uncompressedChecksumType()))));
checksum = ChecksumType.CRC32.newInstance();
digestReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DIGEST)));
dataReader = RandomAccessReader.open(new File(descriptor.filenameFor(Component.DATA)));
try
{

28
src/java/org/apache/cassandra/net/IncomingTcpConnection.java
View File

@ -86,9 +86,9 @@ public class IncomingTcpConnection extends FastThreadLocalThread implements Clos
{
try
{
if (version < MessagingService.VERSION_20)
if (version < MessagingService.VERSION_30)
throw new UnsupportedOperationException(String.format("Unable to read obsolete message version %s; "
+ "The earliest version supported is 2.0.0",
+ "The earliest version supported is 3.0.0",
version));
receiveMessages();
@ -155,18 +155,11 @@ public class IncomingTcpConnection extends FastThreadLocalThread implements Clos
if (compressed)
{
logger.trace("Upgrading incoming connection to be compressed");
if (version < MessagingService.VERSION_21)
{
in = new DataInputStreamPlus(new SnappyInputStream(socket.getInputStream()));
}
else
{
LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
in = new DataInputStreamPlus(new LZ4BlockInputStream(socket.getInputStream(),
decompressor,
checksum));
}
LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
in = new DataInputStreamPlus(new LZ4BlockInputStream(socket.getInputStream(),
decompressor,
checksum));
}
else
{
@ -183,11 +176,8 @@ public class IncomingTcpConnection extends FastThreadLocalThread implements Clos
private InetAddress receiveMessage(DataInputPlus input, int version) throws IOException
{
int id;
if (version < MessagingService.VERSION_20)
id = Integer.parseInt(input.readUTF());
else
id = input.readInt();
int id = input.readInt();
long currentTime = ApproximateTime.currentTimeMillis();
MessageIn message = MessageIn.read(input, version, id, MessageIn.readConstructionTime(from, input, currentTime));
if (message == null)

2
src/java/org/apache/cassandra/net/MessageOut.java
View File

@ -104,7 +104,7 @@ public class MessageOut<T>
{
CompactEndpointSerializationHelper.serialize(from, out);
out.writeInt(MessagingService.Verb.convertForMessagingServiceVersion(verb, version).ordinal());
out.writeInt(verb.ordinal());
out.writeInt(parameters.size());
for (Map.Entry<String, byte[]> entry : parameters.entrySet())
{

73
src/java/org/apache/cassandra/net/MessagingService.java
View File

@ -88,10 +88,6 @@ public final class MessagingService implements MessagingServiceMBean
public static final String MBEAN_NAME = "org.apache.cassandra.net:type=MessagingService";
// 8 bits version, so don't waste versions
public static final int VERSION_12 = 6;
public static final int VERSION_20 = 7;
public static final int VERSION_21 = 8;
public static final int VERSION_22 = 9;
public static final int VERSION_30 = 10;
public static final int current_version = VERSION_30;
@ -105,9 +101,6 @@ public final class MessagingService implements MessagingServiceMBean
*/
public static final int PROTOCOL_MAGIC = 0xCA552DFA;
private boolean allNodesAtLeast22 = true;
private boolean allNodesAtLeast30 = true;
public final MessagingMetrics metrics = new MessagingMetrics();
/* All verb handler identifiers */
@ -236,16 +229,6 @@ public final class MessagingService implements MessagingServiceMBean
UNUSED_5,
;
// This is to support a "late" choice of the verb based on the messaging service version.
// See CASSANDRA-12249 for more details.
public static Verb convertForMessagingServiceVersion(Verb verb, int version)
{
if (verb == PAGED_RANGE && version >= VERSION_30)
return RANGE_SLICE;
return verb;
}
public long getTimeout()
{
return DatabaseDescriptor.getRpcTimeout();
@ -319,9 +302,9 @@ public final class MessagingService implements MessagingServiceMBean
put(Verb.MUTATION, Mutation.serializer);
put(Verb.READ_REPAIR, Mutation.serializer);
put(Verb.READ, ReadCommand.readSerializer);
put(Verb.RANGE_SLICE, ReadCommand.rangeSliceSerializer);
put(Verb.PAGED_RANGE, ReadCommand.pagedRangeSerializer);
put(Verb.READ, ReadCommand.serializer);
put(Verb.RANGE_SLICE, ReadCommand.serializer);
put(Verb.PAGED_RANGE, ReadCommand.serializer);
put(Verb.BOOTSTRAP_TOKEN, BootStrapper.StringSerializer.instance);
put(Verb.REPAIR_MESSAGE, RepairMessage.serializer);
put(Verb.GOSSIP_DIGEST_ACK, GossipDigestAck.serializer);
@ -350,8 +333,8 @@ public final class MessagingService implements MessagingServiceMBean
put(Verb.HINT, HintResponse.serializer);
put(Verb.READ_REPAIR, WriteResponse.serializer);
put(Verb.COUNTER_MUTATION, WriteResponse.serializer);
put(Verb.RANGE_SLICE, ReadResponse.rangeSliceSerializer);
put(Verb.PAGED_RANGE, ReadResponse.rangeSliceSerializer);
put(Verb.RANGE_SLICE, ReadResponse.serializer);
put(Verb.PAGED_RANGE, ReadResponse.serializer);
put(Verb.READ, ReadResponse.serializer);
put(Verb.TRUNCATE, TruncateResponse.serializer);
put(Verb.SNAPSHOT, null);
@ -1041,16 +1024,6 @@ public final class MessagingService implements MessagingServiceMBean
return packed >>> (start + 1) - count & ~(-1 << count);
}
public boolean areAllNodesAtLeast22()
{
return allNodesAtLeast22;
}
public boolean areAllNodesAtLeast30()
{
return allNodesAtLeast30;
}
/**
* @return the last version associated with address, or @param version if this is the first such version
*/
@ -1058,50 +1031,16 @@ public final class MessagingService implements MessagingServiceMBean
{
// We can't talk to someone from the future
version = Math.min(version, current_version);
logger.trace("Setting version {} for {}", version, endpoint);
if (version < VERSION_22)
allNodesAtLeast22 = false;
if (version < VERSION_30)
allNodesAtLeast30 = false;
Integer v = versions.put(endpoint, version);
// if the version was increased to 2.2 or later see if the min version across the cluster has changed
if (v != null && (v < VERSION_30 && version >= VERSION_22))
refreshAllNodeMinVersions();
return v == null ? version : v;
}
public void resetVersion(InetAddress endpoint)
{
logger.trace("Resetting version for {}", endpoint);
Integer removed = versions.remove(endpoint);
if (removed != null && removed <= VERSION_30)
refreshAllNodeMinVersions();
}
private void refreshAllNodeMinVersions()
{
boolean anyNodeLowerThan30 = false;
for (Integer version : versions.values())
{
if (version < MessagingService.VERSION_30)
{
anyNodeLowerThan30 = true;
allNodesAtLeast30 = false;
}
if (version < MessagingService.VERSION_22)
{
allNodesAtLeast22 = false;
return;
}
}
allNodesAtLeast22 = true;
allNodesAtLeast30 = !anyNodeLowerThan30;
versions.remove(endpoint);
}
public int getVersion(InetAddress endpoint)

35
src/java/org/apache/cassandra/net/OutboundTcpConnection.java
View File

@ -336,11 +336,7 @@ public class OutboundTcpConnection extends FastThreadLocalThread
private void writeInternal(MessageOut message, int id, long timestamp) throws IOException
{
out.writeInt(MessagingService.PROTOCOL_MAGIC);
if (targetVersion < MessagingService.VERSION_20)
out.writeUTF(String.valueOf(id));
else
out.writeInt(id);
out.writeInt(id);
// int cast cuts off the high-order half of the timestamp, which we can assume remains
// the same between now and when the recipient reconstructs it.
@ -427,9 +423,7 @@ public class OutboundTcpConnection extends FastThreadLocalThread
int maxTargetVersion = handshakeVersion(in);
if (maxTargetVersion == NO_VERSION)
{
// no version is returned, so disconnect an try again: we will either get
// a different target version (targetVersion < MessagingService.VERSION_12)
// or if the same version the handshake will finally succeed
// no version is returned, so disconnect an try again
logger.trace("Target max version is {}; no version information yet, will retry", maxTargetVersion);
if (DatabaseDescriptor.getSeeds().contains(poolReference.endPoint()))
logger.warn("Seed gossip version is {}; will not connect with that version", maxTargetVersion);
@ -461,22 +455,15 @@ public class OutboundTcpConnection extends FastThreadLocalThread
{
out.flush();
logger.trace("Upgrading OutputStream to {} to be compressed", poolReference.endPoint());
if (targetVersion < MessagingService.VERSION_21)
{
// Snappy is buffered, so no need for extra buffering output stream
out = new WrappedDataOutputStreamPlus(new SnappyOutputStream(socket.getOutputStream()));
}
else
{
// TODO: custom LZ4 OS that supports BB write methods
LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(LZ4_HASH_SEED).asChecksum();
out = new WrappedDataOutputStreamPlus(new LZ4BlockOutputStream(socket.getOutputStream(),
1 << 14, // 16k block size
compressor,
checksum,
true)); // no async flushing
}
// TODO: custom LZ4 OS that supports BB write methods
LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(LZ4_HASH_SEED).asChecksum();
out = new WrappedDataOutputStreamPlus(new LZ4BlockOutputStream(socket.getOutputStream(),
1 << 14, // 16k block size
compressor,
checksum,
true)); // no async flushing
}
logger.debug("Done connecting to {}", poolReference.endPoint());
return true;

27
src/java/org/apache/cassandra/repair/RepairJobDesc.java
View File

@ -93,12 +93,10 @@ public class RepairJobDesc
{
public void serialize(RepairJobDesc desc, DataOutputPlus out, int version) throws IOException
{
if (version >= MessagingService.VERSION_21)
{
out.writeBoolean(desc.parentSessionId != null);
if (desc.parentSessionId != null)
UUIDSerializer.serializer.serialize(desc.parentSessionId, out, version);
}
out.writeBoolean(desc.parentSessionId != null);
if (desc.parentSessionId != null)
UUIDSerializer.serializer.serialize(desc.parentSessionId, out, version);
UUIDSerializer.serializer.serialize(desc.sessionId, out, version);
out.writeUTF(desc.keyspace);
out.writeUTF(desc.columnFamily);
@ -111,11 +109,8 @@ public class RepairJobDesc
public RepairJobDesc deserialize(DataInputPlus in, int version) throws IOException
{
UUID parentSessionId = null;
if (version >= MessagingService.VERSION_21)
{
if (in.readBoolean())
parentSessionId = UUIDSerializer.serializer.deserialize(in, version);
}
if (in.readBoolean())
parentSessionId = UUIDSerializer.serializer.deserialize(in, version);
UUID sessionId = UUIDSerializer.serializer.deserialize(in, version);
String keyspace = in.readUTF();
String columnFamily = in.readUTF();
@ -136,13 +131,9 @@ public class RepairJobDesc
public long serializedSize(RepairJobDesc desc, int version)
{
int size = 0;
if (version >= MessagingService.VERSION_21)
{
size += TypeSizes.sizeof(desc.parentSessionId != null);
if (desc.parentSessionId != null)
size += UUIDSerializer.serializer.serializedSize(desc.parentSessionId, version);
}
int size = TypeSizes.sizeof(desc.parentSessionId != null);
if (desc.parentSessionId != null)
size += UUIDSerializer.serializer.serializedSize(desc.parentSessionId, version);
size += UUIDSerializer.serializer.serializedSize(desc.sessionId, version);
size += TypeSizes.sizeof(desc.keyspace);
size += TypeSizes.sizeof(desc.columnFamily);

2
src/java/org/apache/cassandra/repair/Validator.java
View File

@ -218,7 +218,7 @@ public class Validator implements Runnable
validated++;
// MerkleTree uses XOR internally, so we want lots of output bits here
CountingDigest digest = new CountingDigest(FBUtilities.newMessageDigest("SHA-256"));
UnfilteredRowIterators.digest(null, partition, digest, MessagingService.current_version);
UnfilteredRowIterators.digest(partition, digest, MessagingService.current_version);
// only return new hash for merkle tree in case digest was updated - see CASSANDRA-8979
return digest.count > 0
? new MerkleTree.RowHash(partition.partitionKey().getToken(), digest.digest(), digest.count)

1099
src/java/org/apache/cassandra/schema/LegacySchemaMigrator.java
View File

File diff suppressed because it is too large Load Diff

5
src/java/org/apache/cassandra/service/AbstractReadExecutor.java
View File

@ -106,7 +106,7 @@ public abstract class AbstractReadExecutor
if (traceState != null)
traceState.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
logger.trace("reading {} from {}", readCommand.isDigestQuery() ? "digest" : "data", endpoint);
MessageOut<ReadCommand> message = readCommand.createMessage(MessagingService.instance().getVersion(endpoint));
MessageOut<ReadCommand> message = readCommand.createMessage();
MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
}
@ -291,8 +291,7 @@ public abstract class AbstractReadExecutor
if (traceState != null)
traceState.trace("speculating read retry on {}", extraReplica);
logger.trace("speculating read retry on {}", extraReplica);
int version = MessagingService.instance().getVersion(extraReplica);
MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(version), extraReplica, handler);
MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(), extraReplica, handler);
speculated = true;
cfs.metric.speculativeRetries.inc();

4
src/java/org/apache/cassandra/service/CacheService.java
View File

@ -454,10 +454,6 @@ public class CacheService implements CacheServiceMBean
{
public void serialize(KeyCacheKey key, DataOutputPlus out, ColumnFamilyStore cfs) throws IOException
{
//Don't serialize old format entries since we didn't bother to implement serialization of both for simplicity
//https://issues.apache.org/jira/browse/CASSANDRA-10778
if (!key.desc.version.storeRows()) return;
RowIndexEntry entry = CacheService.instance.keyCache.getInternal(key);
if (entry == null)
return;

28
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -46,7 +46,6 @@ import com.google.common.util.concurrent.Uninterruptibles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
@ -59,14 +58,12 @@ import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.StartupException;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.hints.LegacyHintsMigrator;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.sstable.CorruptSSTableException;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.metrics.CassandraMetricsRegistry;
import org.apache.cassandra.metrics.DefaultNameFactory;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.schema.LegacySchemaMigrator;
import org.apache.cassandra.thrift.ThriftServer;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.*;
@ -205,18 +202,6 @@ public class CassandraDaemon
exitOrFail(e.returnCode, e.getMessage(), e.getCause());
}
try
{
if (SystemKeyspace.snapshotOnVersionChange())
{
SystemKeyspace.migrateDataDirs();
}
}
catch (IOException e)
{
exitOrFail(3, e.getMessage(), e.getCause());
}
// We need to persist this as soon as possible after startup checks.
// This should be the first write to SystemKeyspace (CASSANDRA-11742)
SystemKeyspace.persistLocalMetadata();
@ -249,13 +234,6 @@ public class CassandraDaemon
}
});
/*
* Migrate pre-3.0 keyspaces, tables, types, functions, and aggregates, to their new 3.0 storage.
* We don't (and can't) wait for commit log replay here, but we don't need to - all schema changes force
* explicit memtable flushes.
*/
LegacySchemaMigrator.migrate();
// Populate token metadata before flushing, for token-aware sstable partitioning (#6696)
StorageService.instance.populateTokenMetadata();
@ -333,12 +311,6 @@ public class CassandraDaemon
// Re-populate token metadata after commit log recover (new peers might be loaded onto system keyspace #10293)
StorageService.instance.populateTokenMetadata();
// migrate any legacy (pre-3.0) hints from system.hints table into the new store
new LegacyHintsMigrator(DatabaseDescriptor.getHintsDirectory(), DatabaseDescriptor.getMaxHintsFileSize()).migrate();
// migrate any legacy (pre-3.0) batch entries from system.batchlog to system.batches (new table format)
LegacyBatchlogMigrator.migrate();
// enable auto compaction
for (Keyspace keyspace : Keyspace.all())
{

2
src/java/org/apache/cassandra/service/DataResolver.java
View File

@ -512,7 +512,7 @@ public class DataResolver extends ResponseResolver
if (StorageProxy.canDoLocalRequest(source))
StageManager.getStage(Stage.READ).maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(retryCommand, handler));
else
MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(MessagingService.current_version), source, handler);
MessagingService.instance().sendRRWithFailure(retryCommand.createMessage(), source, handler);
// We don't call handler.get() because we want to preserve tombstones since we're still in the middle of merging node results.
handler.awaitResults();

5
src/java/org/apache/cassandra/service/ReadCallback.java
View File

@ -247,10 +247,7 @@ public class ReadCallback implements IAsyncCallbackWithFailure<ReadResponse>
AsyncRepairCallback repairHandler = new AsyncRepairCallback(repairResolver, endpoints.size());
for (InetAddress endpoint : endpoints)
{
MessageOut<ReadCommand> message = command.createMessage(MessagingService.instance().getVersion(endpoint));
MessagingService.instance().sendRR(message, endpoint, repairHandler);
}
MessagingService.instance().sendRR(command.createMessage(), endpoint, repairHandler);
}
}
}

7
src/java/org/apache/cassandra/service/StartupChecks.java
View File

@ -259,14 +259,15 @@ public class StartupChecks
FileVisitor<Path> sstableVisitor = new SimpleFileVisitor<Path>()
{
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
{
if (!Descriptor.isValidFile(file.getFileName().toString()))
File file = path.toFile();
if (!Descriptor.isValidFile(file))
return FileVisitResult.CONTINUE;
try
{
if (!Descriptor.fromFilename(file.toString()).isCompatible())
if (!Descriptor.fromFilename(file).isCompatible())
invalid.add(file.toString());
}
catch (Exception e)

72
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -42,7 +42,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.batchlog.Batch;
import org.apache.cassandra.batchlog.BatchlogManager;
import org.apache.cassandra.batchlog.LegacyBatchlogMigrator;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
import org.apache.cassandra.config.CFMetaData;
@ -909,10 +908,10 @@ public class StorageProxy implements StorageProxyMBean
batchConsistencyLevel = consistency_level;
}
final BatchlogEndpoints batchlogEndpoints = getBatchlogEndpoints(localDataCenter, batchConsistencyLevel);
final Collection<InetAddress> batchlogEndpoints = getBatchlogEndpoints(localDataCenter, batchConsistencyLevel);
final UUID batchUUID = UUIDGen.getTimeUUID();
BatchlogResponseHandler.BatchlogCleanup cleanup = new BatchlogResponseHandler.BatchlogCleanup(mutations.size(),
() -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID, queryStartNanoTime));
() -> asyncRemoveFromBatchlog(batchlogEndpoints, batchUUID));
// add a handler for each mutation - includes checking availability, but doesn't initiate any writes, yet
for (Mutation mutation : mutations)
@ -969,33 +968,19 @@ public class StorageProxy implements StorageProxyMBean
return replica.equals(FBUtilities.getBroadcastAddress());
}
private static void syncWriteToBatchlog(Collection<Mutation> mutations, BatchlogEndpoints endpoints, UUID uuid, long queryStartNanoTime)
private static void syncWriteToBatchlog(Collection<Mutation> mutations, Collection<InetAddress> endpoints, UUID uuid, long queryStartNanoTime)
throws WriteTimeoutException, WriteFailureException
{
WriteResponseHandler<?> handler = new WriteResponseHandler<>(endpoints.all,
WriteResponseHandler<?> handler = new WriteResponseHandler<>(endpoints,
Collections.<InetAddress>emptyList(),
endpoints.all.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO,
endpoints.size() == 1 ? ConsistencyLevel.ONE : ConsistencyLevel.TWO,
Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME),
null,
WriteType.BATCH_LOG,
queryStartNanoTime);
Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations);
if (!endpoints.current.isEmpty())
syncWriteToBatchlog(handler, batch, endpoints.current);
if (!endpoints.legacy.isEmpty())
LegacyBatchlogMigrator.syncWriteToBatchlog(handler, batch, endpoints.legacy);
handler.get();
}
private static void syncWriteToBatchlog(WriteResponseHandler<?> handler, Batch batch, Collection<InetAddress> endpoints)
throws WriteTimeoutException, WriteFailureException
{
MessageOut<Batch> message = new MessageOut<>(MessagingService.Verb.BATCH_STORE, batch, Batch.serializer);
for (InetAddress target : endpoints)
{
logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, target, batch.size());
@ -1005,15 +990,7 @@ public class StorageProxy implements StorageProxyMBean
else
MessagingService.instance().sendRR(message, target, handler);
}
}
private static void asyncRemoveFromBatchlog(BatchlogEndpoints endpoints, UUID uuid, long queryStartNanoTime)
{
if (!endpoints.current.isEmpty())
asyncRemoveFromBatchlog(endpoints.current, uuid);
if (!endpoints.legacy.isEmpty())
LegacyBatchlogMigrator.asyncRemoveFromBatchlog(endpoints.legacy, uuid, queryStartNanoTime);
handler.get();
}
private static void asyncRemoveFromBatchlog(Collection<InetAddress> endpoints, UUID uuid)
@ -1159,31 +1136,6 @@ public class StorageProxy implements StorageProxyMBean
}
}
/*
* A class to filter batchlog endpoints into legacy endpoints (version < 3.0) or not.
*/
private static final class BatchlogEndpoints
{
public final Collection<InetAddress> all;
public final Collection<InetAddress> current;
public final Collection<InetAddress> legacy;
BatchlogEndpoints(Collection<InetAddress> endpoints)
{
all = endpoints;
current = new ArrayList<>(2);
legacy = new ArrayList<>(2);
for (InetAddress ep : endpoints)
{
if (MessagingService.instance().getVersion(ep) >= MessagingService.VERSION_30)
current.add(ep);
else
legacy.add(ep);
}
}
}
/*
* Replicas are picked manually:
* - replicas should be alive according to the failure detector
@ -1191,7 +1143,7 @@ public class StorageProxy implements StorageProxyMBean
* - choose min(2, number of qualifying candiates above)
* - allow the local node to be the only replica only if it's a single-node DC
*/
private static BatchlogEndpoints getBatchlogEndpoints(String localDataCenter, ConsistencyLevel consistencyLevel)
private static Collection<InetAddress> getBatchlogEndpoints(String localDataCenter, ConsistencyLevel consistencyLevel)
throws UnavailableException
{
TokenMetadata.Topology topology = StorageService.instance.getTokenMetadata().cachedOnlyTokenMap().getTopology();
@ -1202,12 +1154,12 @@ public class StorageProxy implements StorageProxyMBean
if (chosenEndpoints.isEmpty())
{
if (consistencyLevel == ConsistencyLevel.ANY)
return new BatchlogEndpoints(Collections.singleton(FBUtilities.getBroadcastAddress()));
return Collections.singleton(FBUtilities.getBroadcastAddress());
throw new UnavailableException(ConsistencyLevel.ONE, 1, 0);
}
return new BatchlogEndpoints(chosenEndpoints);
return chosenEndpoints;
}
/**
@ -1816,9 +1768,8 @@ public class StorageProxy implements StorageProxyMBean
for (InetAddress endpoint : executor.getContactedReplicas())
{
MessageOut<ReadCommand> message = command.createMessage(MessagingService.instance().getVersion(endpoint));
Tracing.trace("Enqueuing full data read to {}", endpoint);
MessagingService.instance().sendRRWithFailure(message, endpoint, repairHandler);
MessagingService.instance().sendRRWithFailure(command.createMessage(), endpoint, repairHandler);
}
}
}
@ -2218,9 +2169,8 @@ public class StorageProxy implements StorageProxyMBean
{
for (InetAddress endpoint : toQuery.filteredEndpoints)
{
MessageOut<ReadCommand> message = rangeCommand.createMessage(MessagingService.instance().getVersion(endpoint));
Tracing.trace("Enqueuing request to {}", endpoint);
MessagingService.instance().sendRRWithFailure(message, endpoint, handler);
MessagingService.instance().sendRRWithFailure(rangeCommand.createMessage(), endpoint, handler);
}
}

11
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -257,12 +257,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
legacyProgressSupport = new LegacyJMXProgressSupport(this, jmxObjectName);
ReadCommandVerbHandler readHandler = new ReadCommandVerbHandler();
/* register the verb handlers */
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.MUTATION, new MutationVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ_REPAIR, new ReadRepairVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, new ReadCommandVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, new RangeSliceVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAGED_RANGE, new RangeSliceVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.READ, readHandler);
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.RANGE_SLICE, readHandler);
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAGED_RANGE, readHandler);
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.COUNTER_MUTATION, new CounterMutationVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.TRUNCATE, new TruncateVerbHandler());
MessagingService.instance().registerVerbHandlers(MessagingService.Verb.PAXOS_PREPARE, new PrepareVerbHandler());
@ -2082,8 +2084,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public boolean isRpcReady(InetAddress endpoint)
{
return MessagingService.instance().getVersion(endpoint) < MessagingService.VERSION_22 ||
Gossiper.instance.getEndpointStateForEndpoint(endpoint).isRpcReady();
return Gossiper.instance.getEndpointStateForEndpoint(endpoint).isRpcReady();
}
public void setRpcReady(boolean value)

16
src/java/org/apache/cassandra/service/paxos/Commit.java
View File

@ -113,32 +113,20 @@ public class Commit
{
public void serialize(Commit commit, DataOutputPlus out, int version) throws IOException
{
if (version < MessagingService.VERSION_30)
ByteBufferUtil.writeWithShortLength(commit.update.partitionKey().getKey(), out);
UUIDSerializer.serializer.serialize(commit.ballot, out, version);
PartitionUpdate.serializer.serialize(commit.update, out, version);
}
public Commit deserialize(DataInputPlus in, int version) throws IOException
{
ByteBuffer key = null;
if (version < MessagingService.VERSION_30)
key = ByteBufferUtil.readWithShortLength(in);
UUID ballot = UUIDSerializer.serializer.deserialize(in, version);
PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, key);
PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL);
return new Commit(ballot, update);
}
public long serializedSize(Commit commit, int version)
{
int size = 0;
if (version < MessagingService.VERSION_30)
size += ByteBufferUtil.serializedSizeWithShortLength(commit.update.partitionKey().getKey());
return size
+ UUIDSerializer.serializer.serializedSize(commit.ballot, version)
return UUIDSerializer.serializer.serializedSize(commit.ballot, version)
+ PartitionUpdate.serializer.serializedSize(commit.update, version);
}
}

39
src/java/org/apache/cassandra/service/paxos/PrepareResponse.java
View File

@ -69,51 +69,22 @@ public class PrepareResponse
{
out.writeBoolean(response.promised);
Commit.serializer.serialize(response.inProgressCommit, out, version);
if (version < MessagingService.VERSION_30)
{
UUIDSerializer.serializer.serialize(response.mostRecentCommit.ballot, out, version);
PartitionUpdate.serializer.serialize(response.mostRecentCommit.update, out, version);
}
else
{
Commit.serializer.serialize(response.mostRecentCommit, out, version);
}
Commit.serializer.serialize(response.mostRecentCommit, out, version);
}
public PrepareResponse deserialize(DataInputPlus in, int version) throws IOException
{
boolean success = in.readBoolean();
Commit inProgress = Commit.serializer.deserialize(in, version);
Commit mostRecent;
if (version < MessagingService.VERSION_30)
{
UUID ballot = UUIDSerializer.serializer.deserialize(in, version);
PartitionUpdate update = PartitionUpdate.serializer.deserialize(in, version, SerializationHelper.Flag.LOCAL, inProgress.update.partitionKey());
mostRecent = new Commit(ballot, update);
}
else
{
mostRecent = Commit.serializer.deserialize(in, version);
}
Commit mostRecent = Commit.serializer.deserialize(in, version);
return new PrepareResponse(success, inProgress, mostRecent);
}
public long serializedSize(PrepareResponse response, int version)
{
long size = TypeSizes.sizeof(response.promised)
+ Commit.serializer.serializedSize(response.inProgressCommit, version);
if (version < MessagingService.VERSION_30)
{
size += UUIDSerializer.serializer.serializedSize(response.mostRecentCommit.ballot, version);
size += PartitionUpdate.serializer.serializedSize(response.mostRecentCommit.update, version);
}
else
{
size += Commit.serializer.serializedSize(response.mostRecentCommit, version);
}
return size;
return TypeSizes.sizeof(response.promised)
+ Commit.serializer.serializedSize(response.inProgressCommit, version)
+ Commit.serializer.serializedSize(response.mostRecentCommit, version);
}
}
}

11
src/java/org/apache/cassandra/streaming/StreamReader.java
View File

@ -196,16 +196,7 @@ public class StreamReader
long totalSize, UUID sessionId) throws IOException
{
this.metadata = metadata;
// streaming pre-3.0 sstables require mark/reset support from source stream
if (version.correspondingMessagingVersion() < MessagingService.VERSION_30)
{
logger.trace("Initializing rewindable input stream for reading legacy sstable with {} bytes with following " +
"parameters: initial_mem_buffer_size={}, max_mem_buffer_size={}, max_spill_file_size={}.",
totalSize, INITIAL_MEM_BUFFER_SIZE, MAX_MEM_BUFFER_SIZE, MAX_SPILL_FILE_SIZE);
File bufferFile = getTempBufferFile(metadata, totalSize, sessionId);
this.in = new RewindableDataInputStreamPlus(in, INITIAL_MEM_BUFFER_SIZE, MAX_MEM_BUFFER_SIZE, bufferFile, MAX_SPILL_FILE_SIZE);
} else
this.in = new DataInputPlus.DataInputStreamPlus(in);
this.in = new DataInputPlus.DataInputStreamPlus(in);
this.helper = new SerializationHelper(metadata, version.correspondingMessagingVersion(), SerializationHelper.Flag.PRESERVE_SIZE);
this.header = header;
}

3
src/java/org/apache/cassandra/streaming/compress/CompressedStreamReader.java
View File

@ -35,6 +35,7 @@ import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamReader;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.messages.FileMessageHeader;
import org.apache.cassandra.utils.ChecksumType;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@ -81,7 +82,7 @@ public class CompressedStreamReader extends StreamReader
cfs.getColumnFamilyName());
CompressedInputStream cis = new CompressedInputStream(Channels.newInputStream(channel), compressionInfo,
inputVersion.compressedChecksumType(), cfs::getCrcCheckChance);
ChecksumType.CRC32, cfs::getCrcCheckChance);
TrackedInputStream in = new TrackedInputStream(cis);
StreamDeserializer deserializer = new StreamDeserializer(cfs.metadata, in, inputVersion, getHeader(cfs.metadata),

25
src/java/org/apache/cassandra/streaming/messages/FileMessageHeader.java
View File

@ -189,13 +189,7 @@ public class FileMessageHeader
UUIDSerializer.serializer.serialize(header.cfId, out, version);
out.writeInt(header.sequenceNumber);
out.writeUTF(header.version.toString());
//We can't stream to a node that doesn't understand a new sstable format
if (version < StreamMessage.VERSION_22 && header.format != SSTableFormat.Type.LEGACY && header.format != SSTableFormat.Type.BIG)
throw new UnsupportedOperationException("Can't stream non-legacy sstables to nodes < 2.2");
if (version >= StreamMessage.VERSION_22)
out.writeUTF(header.format.name);
out.writeUTF(header.format.name);
out.writeLong(header.estimatedKeys);
out.writeInt(header.sections.size());
@ -212,8 +206,7 @@ public class FileMessageHeader
out.writeLong(header.repairedAt);
out.writeInt(header.sstableLevel);
if (version >= StreamMessage.VERSION_30 && header.version.storeRows())
SerializationHeader.serializer.serialize(header.version, header.header, out);
SerializationHeader.serializer.serialize(header.version, header.header, out);
return compressionInfo;
}
@ -222,10 +215,7 @@ public class FileMessageHeader
UUID cfId = UUIDSerializer.serializer.deserialize(in, MessagingService.current_version);
int sequenceNumber = in.readInt();
Version sstableVersion = SSTableFormat.Type.current().info.getVersion(in.readUTF());
SSTableFormat.Type format = SSTableFormat.Type.LEGACY;
if (version >= StreamMessage.VERSION_22)
format = SSTableFormat.Type.validate(in.readUTF());
SSTableFormat.Type format = SSTableFormat.Type.validate(in.readUTF());
long estimatedKeys = in.readLong();
int count = in.readInt();
@ -235,9 +225,7 @@ public class FileMessageHeader
CompressionInfo compressionInfo = CompressionInfo.serializer.deserialize(in, MessagingService.current_version);
long repairedAt = in.readLong();
int sstableLevel = in.readInt();
SerializationHeader.Component header = version >= StreamMessage.VERSION_30 && sstableVersion.storeRows()
? SerializationHeader.serializer.deserialize(sstableVersion, in)
: null;
SerializationHeader.Component header = SerializationHeader.serializer.deserialize(sstableVersion, in);
return new FileMessageHeader(cfId, sequenceNumber, sstableVersion, format, estimatedKeys, sections, compressionInfo, repairedAt, sstableLevel, header);
}
@ -247,10 +235,7 @@ public class FileMessageHeader
long size = UUIDSerializer.serializer.serializedSize(header.cfId, version);
size += TypeSizes.sizeof(header.sequenceNumber);
size += TypeSizes.sizeof(header.version.toString());
if (version >= StreamMessage.VERSION_22)
size += TypeSizes.sizeof(header.format.name);
size += TypeSizes.sizeof(header.format.name);
size += TypeSizes.sizeof(header.estimatedKeys);
size += TypeSizes.sizeof(header.sections.size());

2
src/java/org/apache/cassandra/streaming/messages/StreamMessage.java
View File

@ -33,8 +33,6 @@ import org.apache.cassandra.streaming.StreamSession;
public abstract class StreamMessage
{
/** Streaming protocol version */
public static final int VERSION_20 = 2;
public static final int VERSION_22 = 3;
public static final int VERSION_30 = 4;
public static final int CURRENT_VERSION = VERSION_30;

9
src/java/org/apache/cassandra/tools/SSTableExport.java
View File

@ -93,8 +93,8 @@ public class SSTableExport
*/
public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException
{
if (!desc.version.storeRows())
throw new IOException("pre-3.0 SSTable is not supported.");
if (!desc.version.isCompatible())
throw new IOException("Cannot process old and unsupported SSTable version.");
EnumSet<MetadataType> types = EnumSet.of(MetadataType.STATS, MetadataType.HEADER);
Map<MetadataType, MetadataComponent> sstableMetadata = desc.getMetadataSerializer().deserialize(desc, types);
@ -162,11 +162,6 @@ public class SSTableExport
: cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
if (Descriptor.isLegacyFile(new File(ssTableFileName)))
{
System.err.println("Unsupported legacy sstable");
System.exit(1);
}
if (!new File(ssTableFileName).exists())
{
System.err.println("Cannot find file " + ssTableFileName);

2
src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
View File

@ -215,7 +215,7 @@ public class SSTableMetadataViewer
try (DataInputStream iStream = new DataInputStream(new FileInputStream(summariesFile)))
{
Pair<DecoratedKey, DecoratedKey> firstLast = new IndexSummary.IndexSummarySerializer().deserializeFirstLastKey(iStream, partitioner, descriptor.version.hasSamplingLevel());
Pair<DecoratedKey, DecoratedKey> firstLast = new IndexSummary.IndexSummarySerializer().deserializeFirstLastKey(iStream, partitioner);
out.printf("First token: %s (key=%s)%n", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey()));
out.printf("Last token: %s (key=%s)%n", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey()));
}

21
src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
View File

@ -82,21 +82,20 @@ public class SSTableRepairedAtSetter
for (String fname: fileNames)
{
Descriptor descriptor = Descriptor.fromFilename(fname);
if (descriptor.version.hasRepairedAt())
if (!descriptor.version.isCompatible())
{
if (setIsRepaired)
{
FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, f.toMillis());
}
else
{
descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
}
System.err.println("SSTable " + fname + " is in a old and unsupported format");
continue;
}
if (setIsRepaired)
{
FileTime f = Files.getLastModifiedTime(new File(descriptor.filenameFor(Component.DATA)).toPath());
descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, f.toMillis());
}
else
{
System.err.println("SSTable " + fname + " does not have repaired property, run upgradesstables");
descriptor.getMetadataSerializer().mutateRepairedAt(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
}
}
}

5
src/java/org/apache/cassandra/tools/StandaloneSplitter.java
View File

@ -70,12 +70,11 @@ public class StandaloneSplitter
continue;
}
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
Descriptor desc = SSTable.tryDescriptorFromFilename(file);
if (desc == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null)
ksName = desc.ksname;

18
src/java/org/apache/cassandra/utils/BloomFilter.java
View File

@ -37,18 +37,12 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
public final IBitSet bitset;
public final int hashCount;
/**
* CASSANDRA-8413: 3.0 (inverted) bloom filters have no 'static' bits caused by using the same upper bits
* for both bloom filter and token distribution.
*/
public final boolean oldBfHashOrder;
BloomFilter(int hashCount, IBitSet bitset, boolean oldBfHashOrder)
BloomFilter(int hashCount, IBitSet bitset)
{
super(bitset);
this.hashCount = hashCount;
this.bitset = bitset;
this.oldBfHashOrder = oldBfHashOrder;
}
private BloomFilter(BloomFilter copy)
@ -56,7 +50,6 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
super(copy);
this.hashCount = copy.hashCount;
this.bitset = copy.bitset;
this.oldBfHashOrder = copy.oldBfHashOrder;
}
public long serializedSize()
@ -101,13 +94,6 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
@Inline
private void setIndexes(long base, long inc, int count, long max, long[] results)
{
if (oldBfHashOrder)
{
long x = inc;
inc = base;
base = x;
}
for (int i = 0; i < count; i++)
{
results[i] = FBUtilities.abs(base % max);
@ -155,7 +141,7 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
public String toString()
{
return "BloomFilter[hashCount=" + hashCount + ";oldBfHashOrder=" + oldBfHashOrder + ";capacity=" + bitset.capacity() + ']';
return "BloomFilter[hashCount=" + hashCount + ";capacity=" + bitset.capacity() + ']';
}
public void addTo(Ref.IdentityCollection identities)

8
src/java/org/apache/cassandra/utils/BloomFilterSerializer.java
View File

@ -38,18 +38,18 @@ final class BloomFilterSerializer
bf.bitset.serialize(out);
}
public static BloomFilter deserialize(DataInput in, boolean oldBfHashOrder) throws IOException
public static BloomFilter deserialize(DataInput in) throws IOException
{
return deserialize(in, false, oldBfHashOrder);
return deserialize(in, false);
}
@SuppressWarnings("resource")
public static BloomFilter deserialize(DataInput in, boolean offheap, boolean oldBfHashOrder) throws IOException
public static BloomFilter deserialize(DataInput in, boolean offheap) throws IOException
{
int hashes = in.readInt();
IBitSet bs = offheap ? OffHeapBitSet.deserialize(in) : OpenBitSet.deserialize(in);
return new BloomFilter(hashes, bs, oldBfHashOrder);
return new BloomFilter(hashes, bs);
}
/**

16
src/java/org/apache/cassandra/utils/FilterFactory.java
View File

@ -40,16 +40,16 @@ public class FilterFactory
BloomFilterSerializer.serialize((BloomFilter) bf, output);
}
public static IFilter deserialize(DataInput input, boolean offheap, boolean oldBfHashOrder) throws IOException
public static IFilter deserialize(DataInput input, boolean offheap) throws IOException
{
return BloomFilterSerializer.deserialize(input, offheap, oldBfHashOrder);
return BloomFilterSerializer.deserialize(input, offheap);
}
/**
* @return A BloomFilter with the lowest practical false positive
* probability for the given number of elements.
*/
public static IFilter getFilter(long numElements, int targetBucketsPerElem, boolean offheap, boolean oldBfHashOrder)
public static IFilter getFilter(long numElements, int targetBucketsPerElem, boolean offheap)
{
int maxBucketsPerElement = Math.max(1, BloomCalculations.maxBucketsPerElement(numElements));
int bucketsPerElement = Math.min(targetBucketsPerElem, maxBucketsPerElement);
@ -58,7 +58,7 @@ public class FilterFactory
logger.warn("Cannot provide an optimal BloomFilter for {} elements ({}/{} buckets per element).", numElements, bucketsPerElement, targetBucketsPerElem);
}
BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement);
return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap, oldBfHashOrder);
return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap);
}
/**
@ -68,21 +68,21 @@ public class FilterFactory
* Asserts that the given probability can be satisfied using this
* filter.
*/
public static IFilter getFilter(long numElements, double maxFalsePosProbability, boolean offheap, boolean oldBfHashOrder)
public static IFilter getFilter(long numElements, double maxFalsePosProbability, boolean offheap)
{
assert maxFalsePosProbability <= 1.0 : "Invalid probability";
if (maxFalsePosProbability == 1.0)
return new AlwaysPresentFilter();
int bucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
BloomCalculations.BloomSpecification spec = BloomCalculations.computeBloomSpec(bucketsPerElement, maxFalsePosProbability);
return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap, oldBfHashOrder);
return createFilter(spec.K, numElements, spec.bucketsPerElement, offheap);
}
@SuppressWarnings("resource")
private static IFilter createFilter(int hash, long numElements, int bucketsPer, boolean offheap, boolean oldBfHashOrder)
private static IFilter createFilter(int hash, long numElements, int bucketsPer, boolean offheap)
{
long numBits = (numElements * bucketsPer) + BITSET_EXCESS;
IBitSet bitset = offheap ? new OffHeapBitSet(numBits) : new OpenBitSet(numBits);
return new BloomFilter(hash, bitset, oldBfHashOrder);
return new BloomFilter(hash, bitset);
}
}

48
src/java/org/apache/cassandra/utils/MerkleTree.java
View File

@ -845,16 +845,6 @@ public class MerkleTree implements Serializable
{
public void serialize(Inner inner, DataOutputPlus out, int version) throws IOException
{
if (version < MessagingService.VERSION_30)
{
if (inner.hash == null)
out.writeInt(-1);
else
{
out.writeInt(inner.hash.length);
out.write(inner.hash);
}
}
Token.serializer.serialize(inner.token, out, version);
Hashable.serializer.serialize(inner.lchild, out, version);
Hashable.serializer.serialize(inner.rchild, out, version);
@ -862,13 +852,6 @@ public class MerkleTree implements Serializable
public Inner deserialize(DataInput in, IPartitioner p, int version) throws IOException
{
if (version < MessagingService.VERSION_30)
{
int hashLen = in.readInt();
byte[] hash = hashLen >= 0 ? new byte[hashLen] : null;
if (hash != null)
in.readFully(hash);
}
Token token = Token.serializer.deserialize(in, p, version);
Hashable lchild = Hashable.serializer.deserialize(in, p, version);
Hashable rchild = Hashable.serializer.deserialize(in, p, version);
@ -877,18 +860,9 @@ public class MerkleTree implements Serializable
public long serializedSize(Inner inner, int version)
{
long size = 0;
if (version < MessagingService.VERSION_30)
{
size += inner.hash == null
? TypeSizes.sizeof(-1)
: TypeSizes.sizeof(inner.hash().length) + inner.hash().length;
}
size += Token.serializer.serializedSize(inner.token, version)
+ Hashable.serializer.serializedSize(inner.lchild, version)
+ Hashable.serializer.serializedSize(inner.rchild, version);
return size;
return Token.serializer.serializedSize(inner.token, version)
+ Hashable.serializer.serializedSize(inner.lchild, version)
+ Hashable.serializer.serializedSize(inner.rchild, version);
}
}
}
@ -938,24 +912,18 @@ public class MerkleTree implements Serializable
{
if (leaf.hash == null)
{
if (version < MessagingService.VERSION_30)
out.writeInt(-1);
else
out.writeByte(-1);
out.writeByte(-1);
}
else
{
if (version < MessagingService.VERSION_30)
out.writeInt(leaf.hash.length);
else
out.writeByte(leaf.hash.length);
out.writeByte(leaf.hash.length);
out.write(leaf.hash);
}
}
public Leaf deserialize(DataInput in, IPartitioner p, int version) throws IOException
{
int hashLen = version < MessagingService.VERSION_30 ? in.readInt() : in.readByte();
int hashLen = in.readByte();
byte[] hash = hashLen < 0 ? null : new byte[hashLen];
if (hash != null)
in.readFully(hash);
@ -964,11 +932,9 @@ public class MerkleTree implements Serializable
public long serializedSize(Leaf leaf, int version)
{
long size = version < MessagingService.VERSION_30 ? TypeSizes.sizeof(1) : 1;
long size = 1;
if (leaf.hash != null)
{
size += leaf.hash().length;
}
return size;
}
}

BIN
test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-CompressionInfo.db
View File

Binary file not shown.

BIN
test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Data.db
View File

Binary file not shown.

1
test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Digest.adler32
View File

@ -1 +0,0 @@
408097082

BIN
test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Filter.db
View File

Binary file not shown.

BIN
test/data/invalid-legacy-sstables/Keyspace1/cf_with_duplicates_2_0/lb-1-big-Index.db
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More