Merge branch cassandra-3.11 into cassandra-4.0

This commit is contained in:
Benjamin Lerer 2021-05-06 11:37:34 +02:00
commit c4fab930ca
178 changed files with 355 additions and 73 deletions

View File

@ -6,6 +6,7 @@ Merged from 3.11:
* Nodetool garbagecollect should retain SSTableLevel for LCS (CASSANDRA-16634)
* Ignore stale acks received in the shadow round (CASSANDRA-16588)
Merged from 3.0:
* Prevent loss of commit log data when moving sstables between nodes (CASSANDRA-16619)
* Fix materialized view builders inserting truncated data (CASSANDRA-16567)
4.0-rc1

View File

@ -1101,8 +1101,7 @@ public final class SystemKeyspace
}
/**
* Read the host ID from the system keyspace, creating (and storing) one if
* none exists.
* Read the host ID from the system keyspace.
*/
public static UUID getLocalHostId()
{
@ -1110,11 +1109,24 @@ public final class SystemKeyspace
UntypedResultSet result = executeInternal(format(req, LOCAL, LOCAL));
// Look up the Host UUID (return it if found)
if (!result.isEmpty() && result.one().has("host_id"))
if (result != null && !result.isEmpty() && result.one().has("host_id"))
return result.one().getUUID("host_id");
return null;
}
/**
* Read the host ID from the system keyspace, creating (and storing) one if
* none exists.
*/
public static synchronized UUID getOrInitializeLocalHostId()
{
UUID hostId = getLocalHostId();
if (hostId != null)
return hostId;
// ID not found, generate a new one, persist, and then return it.
UUID hostId = UUID.randomUUID();
hostId = UUID.randomUUID();
logger.warn("No host ID found, created {} (Note: This should happen exactly once per node).", hostId);
return setLocalHostId(hostId);
}
@ -1122,7 +1134,7 @@ public final class SystemKeyspace
/**
* Sets the local host ID explicitly. Should only be called outside of SystemTable when replacing a node.
*/
public static UUID setLocalHostId(UUID hostId)
public static synchronized UUID setLocalHostId(UUID hostId)
{
String req = "INSERT INTO system.%s (key, host_id) VALUES ('%s', ?)";
executeInternal(format(req, LOCAL, LOCAL), hostId);

View File

@ -44,7 +44,6 @@ import org.apache.cassandra.schema.CompressionParams;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.security.EncryptionContext;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.MBeanWrapper;
@ -190,18 +189,23 @@ public class CommitLog implements CommitLogMBean
*/
public int recoverFiles(File... clogs) throws IOException
{
CommitLogReplayer replayer = CommitLogReplayer.construct(this);
CommitLogReplayer replayer = CommitLogReplayer.construct(this, getLocalHostId());
replayer.replayFiles(clogs);
return replayer.blockForWrites();
}
public void recoverPath(String path) throws IOException
{
CommitLogReplayer replayer = CommitLogReplayer.construct(this);
CommitLogReplayer replayer = CommitLogReplayer.construct(this, getLocalHostId());
replayer.replayPath(new File(path), false);
replayer.blockForWrites();
}
private static UUID getLocalHostId()
{
return Optional.ofNullable(StorageService.instance.getLocalHostUUID()).orElseGet(SystemKeyspace::getLocalHostId);
}
/**
* Perform recovery on a single commit log. Kept w/sub-optimal name due to coupling w/MBean / JMX
*/

View File

@ -51,6 +51,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.WrappedRunnable;
@ -99,7 +100,7 @@ public class CommitLogReplayer implements CommitLogReadHandler
this.commitLogReader = new CommitLogReader();
}
public static CommitLogReplayer construct(CommitLog commitLog)
public static CommitLogReplayer construct(CommitLog commitLog, UUID localHostId)
{
// compute per-CF and global replay intervals
Map<TableId, IntervalSet<CommitLogPosition>> cfPersisted = new HashMap<>();
@ -129,7 +130,7 @@ public class CommitLogReplayer implements CommitLogReadHandler
}
}
IntervalSet<CommitLogPosition> filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt);
IntervalSet<CommitLogPosition> filter = persistedIntervals(cfs.getLiveSSTables(), truncatedAt, localHostId);
cfPersisted.put(cfs.metadata.id, filter);
}
CommitLogPosition globalPosition = firstNotCovered(cfPersisted.values());
@ -285,11 +286,25 @@ public class CommitLogReplayer implements CommitLogReadHandler
* A set of known safe-to-discard commit log replay positions, based on
* the range covered by on disk sstables and those prior to the most recent truncation record
*/
public static IntervalSet<CommitLogPosition> persistedIntervals(Iterable<SSTableReader> onDisk, CommitLogPosition truncatedAt)
public static IntervalSet<CommitLogPosition> persistedIntervals(Iterable<SSTableReader> onDisk,
CommitLogPosition truncatedAt,
UUID localhostId)
{
IntervalSet.Builder<CommitLogPosition> builder = new IntervalSet.Builder<>();
List<String> skippedSSTables = new ArrayList<>();
for (SSTableReader reader : onDisk)
builder.addAll(reader.getSSTableMetadata().commitLogIntervals);
{
UUID originatingHostId = reader.getSSTableMetadata().originatingHostId;
if (originatingHostId != null && originatingHostId.equals(localhostId))
builder.addAll(reader.getSSTableMetadata().commitLogIntervals);
else
skippedSSTables.add(reader.getFilename());
}
if (!skippedSSTables.isEmpty()) {
logger.warn("Origin of {} sstables is unknown or doesn't match the local node; commitLogIntervals for them were ignored", skippedSSTables.size());
logger.debug("Ignored commitLogIntervals from the following sstables: {}", skippedSSTables);
}
if (truncatedAt != null)
builder.add(CommitLogPosition.NONE, truncatedAt);

View File

@ -67,7 +67,7 @@ class ViewBuilder
private final ColumnFamilyStore baseCfs;
private final View view;
private final String ksName;
private final UUID localHostId = SystemKeyspace.getLocalHostId();
private final UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId();
private final Set<Range<Token>> builtRanges = Sets.newConcurrentHashSet();
private final Map<Range<Token>, Pair<Token, Long>> pendingRanges = Maps.newConcurrentMap();
private final Set<ViewBuilderTask> tasks = Sets.newConcurrentHashSet();

View File

@ -115,4 +115,6 @@ public abstract class Version
{
return version != null ? version.hashCode() : 0;
}
public abstract boolean hasOriginatingHostId();
}

View File

@ -112,7 +112,7 @@ public class BigFormat implements SSTableFormat
// we always incremented the major version.
static class BigVersion extends Version
{
public static final String current_version = "na";
public static final String current_version = "nb";
public static final String earliest_supported_version = "ma";
// ma (3.0.0): swap bf hash order
@ -120,8 +120,10 @@ public class BigFormat implements SSTableFormat
// mb (3.0.7, 3.7): commit log lower bound included
// mc (3.0.8, 3.9): commit log intervals included
// md (3.0.18, 3.11.4): corrected sstable min/max clustering
// me (3.0.25, 3.11.11): added hostId of the node from which the sstable originated
// na (4.0.0): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
// na (4.0-rc1): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format
// nb (4.0.0): originating host id
//
// NOTE: when adding a new version, please add that to LegacySSTableTest, too.
@ -130,6 +132,7 @@ public class BigFormat implements SSTableFormat
private final boolean hasCommitLogLowerBound;
private final boolean hasCommitLogIntervals;
private final boolean hasAccurateMinMax;
private final boolean hasOriginatingHostId;
public final boolean hasMaxCompressedLength;
private final boolean hasPendingRepair;
private final boolean hasMetadataChecksum;
@ -151,6 +154,7 @@ public class BigFormat implements SSTableFormat
hasCommitLogLowerBound = version.compareTo("mb") >= 0;
hasCommitLogIntervals = version.compareTo("mc") >= 0;
hasAccurateMinMax = version.compareTo("md") >= 0;
hasOriginatingHostId = version.matches("(m[e-z])|(n[b-z])");
hasMaxCompressedLength = version.compareTo("na") >= 0;
hasPendingRepair = version.compareTo("na") >= 0;
hasIsTransient = version.compareTo("na") >= 0;
@ -216,6 +220,11 @@ public class BigFormat implements SSTableFormat
return isCompatible() && version.charAt(0) == current_version.charAt(0);
}
public boolean hasOriginatingHostId()
{
return hasOriginatingHostId;
}
@Override
public boolean hasMaxCompressedLength()
{

View File

@ -19,7 +19,6 @@ package org.apache.cassandra.io.sstable.metadata;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
@ -27,20 +26,18 @@ import java.util.Map;
import java.util.UUID;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
import com.clearspring.analytics.stream.cardinality.ICardinality;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.partitions.PartitionStatisticsCollector;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.MurmurHash;
import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
@ -89,6 +86,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
-1,
-1,
null,
null,
false);
}
@ -117,10 +115,17 @@ public class MetadataCollector implements PartitionStatisticsCollector
protected ICardinality cardinality = new HyperLogLogPlus(13, 25);
private final ClusteringComparator comparator;
private final UUID originatingHostId;
public MetadataCollector(ClusteringComparator comparator)
{
this.comparator = comparator;
this(comparator, StorageService.instance.getLocalHostUUID());
}
public MetadataCollector(ClusteringComparator comparator, UUID originatingHostId)
{
this.comparator = comparator;
this.originatingHostId = originatingHostId;
}
public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
@ -128,11 +133,14 @@ public class MetadataCollector implements PartitionStatisticsCollector
this(comparator);
IntervalSet.Builder<CommitLogPosition> intervals = new IntervalSet.Builder<>();
for (SSTableReader sstable : sstables)
if (originatingHostId != null)
{
intervals.addAll(sstable.getSSTableMetadata().commitLogIntervals);
for (SSTableReader sstable : sstables)
{
if (originatingHostId.equals(sstable.getSSTableMetadata().originatingHostId))
intervals.addAll(sstable.getSSTableMetadata().commitLogIntervals);
}
}
commitLogIntervals(intervals.build());
sstableLevel(level);
}
@ -265,6 +273,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
isTransient));
components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));

View File

@ -36,10 +36,12 @@ import org.apache.cassandra.io.ISerializer;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.EstimatedHistogram;
import org.apache.cassandra.utils.streamhist.TombstoneHistogram;
import org.apache.cassandra.utils.UUIDSerializer;
import org.apache.cassandra.utils.UUIDSerializer;
/**
* SSTable metadata that always stay on heap.
@ -67,6 +69,7 @@ public class StatsMetadata extends MetadataComponent
public final long repairedAt;
public final long totalColumnsSet;
public final long totalRows;
public final UUID originatingHostId;
public final UUID pendingRepair;
public final boolean isTransient;
// just holds the current encoding stats to avoid allocating - it is not serialized
@ -90,6 +93,7 @@ public class StatsMetadata extends MetadataComponent
long repairedAt,
long totalColumnsSet,
long totalRows,
UUID originatingHostId,
UUID pendingRepair,
boolean isTransient)
{
@ -111,6 +115,7 @@ public class StatsMetadata extends MetadataComponent
this.repairedAt = repairedAt;
this.totalColumnsSet = totalColumnsSet;
this.totalRows = totalRows;
this.originatingHostId = originatingHostId;
this.pendingRepair = pendingRepair;
this.isTransient = isTransient;
this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL);
@ -165,6 +170,7 @@ public class StatsMetadata extends MetadataComponent
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
isTransient);
}
@ -189,6 +195,7 @@ public class StatsMetadata extends MetadataComponent
newRepairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
newPendingRepair,
newIsTransient);
}
@ -219,6 +226,7 @@ public class StatsMetadata extends MetadataComponent
.append(hasLegacyCounterShards, that.hasLegacyCounterShards)
.append(totalColumnsSet, that.totalColumnsSet)
.append(totalRows, that.totalRows)
.append(originatingHostId, that.originatingHostId)
.append(pendingRepair, that.pendingRepair)
.build();
}
@ -245,6 +253,7 @@ public class StatsMetadata extends MetadataComponent
.append(hasLegacyCounterShards)
.append(totalColumnsSet)
.append(totalRows)
.append(originatingHostId)
.append(pendingRepair)
.build();
}
@ -276,6 +285,12 @@ public class StatsMetadata extends MetadataComponent
size += CommitLogPosition.serializer.serializedSize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE));
if (version.hasCommitLogIntervals())
size += commitLogPositionSetSerializer.serializedSize(component.commitLogIntervals);
if (version.hasOriginatingHostId())
{
size += 1; // boolean: is originatingHostId present
if (component.originatingHostId != null)
size += UUIDSerializer.serializer.serializedSize(component.originatingHostId, version.correspondingMessagingVersion());
}
if (version.hasPendingRepair())
{
@ -322,6 +337,18 @@ public class StatsMetadata extends MetadataComponent
CommitLogPosition.serializer.serialize(component.commitLogIntervals.lowerBound().orElse(CommitLogPosition.NONE), out);
if (version.hasCommitLogIntervals())
commitLogPositionSetSerializer.serialize(component.commitLogIntervals, out);
if (version.hasOriginatingHostId())
{
if (component.originatingHostId != null)
{
out.writeByte(1);
UUIDSerializer.serializer.serialize(component.originatingHostId, out, 0);
}
else
{
out.writeByte(0);
}
}
if (version.hasPendingRepair())
{
@ -412,6 +439,10 @@ public class StatsMetadata extends MetadataComponent
else
commitLogIntervals = new IntervalSet<CommitLogPosition>(commitLogLowerBound, commitLogUpperBound);
UUID originatingHostId = null;
if (version.hasOriginatingHostId() && in.readByte() != 0)
originatingHostId = UUIDSerializer.serializer.deserialize(in, 0);
UUID pendingRepair = null;
if (version.hasPendingRepair() && in.readByte() != 0)
{
@ -438,6 +469,7 @@ public class StatsMetadata extends MetadataComponent
repairedAt,
totalColumnsSet,
totalRows,
originatingHostId,
pendingRepair,
isTransient);
}

View File

@ -596,7 +596,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
throw new RuntimeException(e);
}
UUID localHostId = SystemKeyspace.getLocalHostId();
UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId();
if (isReplacingSameAddress())
{
@ -904,7 +904,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
MessagingService.instance().listen();
UUID localHostId = SystemKeyspace.getLocalHostId();
UUID localHostId = SystemKeyspace.getOrInitializeLocalHostId();
if (replacing)
{

View File

@ -136,7 +136,7 @@ public class CounterId implements Comparable<CounterId>
LocalCounterIdHolder()
{
current = new AtomicReference<>(wrap(ByteBufferUtil.bytes(SystemKeyspace.getLocalHostId())));
current = new AtomicReference<>(wrap(ByteBufferUtil.bytes(SystemKeyspace.getOrInitializeLocalHostId())));
}
CounterId get()

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

View File

@ -0,0 +1,8 @@
Summary.db
Statistics.db
CompressionInfo.db
TOC.txt
Index.db
Filter.db
Data.db
Digest.crc32

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
CompressionInfo.db
Index.db
Digest.crc32
Filter.db
Summary.db
TOC.txt

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
CompressionInfo.db
Index.db
Digest.crc32
Filter.db
Summary.db
TOC.txt

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

View File

@ -0,0 +1,8 @@
Data.db
Statistics.db
CompressionInfo.db
Index.db
Digest.crc32
Filter.db
Summary.db
TOC.txt

Some files were not shown because too many files have changed in this diff Show More