diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java index aee1496cb5..cebc174e24 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/AlterTableStatement.java @@ -711,6 +711,9 @@ public abstract class AlterTableStatement extends AlterSchemaStatement params = validateAndUpdateTransactionalMigration(table.isCounter(), table.params, params); + if (table.replicationType().isTracked() && params.memtable.factory().streamFromMemtable()) + throw ire("Cannot use mutation tracking with persistent memtables"); + return keyspace.withSwapped(keyspace.tables.withSwapped(table.withSwapped(params))); } } diff --git a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java index fc0979ff4f..089fbb2416 100644 --- a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java +++ b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java @@ -19,6 +19,7 @@ package org.apache.cassandra.db; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.tracing.Tracing; public class CassandraTableWriteHandler implements TableWriteHandler @@ -31,10 +32,10 @@ public class CassandraTableWriteHandler implements TableWriteHandler } @Override - public void write(PartitionUpdate update, WriteContext context, boolean updateIndexes) + public void write(MutationId mutationId, PartitionUpdate update, WriteContext context, boolean updateIndexes) { CassandraWriteContext ctx = CassandraWriteContext.fromContext(context); Tracing.trace("Adding to {} memtable", update.metadata().name); - cfs.apply(update, ctx, updateIndexes); + cfs.apply(mutationId, update, ctx, updateIndexes); } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index a60d4a01a7..f3f8ef70ec 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -136,6 +136,7 @@ import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.repair.TableRepairManager; import org.apache.cassandra.repair.consistent.admin.CleanupSummary; import org.apache.cassandra.repair.consistent.admin.PendingStat; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.CompactionParams; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; @@ -674,19 +675,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return memtableFactory.streamFromMemtable(); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, ILifecycleTransaction txn) { - return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, null, 0, header, txn); + return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, 0, header, txn); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet commitLogPositions, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, SerializationHeader header, ILifecycleTransaction txn) { - return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, 0, header, txn); + return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, commitLogPositions, 0, header, txn); } - public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn) + public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn) { - return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn); + return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn); } public boolean supportsEarlyOpen() @@ -1498,7 +1499,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner * @param context write context for current update * @param updateIndexes whether secondary indexes should be updated */ - public void apply(PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes) + public void apply(MutationId mutationId, PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes) { long start = nanoTime(); @@ -1508,7 +1509,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner { Memtable mt = data.getMemtableFor(opGroup, commitLogPosition); UpdateTransaction indexer = newUpdateTransaction(update, context, updateIndexes, mt); - long timeDelta = mt.put(update, indexer, opGroup); + long timeDelta = mt.put(mutationId, update, indexer, opGroup); DecoratedKey key = update.partitionKey(); invalidateCachedPartition(key); metric.topWritePartitionFrequency.addSample(key.getKey(), 1); @@ -2419,12 +2420,14 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner return null; List> dataSets = new ArrayList<>(ranges.size()); + CoordinatorLogBoundariesBuilder boundaries = new CoordinatorLogBoundariesBuilder(); IntervalSet.Builder commitLogIntervals = new IntervalSet.Builder(); long keys = 0; for (Range range : ranges) { Memtable.FlushablePartitionSet dataSet = current.getFlushSet(range.left, range.right); dataSets.add(dataSet); + boundaries.addAll(dataSet.coordinatorLogBoundaries()); commitLogIntervals.add(dataSet.commitLogLowerBound(), dataSet.commitLogUpperBound()); keys += dataSet.partitionCount(); } @@ -2438,6 +2441,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner 0, repairSessionID, false, + boundaries.build(), commitLogIntervals.build(), new SerializationHeader(true, firstDataSet.metadata(), diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java new file mode 100644 index 0000000000..7b506bd3d5 --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundaries.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.io.IOException; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.utils.vint.VIntCoding; + +/** + * Max mutation ID present in this SSTable for each coordinator log, to determine whether an SSTable is reconciled or + * not. Once max mutation IDs are reconciled, next compaction can safely mark this SSTabled as repaired. Note that peers + * may have reconciled all mutations included in an SSTable, but {@link StatsMetadata#repairedAt} is dependent on + * compaction timing, so "nodetool repair --validate" may report temporary disagreements on the repaired set. + *

+ * A reference to this class should be treated as immutable. Do not cast to {@link CoordinatorLogBoundariesMap}. + * Iterable over {@link CoordinatorLogId}. + */ +public interface CoordinatorLogBoundaries extends Iterable +{ + int maxOffset(long logId); + MutationId max(long logId); + int size(); + + IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(CoordinatorLogBoundaries boundaries, DataOutputPlus out, int version) throws IOException + { + if (version < MessagingService.VERSION_52) + return; + out.writeUnsignedVInt32(boundaries.size()); + for (long logId : boundaries) + MutationId.serializer.serialize(boundaries.max(logId), out, version); + } + + @Override + public CoordinatorLogBoundaries deserialize(DataInputPlus in, int version) throws IOException + { + if (version < MessagingService.VERSION_52) + return CoordinatorLogBoundaries.NONE; + int size = in.readUnsignedVInt32(); + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(size); + for (int i = 0; i < size; i++) + { + MutationId mutationId = MutationId.serializer.deserialize(in, version); + boundaries.add(mutationId); + } + return boundaries; + } + + @Override + public long serializedSize(CoordinatorLogBoundaries boundaries, int version) + { + if (version < MessagingService.VERSION_52) + return 0; + long size = 0; + size += VIntCoding.computeUnsignedVIntSize(boundaries.size()); + for (long logId : boundaries) + size += MutationId.serializer.serializedSize(boundaries.max(logId), version); + return size; + } + }; + + CoordinatorLogBoundaries NONE = new CoordinatorLogBoundariesMap(); +} diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java new file mode 100644 index 0000000000..f9339057cc --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesBuilder.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.Iterator; + +import javax.annotation.concurrent.NotThreadSafe; + +import com.google.common.collect.Iterators; + +import org.agrona.collections.Long2ObjectHashMap; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.ShortMutationId; + +@NotThreadSafe +public class CoordinatorLogBoundariesBuilder +{ + private static final MutationId NONE = MutationId.none(); + private static final int NONE_OFFSET = NONE.offset(); + private final Long2ObjectHashMap ids; + + public CoordinatorLogBoundariesBuilder() + { + this.ids = new Long2ObjectHashMap<>(); + } + + public CoordinatorLogBoundariesBuilder add(MutationId mutationId) + { + if (mutationId.isNone()) + return this; + long logId = mutationId.logId(); + + MutationId existing = ids.get(logId); + if (existing == null || ShortMutationId.comparator.compare(existing, mutationId) < 0) + ids.put(mutationId.logId(), mutationId); + return this; + } + + public CoordinatorLogBoundariesBuilder addAll(CoordinatorLogBoundaries boundaries) + { + for (long logId : boundaries) + add(boundaries.max(logId)); + return this; + } + + public CoordinatorLogBoundaries build() + { + return new CoordinatorLogBoundaries() + { + @Override + public int maxOffset(long logId) + { + MutationId id = ids.get(logId); + return id == null ? NONE_OFFSET : id.offset(); + } + + @Override + public MutationId max(long logId) + { + return ids.getOrDefault(logId, CoordinatorLogBoundariesBuilder.NONE); + } + + @Override + public int size() + { + return ids.size(); + } + + @Override + public Iterator iterator() + { + return Iterators.unmodifiableIterator(ids.keySet().iterator()); + } + }; + } +} diff --git a/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java new file mode 100644 index 0000000000..b96e9233c1 --- /dev/null +++ b/src/java/org/apache/cassandra/db/CoordinatorLogBoundariesMap.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.Iterator; + +import javax.annotation.concurrent.ThreadSafe; + +import com.google.common.collect.Iterators; + +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.ShortMutationId; +import org.jctools.maps.NonBlockingHashMapLong; + +/** + * A replica can only receive writes from another replica it shares ranges with, and tracked writes are executed by + * coordinators, so this should contain up to (2*RF - 1) keys. + * Consider wrapping value in AtomicReference to avoid false sharing, see: https://trishagee.com/2011/07/22/dissecting_the_disruptor_why_its_so_fast_part_two__magic_cache_line_padding/ + */ +@ThreadSafe +class CoordinatorLogBoundariesMap extends NonBlockingHashMapLong implements MutableCoordinatorLogBoundaries +{ + private static final MutationId NONE = MutationId.none(); + private static final int NONE_OFFSET = NONE.offset(); + + protected CoordinatorLogBoundariesMap(int size) + { + super(size); + } + + protected CoordinatorLogBoundariesMap() + { + super(); + } + + public void add(MutationId mutationId) + { + long logId = mutationId.logId(); + merge(logId, mutationId, (existing, updating) -> { + if (ShortMutationId.comparator.compare(existing, updating) < 0) + return updating; + return existing; + }); + } + + @Override + public int maxOffset(long logId) + { + MutationId id = get(logId); + return id == null ? NONE_OFFSET : id.offset(); + } + + @Override + public MutationId max(long logId) + { + return getOrDefault(logId, NONE); + } + + @Override + public Iterator iterator() + { + return Iterators.unmodifiableIterator(keySet().iterator()); + } +} diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index ff9a52fefe..532b534aa5 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -583,7 +583,7 @@ public class Keyspace } } - cfs.getWriteHandler().write(upd, ctx, updateIndexes); + cfs.getWriteHandler().write(mutation.id(), upd, ctx, updateIndexes); if (requiresViewUpdate) baseComplete.set(currentTimeMillis()); @@ -628,7 +628,7 @@ public class Keyspace continue; } - cfs.getWriteHandler().write(upd, ctx, true); + cfs.getWriteHandler().write(mutation.id(), upd, ctx, true); } } diff --git a/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java b/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java new file mode 100644 index 0000000000..ffd7f0e948 --- /dev/null +++ b/src/java/org/apache/cassandra/db/MutableCoordinatorLogBoundaries.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import org.apache.cassandra.replication.MutationId; + +public interface MutableCoordinatorLogBoundaries extends CoordinatorLogBoundaries +{ + void add(MutationId mutationId); + + default void addAll(CoordinatorLogBoundaries from) + { + for (long logId : from) + { + MutationId max = from.max(logId); + if (!max.isNone()) + add(max); + } + } + + static MutableCoordinatorLogBoundaries create() + { + return new CoordinatorLogBoundariesMap(); + } + + static MutableCoordinatorLogBoundaries create(int size) + { + return new CoordinatorLogBoundariesMap(size); + } +} diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java index c6361d6c72..1d33e0762a 100644 --- a/src/java/org/apache/cassandra/db/SSTableImporter.java +++ b/src/java/org/apache/cassandra/db/SSTableImporter.java @@ -45,6 +45,7 @@ import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableFormat.Components; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.OutputHandler; @@ -80,6 +81,11 @@ public class SSTableImporter UUID importID = UUID.randomUUID(); logger.info("[{}] Loading new SSTables for {}/{}: {}", importID, cfs.getKeyspaceName(), cfs.getTableName(), options); + // This will be supported in the future + TableMetadata metadata = cfs.metadata(); + if (metadata.replicationType() != null && metadata.replicationType().isTracked()) + throw new IllegalStateException("Can't import into tables with mutation tracking enabled"); + List> listers = getSSTableListers(options.srcPaths); Set currentDescriptors = new HashSet<>(); @@ -110,7 +116,7 @@ public class SSTableImporter { IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, cfs.getPartitioner(), - cfs.metadata().comparator); + metadata.comparator); String keyspace = cfs.getKeyspaceName(); String table = cfs.getTableName(); diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/src/java/org/apache/cassandra/db/TableWriteHandler.java index 7aa709fe52..4cb4d0e7ba 100644 --- a/src/java/org/apache/cassandra/db/TableWriteHandler.java +++ b/src/java/org/apache/cassandra/db/TableWriteHandler.java @@ -19,8 +19,9 @@ package org.apache.cassandra.db; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.replication.MutationId; public interface TableWriteHandler { - void write(PartitionUpdate update, WriteContext context, boolean updateIndexes); + void write(MutationId mutationId, PartitionUpdate update, WriteContext context, boolean updateIndexes); } diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java index b93b5bf66b..7396f39b11 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java @@ -560,6 +560,7 @@ public abstract class AbstractCompactionStrategy long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -571,6 +572,7 @@ public abstract class AbstractCompactionStrategy repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, cfs.metadata, commitLogPositions, sstableLevel, diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java index 4bc7146f83..e4a5427f42 100644 --- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java @@ -28,6 +28,7 @@ import java.util.function.Supplier; import com.google.common.base.Preconditions; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -196,6 +197,7 @@ public abstract class AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index ea370b4601..9104f9fef2 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -70,6 +70,8 @@ import org.apache.cassandra.concurrent.ExecutorFactory; import org.apache.cassandra.concurrent.WrappedExecutorPlus; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -1650,7 +1652,8 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan CompactionIterator ci = new CompactionIterator(OperationType.CLEANUP, Collections.singletonList(scanner), controller, nowInSec, nextTimeUUID(), active, null)) { StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, txn)); + // TODO(aratnofsky): filter coordinatorLogBoundaries to exclude any CoordinatorLogIds we're no longer responsible for, after ownership change + writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, txn)); long lastBytesScanned = 0; while (ci.hasNext()) @@ -1815,6 +1818,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableReader sstable, LifecycleTransaction txn) { @@ -1826,6 +1830,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) .setTransientSSTable(isTransient) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel())) .setSerializationHeader(sstable.header) @@ -1846,14 +1851,16 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan { FileUtils.createDirectory(compactionFileLocation); int minLevel = Integer.MAX_VALUE; - // if all sstables have the same level, we can compact them together without creating overlap during anticompaction - // note that we only anticompact from unrepaired sstables, which is not leveled, but we still keep original level - // after first migration to be able to drop the sstables back in their original place in the repaired sstable manifest + CoordinatorLogBoundariesBuilder boundaries = new CoordinatorLogBoundariesBuilder(); for (SSTableReader sstable : sstables) { + boundaries.addAll(sstable.getCoordinatorLogBoundaries()); + + // if all sstables have the same level, we can compact them together without creating overlap during anticompaction + // note that we only anticompact from unrepaired sstables, which is not leveled, but we still keep original level + // after first migration to be able to drop the sstables back in their original place in the repaired sstable manifest if (minLevel == Integer.MAX_VALUE) minLevel = sstable.getSSTableLevel(); - if (minLevel != sstable.getSSTableLevel()) { minLevel = 0; @@ -1866,6 +1873,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan .setKeyCount(expectedBloomFilterSize) .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(boundaries.build()) .setTransientSSTable(isTransient) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator).sstableLevel(minLevel)) diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java index af5c749929..53285c0f8a 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -226,6 +227,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -251,6 +253,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index d73a3ee6e7..58097fcc7d 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; import org.apache.cassandra.db.SerializationHeader; @@ -1411,6 +1412,7 @@ public class CompactionStrategyManager implements INotificationConsumer long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -1427,6 +1429,7 @@ public class CompactionStrategyManager implements INotificationConsumer repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java index 7336c4543a..e3c25b4433 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java @@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.WriteContext; @@ -458,6 +460,13 @@ public class CompactionTask extends AbstractCompactionTask return isTransient; } + public static CoordinatorLogBoundaries getCoordinatorLogBoundaries(Set sstables) + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (SSTableReader sstable : sstables) + builder.addAll(sstable.getCoordinatorLogBoundaries()); + return builder.build(); + } /* * Checks if we have enough disk space to execute the compaction. Drops the largest sstable out of the Task until diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java index 45e8753394..6716452c95 100644 --- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java +++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -246,6 +247,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -263,6 +265,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, sstableLevel, header, diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java index f548e98a6a..16977ff49a 100644 --- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java @@ -300,6 +300,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, int sstableLevel, SerializationHeader header, @@ -317,6 +318,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, commitLogPositions, header, indexGroups, diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java index 22913a84f6..26664da355 100644 --- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java +++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java @@ -80,6 +80,7 @@ public class Upgrader .setRepairedAt(metadata.repairedAt) .setPendingRepair(metadata.pendingRepair) .setTransientSSTable(metadata.isTransient) + .setCoordinatorLogBoundaries(metadata.coordinatorLogBoundaries) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(sstableMetadataCollector) .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable))) diff --git a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java index b0d3f6ec9c..60d9deedc0 100644 --- a/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -61,6 +62,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter private final long repairedAt; private final TimeUUID pendingRepair; private final boolean isTransient; + private final CoordinatorLogBoundaries coordinatorLogBoundaries; private final IntervalSet commitLogPositions; private final SerializationHeader header; private final Collection indexGroups; @@ -75,6 +77,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, IntervalSet commitLogPositions, SerializationHeader header, Collection indexGroups, @@ -87,6 +90,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.commitLogPositions = commitLogPositions; this.header = header; this.indexGroups = indexGroups; @@ -112,6 +116,7 @@ public class ShardedMultiWriter implements SSTableMultiWriter .setKeyCount(forSplittingKeysBy(boundaries.count())) .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTransientSSTable(isTransient) .setTableMetadataRef(cfs.metadata) .setMetadataCollector(metadataCollector) diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java index 2369b3cbb7..03ff8f35d6 100644 --- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java +++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -61,6 +62,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa protected final long minRepairedAt; protected final TimeUUID pendingRepair; protected final boolean isTransient; + protected final CoordinatorLogBoundaries coordinatorLogBoundaries; protected final SSTableRewriter sstableWriter; protected final ILifecycleTransaction txn; @@ -96,6 +98,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables); pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables); isTransient = CompactionTask.getIsTransient(nonExpiredSSTables); + coordinatorLogBoundaries = CompactionTask.getCoordinatorLogBoundaries(nonExpiredSSTables); DiskBoundaries db = cfs.getDiskBoundaries(); diskBoundaries = db.positions; locations = db.directories; @@ -328,6 +331,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa .setTransientSSTable(isTransient) .setRepairedAt(minRepairedAt) .setPendingRepair(pendingRepair) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .addDefaultComponents(cfs.indexManager.listIndexGroups()) .setCompressionDictionaryManager(cfs.compressionDictionaryManager()); diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java index b439438764..bf42ebea0c 100644 --- a/src/java/org/apache/cassandra/db/memtable/Flushing.java +++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java @@ -244,6 +244,7 @@ public class Flushing ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, + flushSet.coordinatorLogBoundaries(), new IntervalSet<>(flushSet.commitLogLowerBound(), flushSet.commitLogUpperBound()), new SerializationHeader(true, diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java index ce9992b91e..ae7bfdb123 100644 --- a/src/java/org/apache/cassandra/db/memtable/Memtable.java +++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java @@ -26,6 +26,7 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.cassandra.db.CellSourceIdentifier; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -37,6 +38,7 @@ import org.apache.cassandra.db.rows.UnfilteredSource; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.format.SSTableWriter; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.FBUtilities; @@ -183,9 +185,9 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo // Main write and read operations - default long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) + default long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) { - return put(update, indexer, opGroup, false); + return put(mutationId, update, indexer, opGroup, false); } /** @@ -202,7 +204,7 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo * timestamp delta being computed as the difference between the cells and DeletionTimes from any existing partition * and those in {@code update}. See CASSANDRA-7979. */ - long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing); + long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing); // Read operations are provided by the UnfilteredSource interface. @@ -334,6 +336,9 @@ public interface Memtable extends Comparable, UnfilteredSource, CellSo /** Statistics required for writing an sstable efficiently */ EncodingStats encodingStats(); + /** The boundaries in coordinator logs for all included tracked mutations */ + CoordinatorLogBoundaries coordinatorLogBoundaries(); + default TableMetadata metadata() { return memtable().metadata(); diff --git a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java index 68ac5dfa0c..bf342d96b9 100644 --- a/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/ShardedSkipListMemtable.java @@ -32,8 +32,11 @@ import org.github.jamm.Unmetered; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.Slices; @@ -55,6 +58,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -139,11 +143,11 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable * * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; - return shard.put(key, update, indexer, opGroup, assumeMissing); + return shard.put(mutationId, key, update, indexer, opGroup, assumeMissing); } /** @@ -287,7 +291,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable int keyCount = 0; TableMetadata currentTableMetadata = metadata(); - for (Iterator it = getPartitionIterator(from, true, to,false); it.hasNext();) + for (Iterator it = getPartitionIterator(from, true, to, false); it.hasNext(); ) { AtomicBTreePartition en = it.next(); keySize += en.partitionKey().getKey().remaining(); @@ -295,7 +299,15 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable } long partitionKeySize = keySize; int partitionCount = keyCount; - Iterator toFlush = getPartitionIterator(from, true, to,false); + Iterator toFlush = getPartitionIterator(from, true, to, false); + + CoordinatorLogBoundaries flushableBoundaries; + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MemtableShard shard : shards) + builder.addAll(shard.coordinatorLogBoundaries); + flushableBoundaries = builder.build(); + } return new AbstractFlushablePartitionSet() { @@ -336,6 +348,12 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } @@ -361,6 +379,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable private final ColumnsCollector columnsCollector; private final StatsCollector statsCollector; + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); @Unmetered // total pool size should not be included in memtable's deep size private final MemtableAllocator allocator; @@ -376,7 +395,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable this.metadata = metadata; } - public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { Cloner cloner = allocator.cloner(opGroup); AtomicBTreePartition previous = assumeMissing ? null : partitions.get(key); @@ -405,6 +424,7 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable liveDataSize.addAndGet(initialSize + updater.dataSize); columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); currentOperations.addAndGet(update.operationCount()); return updater.colUpdateTimeDelta; } @@ -514,13 +534,13 @@ public class ShardedSkipListMemtable extends AbstractShardedMemtable * * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; synchronized (shard) { - return shard.put(key, update, indexer, opGroup, assumeMissing); + return shard.put(mutationId, key, update, indexer, opGroup, assumeMissing); } } diff --git a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java index 1389b1954b..ea138f6528 100644 --- a/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/SkipListMemtable.java @@ -32,8 +32,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -54,6 +57,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.sstable.SSTableReadsListener; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.ObjectSizes; @@ -86,6 +90,8 @@ public class SkipListMemtable extends AbstractAllocatorMemtable // actually only store DecoratedKey. private final ConcurrentNavigableMap partitions = new ConcurrentSkipListMap<>(); + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); + private final AtomicLong liveDataSize = new AtomicLong(0); protected SkipListMemtable(AtomicReference commitLogLowerBound, TableMetadataRef metadataRef, Owner owner) @@ -115,7 +121,7 @@ public class SkipListMemtable extends AbstractAllocatorMemtable * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { long initialSize = 0; Cloner cloner = allocator.cloner(opGroup); @@ -143,6 +149,7 @@ public class SkipListMemtable extends AbstractAllocatorMemtable liveDataSize.addAndGet(initialSize + updater.dataSize); columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); currentOperations.addAndGet(update.operationCount()); return updater.colUpdateTimeDelta; } @@ -288,6 +295,9 @@ public class SkipListMemtable extends AbstractAllocatorMemtable } final long partitionKeysSize = keysSize; final long partitionCount = keyCount; + CoordinatorLogBoundaries flushableBoundaries = new CoordinatorLogBoundariesBuilder() + .addAll(coordinatorLogBoundaries) + .build(); return new AbstractFlushablePartitionSet() { @@ -334,6 +344,12 @@ public class SkipListMemtable extends AbstractAllocatorMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } diff --git a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java index 29ab18ee6f..ccf10c9bed 100644 --- a/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java +++ b/src/java/org/apache/cassandra/db/memtable/TrieMemtable.java @@ -38,9 +38,12 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; +import org.apache.cassandra.db.CoordinatorLogBoundariesBuilder; import org.apache.cassandra.db.DataRange; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.Slices; @@ -69,6 +72,7 @@ import org.apache.cassandra.index.transactions.UpdateTransaction; import org.apache.cassandra.io.compress.BufferType; import org.apache.cassandra.io.sstable.SSTableReadsListener; import org.apache.cassandra.metrics.TrieMemtableMetricsView; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.utils.Clock; @@ -183,13 +187,13 @@ public class TrieMemtable extends AbstractShardedMemtable * commitLogSegmentPosition should only be null if this is a secondary index, in which case it is *expected* to be null */ @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, boolean assumeMissing) { try { DecoratedKey key = update.partitionKey(); MemtableShard shard = shards[boundaries.getShardForKey(key)]; - long colUpdateTimeDelta = shard.put(key, update, indexer, opGroup); + long colUpdateTimeDelta = shard.put(mutationId, key, update, indexer, opGroup); if (shard.data.reachedAllocatedSizeThreshold() && !switchRequested.getAndSet(true)) { @@ -447,6 +451,14 @@ public class TrieMemtable extends AbstractShardedMemtable partitionKeySize = keySize; partitionCount = keyCount; + CoordinatorLogBoundaries flushableBoundaries; + { + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MemtableShard shard : shards) + builder.addAll(shard.coordinatorLogBoundaries); + flushableBoundaries = builder.build(); + } + return new AbstractFlushablePartitionSet() { private final TableMetadata tableMetadata = TrieMemtable.this.metadata(); @@ -489,6 +501,12 @@ public class TrieMemtable extends AbstractShardedMemtable { return tableMetadata; } + + @Override + public CoordinatorLogBoundaries coordinatorLogBoundaries() + { + return flushableBoundaries; + } }; } @@ -530,6 +548,7 @@ public class TrieMemtable extends AbstractShardedMemtable private final ColumnsCollector columnsCollector; private final StatsCollector statsCollector; + private final MutableCoordinatorLogBoundaries coordinatorLogBoundaries = MutableCoordinatorLogBoundaries.create(); @Unmetered // total pool size should not be included in memtable's deep size private final MemtableAllocator allocator; @@ -547,7 +566,7 @@ public class TrieMemtable extends AbstractShardedMemtable this.metrics = metrics; } - public long put(DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException + public long put(MutationId mutationId, DecoratedKey key, PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup) throws InMemoryTrie.SpaceExhaustedException { BTreePartitionUpdater updater = new BTreePartitionUpdater(allocator, allocator.cloner(opGroup), opGroup, indexer); boolean locked = writeLock.tryLock(); @@ -586,6 +605,7 @@ public class TrieMemtable extends AbstractShardedMemtable columnsCollector.update(update.columns()); statsCollector.update(update.stats()); + coordinatorLogBoundaries.add(mutationId); } } finally diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java index 7482ab0ff2..5f989c481d 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java @@ -74,7 +74,7 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance)) { TrackedDataInputPlus in = new TrackedDataInputPlus(cis); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, inputVersion.format); deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()), session, writer); String filename = writer.getFilename(); String sectionName = filename + '-' + fileSeqNum; diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index f4f79a780c..91047eaa0e 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -151,6 +152,12 @@ public class CassandraOutgoingFile implements OutgoingStream return ref.get().getPendingRepair(); } + @Override + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return ref.get().getCoordinatorLogBoundaries(); + } + @Override public void write(StreamSession session, StreamingDataOutputPlus out, int version) throws IOException { diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index 9a6836d34a..53f25b5a2b 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Directories; @@ -87,6 +88,7 @@ public class CassandraStreamReader implements IStreamReader protected final Version inputVersion; protected final long repairedAt; protected final TimeUUID pendingRepair; + protected final CoordinatorLogBoundaries coordinatorLogBoundaries; protected final int sstableLevel; protected final SerializationHeader.Component header; protected final int fileSeqNum; @@ -106,6 +108,7 @@ public class CassandraStreamReader implements IStreamReader this.inputVersion = streamHeader.version; this.repairedAt = header.repairedAt; this.pendingRepair = header.pendingRepair; + this.coordinatorLogBoundaries = header.coordinatorLogBoundaries; this.sstableLevel = streamHeader.sstableLevel; this.header = streamHeader.serializationHeader; this.fileSeqNum = header.sequenceNumber; @@ -135,7 +138,7 @@ public class CassandraStreamReader implements IStreamReader try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version)) { TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream); - writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, inputVersion.format); + writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, inputVersion.format); deserializer = getDeserializer(cfs.metadata(), in, inputVersion, session, writer); String sequenceName = writer.getFilename() + '-' + fileSeqNum; long lastBytesRead = 0; @@ -176,7 +179,7 @@ public class CassandraStreamReader implements IStreamReader { return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader } - protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat format) throws IOException + protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format) throws IOException { Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize); if (localDir == null) @@ -185,7 +188,7 @@ public class CassandraStreamReader implements IStreamReader StreamReceiver streamReceiver = session.getAggregator(tableId); Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver); ILifecycleTransaction txn = createTxn(); - RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); + RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, coordinatorLogBoundaries, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata())); return new SSTableTxnSingleStreamWriter(txn, writer); } diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java index 895015bc93..373ce67d18 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java @@ -76,6 +76,7 @@ import org.apache.cassandra.index.internal.keys.KeysSearcher; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -544,7 +545,7 @@ public abstract class CassandraIndex implements Index cell)); Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info); PartitionUpdate upd = partitionUpdate(valueKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.trace("Inserted entry into index for value {}", valueKey); } @@ -590,7 +591,7 @@ public abstract class CassandraIndex implements Index { Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion)); PartitionUpdate upd = partitionUpdate(indexKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.trace("Removed index entry for value {}", indexKey); } diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java index 2487e66fd2..f491539387 100644 --- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java @@ -30,7 +30,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Stream; +import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.RegularAndStaticColumns; import org.apache.cassandra.db.SerializationHeader; @@ -69,6 +71,9 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable indexGroups = new ArrayList<>(); } + @VisibleForTesting + public abstract long bytesWritten(); + protected void setSSTableFormatType(SSTableFormat type) { this.format = type; @@ -117,10 +122,15 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable protected SSTableTxnWriter createWriter(SSTable.Owner owner) throws IOException { + // This will prevent cassandra-analytics from producing SSTables for tables with mutation tracking enabled. + // We'll eventually support this with coordinated nodetool import + if (metadata.get().keyspaceReplicationType.isTracked()) + throw new IllegalStateException("Can't create writer for table with mutation tracking enabled"); + SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS); if (makeRangeAware) - return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, header); + return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, format, header); SSTable.Owner effectiveOwner; @@ -142,6 +152,7 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, + CoordinatorLogBoundaries.NONE, header, indexGroups, effectiveOwner); diff --git a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java index 552da199ca..5134f2e6b6 100644 --- a/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/CQLSSTableWriter.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; @@ -161,6 +162,12 @@ public class CQLSSTableWriter implements Closeable .collect(Collectors.toList()); } + @VisibleForTesting + public long bytesWritten() + { + return writer.bytesWritten(); + } + /** * Returns a new builder for a CQLSSTableWriter. * diff --git a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java index 1db8ef679d..b5c0266748 100644 --- a/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.DiskBoundaries; @@ -45,6 +46,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final long repairedAt; private final TimeUUID pendingRepair; private final boolean isTransient; + private final CoordinatorLogBoundaries coordinatorLogBoundaries; private final SSTableFormat format; private final SerializationHeader header; private final ILifecycleTransaction txn; @@ -53,7 +55,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter private final List finishedWriters = new ArrayList<>(); private SSTableMultiWriter currentWriter = null; - public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SSTableFormat format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException + public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException { DiskBoundaries db = cfs.getDiskBoundaries(); directories = db.directories; @@ -63,6 +65,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.format = format; this.txn = txn; this.header = header; @@ -74,7 +77,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter throw new IOException(String.format("Insufficient disk space to store %s", FBUtilities.prettyPrintMemory(totalSize))); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, sstableLevel, header, txn); } } @@ -96,7 +99,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter finishedWriters.add(currentWriter); Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format); - currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, txn); + currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, null, sstableLevel, header, txn); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java index 8c9b2f9796..7a15e779d5 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java @@ -61,6 +61,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter private Buffer buffer = new Buffer(); private final long maxSStableSizeInBytes; private long currentSize; + private long syncedSize; // Used to compute the row serialized size private final SerializationHeader header; @@ -100,6 +101,12 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter return previous; } + @Override + public long bytesWritten() + { + return currentSize + syncedSize; + } + private void countRow(Row row) { // Note that the accounting of a row is a bit inaccurate (it doesn't take some of the file format optimization into account) @@ -164,6 +171,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter put(buffer); buffer = new Buffer(); + syncedSize += currentSize; currentSize = 0; } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java index 7b5fb8a89e..0420190eb2 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java @@ -52,6 +52,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter protected PartitionUpdate.Builder update; private SSTableTxnWriter writer; + private long flushedBytes; /** * Create a SSTable writer for sorted input data. @@ -160,6 +161,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter if (writer == null) return; + flushedBytes += writer.getOnDiskBytesWritten(); Collection finished = writer.finish(shouldOpenSSTables()); notifySSTableProduced(finished); } @@ -175,4 +177,10 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter { getOrCreateWriter().append(update.unfilteredIterator()); } + + @Override + public long bytesWritten() + { + return flushedBytes + (writer != null ? writer.getOnDiskBytesWritten() : 0); + } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java index 3b43dcfdf4..cdd7e3fbe3 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Collection; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; @@ -76,6 +77,11 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem return writer.getOnDiskBytesWritten(); } + public long getBytesWritten() + { + return writer.getBytesWritten(); + } + protected Throwable doCommit(Throwable accumulate) { return writer.commit(txn.commit(accumulate)); @@ -108,10 +114,10 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem } @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup - public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header) + public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header) { LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, header, txn); + SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, header, txn); return new SSTableTxnWriter(txn, writer); } @@ -120,6 +126,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat type, SerializationHeader header) { @@ -129,7 +136,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem SSTableMultiWriter writer; try { - writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, 0, 0, txn, header); + writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, type, 0, 0, txn, header); } catch (IOException e) { @@ -148,13 +155,14 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, Collection indexGroups, SSTable.Owner owner) { // if the column family store does not exist, we create a new default SSTableMultiWriter to use: LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE); - SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, 0, header, indexGroups, txn, owner); + SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogBoundaries, metadata, null, 0, header, indexGroups, txn, owner); return new SSTableTxnWriter(txn, writer); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java index 482b4cf071..38c1e9aae6 100644 --- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.util.Collection; import java.util.Collections; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -117,6 +118,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter long repairedAt, TimeUUID pendingRepair, boolean isTransient, + CoordinatorLogBoundaries coordinatorLogBoundaries, TableMetadataRef metadata, IntervalSet commitLogPositions, int sstableLevel, @@ -136,6 +138,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter .setRepairedAt(repairedAt) .setPendingRepair(pendingRepair) .setTransientSSTable(isTransient) + .setCoordinatorLogBoundaries(coordinatorLogBoundaries) .setTableMetadataRef(metadata) .setMetadataCollector(metadataCollector) .setSerializationHeader(header) diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index ab3171200f..6de1abc839 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -58,6 +58,7 @@ import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.CassandraRelevantProperties; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.SerializationHeader; @@ -1229,6 +1230,11 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource, return sstableMetadata.pendingRepair; } + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return sstableMetadata.coordinatorLogBoundaries; + } + public long getRepairedAt() { return sstableMetadata.repairedAt; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 03865c6ff2..2a1e9cb05c 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -38,6 +38,7 @@ import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compression.CompressionDictionaryManager; @@ -77,6 +78,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional protected long repairedAt; protected TimeUUID pendingRepair; protected boolean isTransient; + protected CoordinatorLogBoundaries coordinatorLogBoundaries; protected long maxDataAge = -1; protected final long keyCount; public final MetadataCollector metadataCollector; @@ -100,11 +102,13 @@ public abstract class SSTableWriter extends SSTable implements Transactional checkNotNull(builder.getIndexGroups()); checkNotNull(builder.getMetadataCollector()); checkNotNull(builder.getSerializationHeader()); + checkNotNull(builder.getCoordinatorLogBoundaries()); this.keyCount = builder.getKeyCount(); this.repairedAt = builder.getRepairedAt(); this.pendingRepair = builder.getPendingRepair(); this.isTransient = builder.isTransientSSTable(); + this.coordinatorLogBoundaries = builder.getCoordinatorLogBoundaries(); this.metadataCollector = builder.getMetadataCollector(); this.header = builder.getSerializationHeader(); this.mmappedRegionsCache = builder.getMmappedRegionsCache(); @@ -348,6 +352,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional repairedAt, pendingRepair, isTransient, + coordinatorLogBoundaries, header, first.retainable().getKey(), last.retainable().getKey()); @@ -454,6 +459,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional private List indexGroups; @Nullable private CompressionDictionaryManager compressionDictionaryManager; + private CoordinatorLogBoundaries coordinatorLogBoundaries; public B setMetadataCollector(MetadataCollector metadataCollector) { @@ -479,6 +485,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional return (B) this; } + public B setCoordinatorLogBoundaries(CoordinatorLogBoundaries coordinatorLogBoundaries) + { + this.coordinatorLogBoundaries = coordinatorLogBoundaries; + return (B) this; + } + public B setTransientSSTable(boolean transientSSTable) { this.transientSSTable = transientSSTable; @@ -569,6 +581,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional return transientSSTable; } + public CoordinatorLogBoundaries getCoordinatorLogBoundaries() + { + return coordinatorLogBoundaries; + } + public SerializationHeader getSerializationHeader() { return serializationHeader; diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java index 07fc965673..5b9ac4e551 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java @@ -190,7 +190,7 @@ public abstract class SortedTableScrubber imp Refs refs = Refs.ref(Collections.singleton(sstable))) { StatsMetadata metadata = sstable.getSSTableMetadata(); - writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)); + writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, transaction)); scrubInternal(writer); @@ -240,7 +240,7 @@ public abstract class SortedTableScrubber imp // out of order partitions/rows, but no bad partition found - we can keep our repairedAt time long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt; SSTableReader newInOrderSstable; - try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, sstable, transaction)) + try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogBoundaries, sstable, transaction)) { for (Partition partition : outOfOrder) inOrderWriter.append(partition.unfilteredIterator()); diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index 2b214ab56d..208a1cfd62 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -96,6 +96,8 @@ public abstract class Version */ public abstract boolean hasTokenSpaceCoverage(); + public abstract boolean hasMutationTrackingMetadata(); + /** * Records in th stats if the sstable has any partition deletions. */ diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index e6b60c2a06..f3f2754a8b 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -434,7 +434,7 @@ public class BigFormat extends AbstractSSTableFormat= 0 ? MessagingService.VERSION_50 : MessagingService.VERSION_30; + correspondingMessagingVersion = version.compareTo("ob") >= 0 ? (version.compareTo("oa") > 0 ? MessagingService.VERSION_61 : MessagingService.VERSION_60) : MessagingService.VERSION_30; hasCommitLogLowerBound = version.compareTo("mb") >= 0; hasCommitLogIntervals = version.compareTo("mc") >= 0; @@ -503,6 +505,7 @@ public class BigFormat extends AbstractSSTableFormat= 0; hasUintDeletionTime = version.compareTo("oa") >= 0; hasTokenSpaceCoverage = version.compareTo("oa") >= 0; + hasMutationTrackingMetadata = version.compareTo("ob") >= 0; } @Override @@ -589,6 +592,12 @@ public class BigFormat extends AbstractSSTableFormat 0 ? MessagingService.VERSION_61 : MessagingService.VERSION_50; + hasMutationTrackingMetadata = version.compareTo("db") >= 0; } @Override @@ -389,6 +392,12 @@ public class BtiFormat extends AbstractSSTableFormat finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey) + public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, CoordinatorLogBoundaries coordinatorLogBoundaries, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey) { assert minClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || minClustering.kind().isStart(); assert maxClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || maxClustering.kind().isEnd(); @@ -481,6 +450,7 @@ public class MetadataCollector implements PartitionStatisticsCollector pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey)); components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality)); diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index 314694a611..6b9f59adf1 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.util.Collections; import java.util.EnumMap; import java.util.EnumSet; @@ -124,17 +125,10 @@ public class MetadataSerializer implements IMetadataSerializer logger.trace("Load metadata for {}", descriptor); File statsFile = descriptor.fileFor(Components.STATS); if (!statsFile.exists()) + throw new NoSuchFileException("Stats component of sstable " + descriptor + " is missing"); + try (RandomAccessReader r = RandomAccessReader.open(statsFile)) { - logger.trace("No sstable stats for {}", descriptor); - components = new EnumMap<>(MetadataType.class); - components.put(MetadataType.STATS, MetadataCollector.defaultStatsMetadata()); - } - else - { - try (RandomAccessReader r = RandomAccessReader.open(statsFile)) - { - components = deserialize(descriptor, r, types); - } + components = deserialize(descriptor, r, types); } return components; } diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index 3ec1a208c5..330efbd14e 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.BufferClusteringBound; import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Slice; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.commitlog.CommitLogPosition; @@ -79,6 +80,7 @@ public class StatsMetadata extends MetadataComponent public final UUID originatingHostId; public final TimeUUID pendingRepair; public final boolean isTransient; + public final CoordinatorLogBoundaries coordinatorLogBoundaries; // just holds the current encoding stats to avoid allocating - it is not serialized public final EncodingStats encodingStats; @@ -122,6 +124,7 @@ public class StatsMetadata extends MetadataComponent TimeUUID pendingRepair, boolean isTransient, boolean hasPartitionLevelDeletions, + CoordinatorLogBoundaries coordinatorLogBoundaries, ByteBuffer firstKey, ByteBuffer lastKey) { @@ -147,6 +150,7 @@ public class StatsMetadata extends MetadataComponent this.originatingHostId = originatingHostId; this.pendingRepair = pendingRepair; this.isTransient = isTransient; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL); this.hasPartitionLevelDeletions = hasPartitionLevelDeletions; this.firstKey = firstKey; @@ -207,6 +211,7 @@ public class StatsMetadata extends MetadataComponent pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } @@ -236,6 +241,7 @@ public class StatsMetadata extends MetadataComponent newPendingRepair, newIsTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } @@ -269,6 +275,7 @@ public class StatsMetadata extends MetadataComponent .append(originatingHostId, that.originatingHostId) .append(pendingRepair, that.pendingRepair) .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions) + .append(coordinatorLogBoundaries, that.coordinatorLogBoundaries) .append(firstKey, that.firstKey) .append(lastKey, that.lastKey) .build(); @@ -299,6 +306,7 @@ public class StatsMetadata extends MetadataComponent .append(originatingHostId) .append(pendingRepair) .append(hasPartitionLevelDeletions) + .append(coordinatorLogBoundaries) .append(firstKey) .append(lastKey) .build(); @@ -386,6 +394,9 @@ public class StatsMetadata extends MetadataComponent size += Double.BYTES; } + if (version.hasMutationTrackingMetadata()) + size += CoordinatorLogBoundaries.serializer.serializedSize(component.coordinatorLogBoundaries, version.correspondingMessagingVersion()); + return size; } @@ -509,6 +520,9 @@ public class StatsMetadata extends MetadataComponent { out.writeDouble(component.tokenSpaceCoverage); } + + if (version.hasMutationTrackingMetadata()) + CoordinatorLogBoundaries.serializer.serialize(component.coordinatorLogBoundaries, out, version.correspondingMessagingVersion()); } private void serializeImprovedMinMax(Version version, StatsMetadata component, DataOutputPlus out) throws IOException @@ -654,6 +668,10 @@ public class StatsMetadata extends MetadataComponent tokenSpaceCoverage = in.readDouble(); } + CoordinatorLogBoundaries coordinatorLogBoundaries = CoordinatorLogBoundaries.NONE; + if (version.hasMutationTrackingMetadata()) + coordinatorLogBoundaries = CoordinatorLogBoundaries.serializer.deserialize(in, version.correspondingMessagingVersion()); + return new StatsMetadata(partitionSizes, columnCounts, commitLogIntervals, @@ -677,6 +695,7 @@ public class StatsMetadata extends MetadataComponent pendingRepair, isTransient, hasPartitionLevelDeletions, + coordinatorLogBoundaries, firstKey, lastKey); } diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLogId.java b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java index f7dd55cce7..1beac3fb79 100644 --- a/src/java/org/apache/cassandra/replication/CoordinatorLogId.java +++ b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.Serializable; import java.util.Comparator; +import com.google.common.annotations.VisibleForTesting; + public class CoordinatorLogId implements Serializable { private static final CoordinatorLogId NONE = new CoordinatorLogId(Integer.MIN_VALUE, Integer.MIN_VALUE); @@ -66,7 +68,8 @@ public class CoordinatorLogId implements Serializable return asLong(hostId, hostLogId); } - static long asLong(int hostId, int hostLogId) + @VisibleForTesting + public static long asLong(int hostId, int hostLogId) { return ((long) hostId << 32) | hostLogId; } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index a22ccfebb1..35da81443e 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -654,6 +654,9 @@ public class TableMetadata implements SchemaElement indexes.validate(this); + if (replicationType() != null && replicationType().isTracked() && params.memtable.factory().streamFromMemtable()) + except("Cannot use mutation tracking with persistent memtables"); + for (ColumnMetadata columnMetadata : columns()) { ColumnConstraints constraints = columnMetadata.getColumnConstraints(); diff --git a/src/java/org/apache/cassandra/streaming/OutgoingStream.java b/src/java/org/apache/cassandra/streaming/OutgoingStream.java index 77386a6225..b3fd1d5900 100644 --- a/src/java/org/apache/cassandra/streaming/OutgoingStream.java +++ b/src/java/org/apache/cassandra/streaming/OutgoingStream.java @@ -21,6 +21,7 @@ package org.apache.cassandra.streaming; import java.io.IOException; import java.util.List; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.TableId; @@ -48,6 +49,8 @@ public interface OutgoingStream long getRepairedAt(); TimeUUID getPendingRepair(); + CoordinatorLogBoundaries getCoordinatorLogBoundaries(); + String getName(); /** diff --git a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java index b83d7863fc..1f3c1b1ec9 100644 --- a/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java +++ b/src/java/org/apache/cassandra/streaming/messages/OutgoingStreamMessage.java @@ -74,7 +74,8 @@ public class OutgoingStreamMessage extends StreamMessage session.sessionIndex(), sequenceNumber, stream.getRepairedAt(), - stream.getPendingRepair()); + stream.getPendingRepair(), + stream.getCoordinatorLogBoundaries()); } public synchronized void serialize(StreamingDataOutputPlus out, int version, StreamSession session) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java index ec9d8e83a2..6ffa324e2e 100644 --- a/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java +++ b/src/java/org/apache/cassandra/streaming/messages/StreamMessageHeader.java @@ -21,6 +21,7 @@ import java.io.IOException; import com.google.common.base.Objects; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -46,6 +47,7 @@ public class StreamMessageHeader public final int sequenceNumber; public final long repairedAt; public final TimeUUID pendingRepair; + public final CoordinatorLogBoundaries coordinatorLogBoundaries; public final InetAddressAndPort sender; public StreamMessageHeader(TableId tableId, @@ -55,7 +57,8 @@ public class StreamMessageHeader int sessionIndex, int sequenceNumber, long repairedAt, - TimeUUID pendingRepair) + TimeUUID pendingRepair, + CoordinatorLogBoundaries coordinatorLogBoundaries) { this.tableId = tableId; this.sender = sender; @@ -65,6 +68,7 @@ public class StreamMessageHeader this.sequenceNumber = sequenceNumber; this.repairedAt = repairedAt; this.pendingRepair = pendingRepair; + this.coordinatorLogBoundaries = coordinatorLogBoundaries; } @Override @@ -119,6 +123,7 @@ public class StreamMessageHeader { header.pendingRepair.serialize(out); } + CoordinatorLogBoundaries.serializer.serialize(header.coordinatorLogBoundaries, out, version); } public StreamMessageHeader deserialize(DataInputPlus in, int version) throws IOException @@ -131,8 +136,9 @@ public class StreamMessageHeader int sequenceNumber = in.readInt(); long repairedAt = in.readLong(); TimeUUID pendingRepair = in.readBoolean() ? TimeUUID.deserialize(in) : null; + CoordinatorLogBoundaries coordinatorLogBoundaries = CoordinatorLogBoundaries.serializer.deserialize(in, version); - return new StreamMessageHeader(tableId, sender, planId, sendByFollower, sessionIndex, sequenceNumber, repairedAt, pendingRepair); + return new StreamMessageHeader(tableId, sender, planId, sendByFollower, sessionIndex, sequenceNumber, repairedAt, pendingRepair, coordinatorLogBoundaries); } public long serializedSize(StreamMessageHeader header, int version) @@ -146,6 +152,7 @@ public class StreamMessageHeader size += TypeSizes.sizeof(header.repairedAt); size += TypeSizes.sizeof(header.pendingRepair != null); size += header.pendingRepair != null ? TimeUUID.sizeInBytes() : 0; + size += CoordinatorLogBoundaries.serializer.serializedSize(header.coordinatorLogBoundaries, version); return size; } diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..ba7ce3f01a Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db new file mode 100644 index 0000000000..ac900e5814 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..048ad49aaf --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +2530067741 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Partitions.db new file mode 100644 index 0000000000..6e4273411f Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db new file mode 100644 index 0000000000..b1ffb7400f Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Rows.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Statistics.db new file mode 100644 index 0000000000..8a8a4f1f4a Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..4528bde73f Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db new file mode 100644 index 0000000000..55c580d0b9 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..024f42fedf --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +1370392555 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Partitions.db new file mode 100644 index 0000000000..6e4273411f Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db new file mode 100644 index 0000000000..b923a40bac Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Rows.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db new file mode 100644 index 0000000000..4311795c8b Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt new file mode 100644 index 0000000000..5bfa06ac54 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_be_index_summary/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Partitions.db +Rows.db +Statistics.db +TOC.txt diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..867a774cb2 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Data.db new file mode 100644 index 0000000000..3f530d2cb6 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..ac6e4ec438 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +53972413 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Partitions.db new file mode 100644 index 0000000000..6e4273411f Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db new file mode 100644 index 0000000000..43d4662852 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Rows.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Statistics.db new file mode 100644 index 0000000000..1c7f09c9d7 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_clust_counter/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..f30e3ceb79 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Data.db new file mode 100644 index 0000000000..f082ef1be0 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..fd953e56ba --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +3597571582 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Partitions.db new file mode 100644 index 0000000000..dc31ba9d1a Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Statistics.db new file mode 100644 index 0000000000..3bc02aa78a Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..0476db58d0 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db new file mode 100644 index 0000000000..6bf3d6ff53 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..687d787888 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +3514184066 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Partitions.db new file mode 100644 index 0000000000..dafb436f43 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Statistics.db new file mode 100644 index 0000000000..e8b44ee599 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_simple_counter/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db new file mode 100644 index 0000000000..5a82349dd2 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Data.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Data.db new file mode 100644 index 0000000000..aa261ff13b Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Data.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 new file mode 100644 index 0000000000..0c820c94b7 --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Digest.crc32 @@ -0,0 +1 @@ +1960712659 \ No newline at end of file diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Filter.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Partitions.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Partitions.db new file mode 100644 index 0000000000..b096fdac14 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Partitions.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Rows.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Rows.db new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Statistics.db b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Statistics.db new file mode 100644 index 0000000000..a3c99f13c9 Binary files /dev/null and b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-Statistics.db differ diff --git a/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt new file mode 100644 index 0000000000..298910cfdc --- /dev/null +++ b/test/data/legacy-sstables/db/legacy_tables/legacy_db_tuple/db-51-bti-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Statistics.db +Digest.crc32 +TOC.txt +CompressionInfo.db +Filter.db +Partitions.db +Rows.db diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..efba218e45 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db new file mode 100644 index 0000000000..fbc8ce33b2 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..f0bd564de9 --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +2672104528 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Index.db new file mode 100644 index 0000000000..de64643dd9 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db new file mode 100644 index 0000000000..ae9b9467a7 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..962fd24580 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db new file mode 100644 index 0000000000..75270dc694 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..eb2c8d0807 --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +2155890726 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Index.db new file mode 100644 index 0000000000..bd25a71d86 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db new file mode 100644 index 0000000000..1d4529efcc Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_be_index_summary/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..e661f52267 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Data.db new file mode 100644 index 0000000000..1912950610 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..a570cc66c8 --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +1361141780 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Index.db new file mode 100644 index 0000000000..4d6412ce59 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db new file mode 100644 index 0000000000..7398ca55ce Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_clust_counter/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..f30e3ceb79 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Data.db new file mode 100644 index 0000000000..0e5836f0a9 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..a83276d891 --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +1375280580 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Index.db new file mode 100644 index 0000000000..e4ed7172a1 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Statistics.db new file mode 100644 index 0000000000..1c84c471c1 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..0476db58d0 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db new file mode 100644 index 0000000000..060f1a3fa2 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..a880c21aeb --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +575257059 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Index.db new file mode 100644 index 0000000000..4cb2af35d1 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Statistics.db new file mode 100644 index 0000000000..23503f1c02 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_simple_counter/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-CompressionInfo.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-CompressionInfo.db new file mode 100644 index 0000000000..5a82349dd2 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-CompressionInfo.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Data.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Data.db new file mode 100644 index 0000000000..40a42c6bd3 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Data.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 new file mode 100644 index 0000000000..a38308e51f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Digest.crc32 @@ -0,0 +1 @@ +383358291 \ No newline at end of file diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db new file mode 100644 index 0000000000..8868e5c180 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Filter.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Index.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Index.db new file mode 100644 index 0000000000..8d7a18f061 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Index.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db new file mode 100644 index 0000000000..163fede284 Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Statistics.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db new file mode 100644 index 0000000000..9b24e0450c Binary files /dev/null and b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-Summary.db differ diff --git a/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-TOC.txt b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-TOC.txt new file mode 100644 index 0000000000..f2df0f413f --- /dev/null +++ b/test/data/legacy-sstables/ob/legacy_tables/legacy_ob_tuple/ob-51-big-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Data.db +Digest.crc32 +Filter.db +Index.db +Statistics.db +Summary.db +TOC.txt diff --git a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java index de54ca648e..ae31097f61 100644 --- a/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/ZeroCopyStreamingBench.java @@ -41,6 +41,7 @@ import org.openjdk.jmh.annotations.Warmup; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.commitlog.CommitLog; @@ -151,7 +152,7 @@ public class ZeroCopyStreamingBench blockStreamReader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, - null), entireSSTableStreamHeader, session); + null, CoordinatorLogBoundaries.NONE), entireSSTableStreamHeader, session); List> requestedRanges = Arrays.asList(new Range<>(sstable.getFirst().minValue().getToken(), sstable.getLast().getToken())); CassandraStreamHeader partialSSTableStreamHeader = @@ -173,7 +174,7 @@ public class ZeroCopyStreamingBench partialStreamReader = new CassandraStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, - null), + null, CoordinatorLogBoundaries.NONE), partialSSTableStreamHeader, session); } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index ab03d6d55f..56f35fe180 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -846,7 +846,7 @@ public class ColumnFamilyStoreTest { @Override - public long put(PartitionUpdate update, UpdateTransaction indexer, Group opGroup, boolean assumeMissing) + public long put(MutationId mutationId, PartitionUpdate update, UpdateTransaction indexer, Group opGroup, boolean assumeMissing) { return 0; } diff --git a/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java new file mode 100644 index 0000000000..93b9c41833 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesLifecycleTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.memtable.Memtable; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.MemtableParams; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableParams; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.assertj.core.api.Assertions; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +@RunWith(Parameterized.class) +public class CoordinatorLogBoundariesLifecycleTest +{ + private static final AtomicInteger keyspaceNumber = new AtomicInteger(); + + static + { + DatabaseDescriptor.daemonInitialization(); + MutationJournal.instance.start(); + } + + @BeforeClass + public static void setupClass() + { + SchemaLoader.prepareServer(); + } + + @Parameter + public String memtableConfig; + + @Parameters(name="{0}") + public static Collection memtableConfigs() + { + List memtableConfigs = new ArrayList<>(); + for (String config : DatabaseDescriptor.getMemtableConfigurations().keySet()) + { + try + { + MemtableParams.get(config); + memtableConfigs.add(config); + } + catch (ConfigurationException e) + { + // skip if the config is meant to validate config parsing + } + } + + memtableConfigs.sort(String.CASE_INSENSITIVE_ORDER); + return memtableConfigs; + } + + private static Mutation createMutation(TableMetadata tableMetadata, int k, int v) + { + DecoratedKey key = tableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes(k)); + MutationId mutationId = MutationTrackingService.instance.nextMutationId(tableMetadata.keyspace, key.getToken()); + SimpleBuilders.MutationBuilder builder = new SimpleBuilders.MutationBuilder(mutationId, tableMetadata.keyspace, key); + PartitionUpdate.SimpleBuilder partition = builder.update(tableMetadata); + partition.row().add("v", v); + Mutation mutation = builder.build(); + Assert.assertFalse(mutation.id().isNone()); + return mutation; + } + + private static MutationId applyMutation(TableMetadata tableMetadata, int k, int v) + { + Mutation mutation = createMutation(tableMetadata, k, v); + mutation.apply(); + return mutation.id(); + } + + private static void assertEmptyMemtable(View view) + { + Assert.assertEquals(0, view.getCurrentMemtable().operationCount()); + } + + private static void assertNonEmptyMemtable(View view) + { + Assert.assertTrue(view.getCurrentMemtable().operationCount() > 0); + } + + private static void assertNumSSTables(View view, int numSSTables) + { + Assert.assertEquals(numSSTables, view.liveSSTables().size()); + } + + private static String nextKeyspaceName() + { + return "ks_" + keyspaceNumber.getAndIncrement(); + } + + /** + * Test that mutation ids go from the memtable to sstables, and are combined during compaction + */ + @Test + public void mutationIdLifecycleTest() + { + String ks = nextKeyspaceName(); + String tbl = "tbl"; + TableMetadata tableMetadata = TableMetadata.builder(ks, tbl) + .addPartitionKeyColumn("k", Int32Type.instance) + .addRegularColumn("v", Int32Type.instance) + .build(); + + TableParams tableParams = tableMetadata.params; + tableParams = tableParams.unbuild().memtable(MemtableParams.get(memtableConfig)).build(); + tableMetadata = tableMetadata.withSwapped(tableParams); + + SchemaLoader.createKeyspace(ks, KeyspaceParams.simple(1, ReplicationType.tracked), tableMetadata); + + // TODO: check against all memtable types (David Capwell style) + Keyspace keyspace = Keyspace.open(ks); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(tbl); + cfs.disableAutoCompaction(); + + // pre-apply + { + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 0); + } + + MutationId id1; + MutationId id2; + // apply 1 + { + id1 = applyMutation(cfs.metadata(), 1, 1); + id2 = applyMutation(cfs.metadata(), 2, 2); + + View view = cfs.getTracker().getView(); + assertNonEmptyMemtable(view); + assertNumSSTables(view, 0); + + Memtable memtable = view.getCurrentMemtable(); + CoordinatorLogBoundaries boundaries = memtable.getFlushSet(null, null).coordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + + // flush 1 + { + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 1); + + SSTableReader sstable = Iterables.getOnlyElement(view.liveSSTables()); + CoordinatorLogBoundaries boundaries = sstable.getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + + MutationId id3; + MutationId id4; + // apply 2 + { + id3 = applyMutation(cfs.metadata(), 3, 3); + id4 = applyMutation(cfs.metadata(), 4, 4); + + View view = cfs.getTracker().getView(); + assertNonEmptyMemtable(view); + assertNumSSTables(view, 1); + + Memtable memtable = view.getCurrentMemtable(); + CoordinatorLogBoundaries boundaries = memtable.getFlushSet(null, null).coordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + + // flush 2 + { + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 2); + + List sstables = Lists.newArrayList(view.liveSSTables()); + sstables.sort(Comparator.comparing(sst -> sst.descriptor.id.asBytes())); + { + CoordinatorLogBoundaries boundaries = sstables.get(0).getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id2.logId())).isEqualTo(id2); + } + { + + CoordinatorLogBoundaries boundaries = sstables.get(1).getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + } + + // compaction + { + cfs.forceMajorCompaction(); + + View view = cfs.getTracker().getView(); + assertEmptyMemtable(view); + assertNumSSTables(view, 1); + + SSTableReader sstable = Iterables.getOnlyElement(view.liveSSTables()); + CoordinatorLogBoundaries boundaries = sstable.getCoordinatorLogBoundaries(); + Assertions.assertThat(boundaries.size()).isEqualTo(1); + Assertions.assertThat(boundaries.max(id4.logId())).isEqualTo(id4); + } + } +} diff --git a/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java new file mode 100644 index 0000000000..a62efd3484 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CoordinatorLogBoundariesTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import org.junit.Test; + +import accord.utils.Gen; +import accord.utils.Gens; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationId; +import org.assertj.core.api.Assertions; + +import static accord.utils.Property.qt; + +public class CoordinatorLogBoundariesTest +{ + private static final Gen LOG_ID_GEN = rs -> { + int hostId = rs.nextInt(1, 4); + int hostLogId = rs.nextInt(1, 11); + return CoordinatorLogId.asLong(hostId, hostLogId); + }; + + private static final Gen SEQUENCE_ID_GEN = rs -> { + int offset = rs.nextBiasedInt(1, 10_000, 1_000_000); + return MutationId.sequenceId(offset, offset); + }; + + private static final Gen MUTATION_ID_GEN = rs -> new MutationId(LOG_ID_GEN.next(rs), SEQUENCE_ID_GEN.next(rs)); + + private static final Gen COORDINATOR_LOG_BOUNDARIES_GEN = rs -> { + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + int numIds = rs.nextBiasedInt(0, 10, 1000); + for (int i = 0; i < numIds; i++) + boundaries.add(MUTATION_ID_GEN.next(rs)); + return boundaries; + }; + + @Test + public void roundtripSerde() + { + qt() + .forAll(COORDINATOR_LOG_BOUNDARIES_GEN) + .check(boundaries -> { + try (DataOutputBuffer outputBuffer = DataOutputBuffer.scratchBuffer.get()) + { + CoordinatorLogBoundaries.serializer.serialize(boundaries, outputBuffer, MessagingService.current_version); + byte[] bytes = outputBuffer.toByteArray(); + try (DataInputBuffer inputBuffer = new DataInputBuffer(bytes)) + { + CoordinatorLogBoundaries deserialized = CoordinatorLogBoundaries.serializer.deserialize(inputBuffer, MessagingService.current_version); + Assertions.assertThat(boundaries).isEqualTo(deserialized); + Assertions.assertThat(bytes.length).isEqualTo(CoordinatorLogBoundaries.serializer.serializedSize(boundaries, MessagingService.current_version)); + } + } + }); + } + + @Test + public void monotonicAdd() + { + qt() + .forAll(Gens.lists(MUTATION_ID_GEN).ofSizeBetween(3, 100)) + .check(ids -> { + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(); + for (MutationId id : ids) + { + int originalOffset = boundaries.maxOffset(id.logId()); + boundaries.add(id); + int updatedOffset = boundaries.maxOffset(id.logId()); + Assertions.assertThat(updatedOffset).isGreaterThanOrEqualTo(originalOffset); + Assertions.assertThat(updatedOffset).isEqualTo(Math.max(originalOffset, id.offset())); + } + }); + } + + @Test + public void monotonicMerge() + { + qt() + .forAll(COORDINATOR_LOG_BOUNDARIES_GEN, COORDINATOR_LOG_BOUNDARIES_GEN) + .check((left, right) -> { + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + boundaries.addAll(left); + boundaries.addAll(right); + CoordinatorLogBoundaries merged = boundaries; + for (Long logId : merged) + { + int leftOffset = left.maxOffset(logId); + int rightOffset = right.maxOffset(logId); + int mergedOffset = merged.maxOffset(logId); + Assertions.assertThat(mergedOffset).isGreaterThanOrEqualTo(leftOffset); + Assertions.assertThat(mergedOffset).isGreaterThanOrEqualTo(rightOffset); + Assertions.assertThat(mergedOffset).isIn(leftOffset, rightOffset); + } + }); + } + + @Test + public void builderEquivalentToMutable() + { + qt() + .forAll(Gens.lists(MUTATION_ID_GEN).ofSizeBetween(3, 100)) + .check(ids -> { + CoordinatorLogBoundariesMap boundaries = new CoordinatorLogBoundariesMap(); + CoordinatorLogBoundariesBuilder builder = new CoordinatorLogBoundariesBuilder(); + for (MutationId id : ids) + { + boundaries.add(id); + builder.add(id); + } + + CoordinatorLogBoundaries fromBuilder = builder.build(); + Assertions.assertThat(fromBuilder).hasSize(boundaries.size()); + for (Long logId : boundaries) + Assertions.assertThat(fromBuilder.max(logId)).isEqualTo(boundaries.max(logId)); + }); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java index 33f1a63238..4008472cee 100644 --- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java +++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java @@ -101,6 +101,7 @@ public class SerializationHeaderTest .setTableMetadataRef(TableMetadataRef.forOfflineTools(schema)) .setKeyCount(1) .setSerializationHeader(header) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .setMetadataCollector(new MetadataCollector(schema.comparator)) .addDefaultComponents(Collections.emptySet()) .build(txn, null)) diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java index cb55129909..2067b9b4ef 100644 --- a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java @@ -277,10 +277,11 @@ public class AntiCompactionTest private SSTableReader writeFile(ColumnFamilyStore cfs, int count) { + assert !cfs.metadata().replicationType().isTracked(); File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { for (int i = 0; i < count; i++) { diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index 79ac2bde96..8270378b25 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -43,6 +43,7 @@ import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SerializationHeader; @@ -1444,7 +1445,7 @@ public class LogTransactionTest extends AbstractTransactionalTest DecoratedKey key = MockSchema.readerBounds(generation); SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, CoordinatorLogBoundaries.NONE, header, key.getKey().slice(), key.getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) @@ -1480,7 +1481,7 @@ public class LogTransactionTest extends AbstractTransactionalTest DecoratedKey key = MockSchema.readerBounds(generation); SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, CoordinatorLogBoundaries.NONE, header, key.getKey().slice(), key.getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java index 6031e4e611..a64dbdb064 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -167,6 +168,7 @@ public class RealTransactionsTest extends SchemaLoader .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) .addDefaultComponents(cfs.indexManager.listIndexGroups()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs)); while (ci.hasNext()) { diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java index 03ac14f364..e4d99f295a 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java @@ -63,6 +63,7 @@ import static java.util.Collections.singleton; public class TrackerTest { + private static final int COMPONENT_STATS_SIZE_BYTES = 4805; private static final class MockListener implements INotificationConsumer { @@ -156,7 +157,7 @@ public class TrackerTest if (reader instanceof KeyCacheSupport) Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); - Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(17 + 121 + 9 + (COMPONENT_STATS_SIZE_BYTES * 3), cfs.metric.liveDiskSpaceUsed.getCount()); } @Test @@ -181,7 +182,7 @@ public class TrackerTest Assert.assertTrue(((KeyCacheSupport)reader).getKeyCache().isEnabled()); } - Assert.assertEquals(17 + 121 + 9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(17 + 121 + 9 + (COMPONENT_STATS_SIZE_BYTES * 3), cfs.metric.liveDiskSpaceUsed.getCount()); Assert.assertEquals(1, listener.senders.size()); Assert.assertEquals(1, listener.received.size()); Assert.assertEquals(tracker, listener.senders.get(0)); @@ -220,8 +221,8 @@ public class TrackerTest tracker.dropSSTables(); LogTransaction.waitForDeletions(); } - Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount()); - Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.totalDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.liveDiskSpaceUsed.getCount()); Assert.assertEquals(1, tracker.getView().sstables.size()); } if (!invalidate) @@ -249,7 +250,7 @@ public class TrackerTest Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(2)).deleting); Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(3)).removed.size()); Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(3)).added.size()); - Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 9, cfs.metric.liveDiskSpaceUsed.getCount()); readers.get(0).selfRef().release(); } else @@ -313,7 +314,7 @@ public class TrackerTest listener.received.clear(); if (reader instanceof KeyCacheSupport) Assert.assertTrue(((KeyCacheSupport) reader).getKeyCache().isEnabled()); - Assert.assertEquals(10, cfs.metric.liveDiskSpaceUsed.getCount()); + Assert.assertEquals(COMPONENT_STATS_SIZE_BYTES + 10, cfs.metric.liveDiskSpaceUsed.getCount()); // test invalidated CFS cfs = MockSchema.newCFS(); diff --git a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java index 3cb982b89b..081a10d48f 100644 --- a/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamWriterTest.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Queue; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.BeforeClass; import org.junit.Test; @@ -163,7 +164,7 @@ public class CassandraEntireSSTableStreamWriterTest .withTableId(sstable.metadata().id) .build(); - CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null), header, session); + CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null, CoordinatorLogBoundaries.NONE), header, session); SSTableTxnSingleStreamWriter sstableWriter = (SSTableTxnSingleStreamWriter) reader.read(new DataInputBuffer(serializedFile.nioBuffer(), false)); StreamingLifecycleTransaction stt = new StreamingLifecycleTransaction(); diff --git a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java index 372cd1603a..3ed00434b1 100644 --- a/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java +++ b/test/unit/org/apache/cassandra/db/streaming/EntireSSTableStreamConcurrentComponentMutationTest.java @@ -44,6 +44,7 @@ import org.junit.runner.RunWith; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; import org.apache.cassandra.db.compaction.CompactionManager; @@ -234,7 +235,7 @@ public class EntireSSTableStreamConcurrentComponentMutationTest concurrentMutations.get(3, TimeUnit.MINUTES); session.prepareReceiving(new StreamSummary(sstable.metadata().id, emptyList(), 1, 5104)); - StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null); + StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null, CoordinatorLogBoundaries.NONE); try (DataInputBuffer in = new DataInputBuffer(serializedFile.nioBuffer(), false)) { diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java index 5833819267..b937ac0fd4 100644 --- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java +++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java @@ -71,6 +71,7 @@ import org.apache.cassandra.index.TargetParser; import org.apache.cassandra.index.transactions.IndexTransaction; import org.apache.cassandra.io.sstable.ReducingKeyIterator; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.TableMetadata; @@ -469,7 +470,7 @@ public class CustomCassandraIndex implements Index cell)); Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info); PartitionUpdate upd = partitionUpdate(valueKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.debug("Inserted entry into index for value {}", valueKey); } @@ -515,7 +516,7 @@ public class CustomCassandraIndex implements Index { Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion)); PartitionUpdate upd = partitionUpdate(indexKey, row); - indexCfs.getWriteHandler().write(upd, ctx, false); + indexCfs.getWriteHandler().write(MutationId.fixme(), upd, ctx, false); logger.debug("Removed index entry for value {}", indexKey); } diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java index a9a304787b..bbf9d1a29c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java @@ -1454,10 +1454,10 @@ public abstract class CQLSSTableWriterTest } CQLSSTableWriter writer = builder.build(); - int rowCount = 30_000; + final long MiB_1_17 = 1_226_833; // Max SSTable size is 1 MiB - // 30_000 rows should take 30_000 * (4 + 37) = 1.17 MiB > 1 MiB, i.e. producing 2 sstables - for (int i = 0; i < rowCount; i++) + // write 1.17 MiB of data, producing 2 sstables + for (int i = 0; writer.bytesWritten() < MiB_1_17; i++) { writer.addRow(i, UUID.randomUUID().toString()); if (closeWriterOnFirstProduced && !produced.isEmpty()) diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 2c80779a29..c9049c2939 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -118,7 +118,7 @@ public class LegacySSTableTest // Get all versions up to the current one. Useful for testing in compatibility mode C18301 private static String[] getValidLegacyVersions() { - String[] versions = {"da", "oa", "nb", "na", "me", "md", "mc", "mb", "ma"}; + String[] versions = {"ob", "oa", "db", "da", "nb", "na", "me", "md", "mc", "mb", "ma"}; return Arrays.stream(versions).filter((v) -> v.compareTo(BigFormat.getInstance().getLatestVersion().toString()) <= 0).toArray(String[]::new); } diff --git a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java index d80377f5cc..c6601491d2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/RangeAwareSSTableWriterTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.BeforeClass; import org.junit.Test; @@ -75,6 +76,7 @@ public class RangeAwareSSTableWriterTest 0, null, false, + CoordinatorLogBoundaries.NONE, DatabaseDescriptor.getSelectedSSTableFormat(), 0, 0, diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java index 8c07d3bec6..15e692d84a 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java @@ -39,6 +39,7 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.SerializationHeader; @@ -133,7 +134,8 @@ public class SSTableFlushObserverTest .setMetadataCollector(new MetadataCollector(cfm.comparator).sstableLevel(0)) .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS)) .setSecondaryIndexGroups(Collections.singleton(indexGroup)) - .addDefaultComponents(Collections.emptySet()); + .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE); assertThat(observer.beginCalled).isFalse(); assertThat(observer.isComplete).isFalse(); @@ -218,6 +220,7 @@ public class SSTableFlushObserverTest .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS)) .setSecondaryIndexGroups(List.of(indexGroup1, indexGroup2)) .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(transaction, null) ).withMessage("Failed to initialize"); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java index 69414329cc..d6c6c747d2 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.Util; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; @@ -939,7 +940,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase File dir = cfs.getDirectories().getDirectoryForNewSSTables(); Descriptor desc = cfs.newSSTableDescriptor(dir); - try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) + try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))) { int end = f == fileCount - 1 ? partitionCount : ((f + 1) * partitionCount) / fileCount; for ( ; i < end ; i++) diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java index c3729901ef..59cb4d14ce 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java @@ -232,8 +232,10 @@ public class SSTableUtils File datafile = (dest == null) ? tempSSTableFile(ksname, cfname, id) : dest.fileFor(Components.DATA); TableMetadata metadata = Schema.instance.getTableMetadata(ksname, cfname); ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.id); + if (cfs.metadata().replicationType().isTracked()) + throw new IllegalStateException("Can't create writer for table with mutation tracking enabled"); SerializationHeader header = appender.header(); - SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, header); + SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, Descriptor.fromFileWithComponent(datafile, false).left, expectedSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, CoordinatorLogBoundaries.NONE, header); while (appender.append(writer)) { /* pass */ } Collection readers = writer.finish(true); diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java index 49523811cb..23e5811d77 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java @@ -35,6 +35,7 @@ import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.CompactionManager; @@ -168,6 +169,7 @@ public class SSTableWriterTestBase extends SchemaLoader .setSecondaryIndexGroups(cfs.indexManager.listIndexGroups()) .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator)) .addDefaultComponents(cfs.indexManager.listIndexGroups()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs); } diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java index b439b9c980..0e239e2c34 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTransactionTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.io.sstable; import java.io.IOException; import java.util.Collection; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.junit.Assert; import org.junit.BeforeClass; @@ -72,7 +73,7 @@ public class SSTableWriterTransactionTest extends AbstractTransactionalTest private TestableBTW(Descriptor desc) { - this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, + this(desc, SSTableTxnWriter.create(cfs, desc, 0, 0, null, false, CoordinatorLogBoundaries.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))); diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java index f641be4149..e4b977e7d4 100644 --- a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java @@ -64,6 +64,7 @@ import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.CounterMutation; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Keyspace; @@ -842,6 +843,7 @@ public class ScrubTest .setMetadataCollector(collector) .setSerializationHeader(header) .addDefaultComponents(Collections.emptySet()) + .setCoordinatorLogBoundaries(CoordinatorLogBoundaries.NONE) .build(txn, cfs); return new TestMultiWriter(writer, txn); diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 4bb7478587..ac0a676039 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.MutableCoordinatorLogBoundaries; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; @@ -50,6 +51,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.Throwables; @@ -154,7 +156,10 @@ public class MetadataSerializerTest collector.updateClusteringValues(Clustering.make(UTF8Type.instance.decompose("cba"), withNulls ? null : Int32Type.instance.decompose(234))); ByteBuffer first = AsciiType.instance.decompose("a"); ByteBuffer last = AsciiType.instance.decompose("b"); - return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last); + MutableCoordinatorLogBoundaries boundaries = MutableCoordinatorLogBoundaries.create(); + boundaries.add(new MutationId(1, 12345)); + boundaries.add(new MutationId(2, 56789)); + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, boundaries, SerializationHeader.make(cfm, Collections.emptyList()), first, last); } private void testVersions(List versions) throws Throwable diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 860f3e6ee5..dc9fa5705d 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -21,6 +21,7 @@ package org.apache.cassandra.schema; import java.io.IOException; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -40,6 +41,7 @@ import org.apache.cassandra.Util; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; @@ -53,6 +55,7 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.StatsComponent; import org.apache.cassandra.io.sstable.format.big.BigFormat; import org.apache.cassandra.io.sstable.format.big.BigFormat.Components; import org.apache.cassandra.io.sstable.format.big.BigTableReader; @@ -62,6 +65,7 @@ import org.apache.cassandra.io.sstable.format.bti.PartitionIndex; import org.apache.cassandra.io.sstable.indexsummary.IndexSummary; import org.apache.cassandra.io.sstable.keycache.KeyCache; import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.DataOutputStreamPlus; @@ -184,6 +188,7 @@ public class MockSchema public static SSTableReader sstable(int generation, int size, boolean keepRef, long firstToken, long lastToken, int level, ColumnFamilyStore cfs, int minLocalDeletionTime, long timestamp) { + assert !cfs.metadata().replicationType().isTracked(); SSTableFormat format = DatabaseDescriptor.getSelectedSSTableFormat(); Descriptor descriptor = new Descriptor(cfs.getDirectories().getDirectoryForNewSSTables(), cfs.getKeyspaceName(), @@ -193,7 +198,7 @@ public class MockSchema if (BigFormat.is(format)) { - Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC); + Set components = ImmutableSet.of(Components.DATA, Components.PRIMARY_INDEX, Components.FILTER, Components.TOC, Components.STATS); for (Component component : components) { File file = descriptor.fileFor(component); @@ -208,17 +213,26 @@ public class MockSchema collector.update(DeletionTime.build(timestamp, minLocalDeletionTime)); BufferDecoratedKey first = readerBounds(firstToken); BufferDecoratedKey last = readerBounds(lastToken); - StatsMetadata metadata = - (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), - 0.01f, - UNREPAIRED_SSTABLE, - null, - false, - header, - first.retainable().getKey().slice(), - last.retainable().getKey().slice()) - .get(MetadataType.STATS); + Map metadataComponents = collector.sstableLevel(level) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), + 0.01f, + UNREPAIRED_SSTABLE, + null, + false, + CoordinatorLogBoundaries.NONE, + header, + first.retainable().getKey().slice(), + last.retainable().getKey().slice()); + StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS); + try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE)) + { + new StatsComponent(metadataComponents).save(descriptor); + out.flush(); + } + catch (IOException ioe) + { + throw new RuntimeException(ioe); + } BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) .setDataFile(fileHandle.sharedCopy()) @@ -226,7 +240,7 @@ public class MockSchema .setIndexSummary(indexSummary.sharedCopy()) .setFilter(FilterFactory.AlwaysPresent) .setMaxDataAge(1L) - .setStatsMetadata(metadata) + .setStatsMetadata(statsMetadata) .setOpenReason(SSTableReader.OpenReason.NORMAL) .setSerializationHeader(header) .setFirst(first) @@ -257,7 +271,7 @@ public class MockSchema BufferDecoratedKey first = readerBounds(firstToken); BufferDecoratedKey last = readerBounds(lastToken); StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, first.retainable().getKey(), last.retainable().getKey()) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, CoordinatorLogBoundaries.NONE, header, first.retainable().getKey(), last.retainable().getKey()) .get(MetadataType.STATS); BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) diff --git a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java index 81645d90ea..5b872f9ddb 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.ServerTestUtils; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.BufferDecoratedKey; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; @@ -464,7 +465,8 @@ public class StreamReaderTest fakeSession, fakeSeq, System.currentTimeMillis(), - pendingRepair); + pendingRepair, + CoordinatorLogBoundaries.NONE); } private static CassandraStreamHeader streamMessageHeader(int...tokens) @@ -503,9 +505,9 @@ public class StreamReaderTest super(header, streamHeader, session); } - protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, SSTableFormat format) throws IOException + protected SSTableTxnSingleStreamWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, TimeUUID pendingRepair, CoordinatorLogBoundaries coordinatorLogBoundaries, SSTableFormat format) throws IOException { - return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, coordinatorLogBoundaries, format); } @Override diff --git a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java index 635970024c..840ddd6241 100644 --- a/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java +++ b/test/unit/org/apache/cassandra/streaming/async/StreamingInboundHandlerTest.java @@ -27,7 +27,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.CoordinatorLogBoundaries; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputBuffer; @@ -63,7 +64,7 @@ public class StreamingInboundHandlerTest @BeforeClass public static void before() { - DatabaseDescriptor.daemonInitialization(); + SchemaLoader.prepareServer(); } @Before @@ -115,7 +116,7 @@ public class StreamingInboundHandlerTest public void StreamDeserializingTask_deserialize_ISM_NoSession() throws IOException { StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, nextTimeUUID(), true, - 0, 0, 0, nextTimeUUID()); + 0, 0, 0, nextTimeUUID(), CoordinatorLogBoundaries.NONE); ByteBuffer temp = ByteBuffer.allocate(1024); DataOutputPlus out = new DataOutputBuffer(temp); @@ -134,7 +135,7 @@ public class StreamingInboundHandlerTest StreamResultFuture future = StreamResultFuture.createFollower(0, planId, StreamOperation.REPAIR, REMOTE_ADDR, streamingChannel, MessagingService.current_version, nextTimeUUID(), PreviewKind.ALL); StreamManager.instance.registerFollower(future); StreamMessageHeader header = new StreamMessageHeader(TableId.generate(), REMOTE_ADDR, planId, false, - 0, 0, 0, nextTimeUUID()); + 0, 0, 0, nextTimeUUID(), CoordinatorLogBoundaries.NONE); // IncomingStreamMessage.serializer.deserialize StreamSession session = StreamManager.instance.findSession(header.sender, header.planId, header.sessionIndex, header.sendByFollower);